@torrent-tv/proxy 2.91.0 → 2.92.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. package/bin/cli.js +3 -2
  2. package/package.json +18 -2
  3. package/routes/api/playback-plan/post.js +21 -4
  4. package/routes/api/sources/files/get.js +20 -28
  5. package/routes/api/sources/post.js +4 -2
  6. package/routes/api/sources/stats/get.js +4 -11
  7. package/routes/api/sources/viewer/post.js +16 -0
  8. package/routes/api/sources/warm/post.js +6 -215
  9. package/routes/api/subtitles/get.js +22 -125
  10. package/routes/api/subtitles/post.js +26 -0
  11. package/routes/api/transcode-sessions/post.js +24 -2
  12. package/routes/media/get.js +80 -0
  13. package/routes/stream/get.js +26 -94
  14. package/routes/transcode/audio-warm/get.js +9 -21
  15. package/routes/transcode/variant-warm/get.js +8 -14
  16. package/server.js +410 -46
  17. package/services/encode/AdmittedInput.js +136 -0
  18. package/services/encode/CushionReport.js +1 -76
  19. package/services/encode/EncodeInputs.js +142 -0
  20. package/services/encode/EncodeOrchestrator.js +4 -1
  21. package/services/encode/EncodeRun.js +13 -4
  22. package/services/encode/EncodeRuns.js +132 -52
  23. package/services/encode/InputFailures.js +37 -0
  24. package/services/encode/MatroskaInput.js +158 -0
  25. package/services/encode/OutputOpening.js +16 -64
  26. package/services/encode/Renditions.js +1 -17
  27. package/services/encode/admitted-command.js +60 -0
  28. package/services/encode/annex-b-input.js +37 -0
  29. package/services/encode/bit-payload.js +29 -0
  30. package/services/encode/output/OutputCatalog.js +1 -1
  31. package/services/encode/output/Timeline.js +11 -0
  32. package/services/encode/pcm-normalization.js +30 -0
  33. package/services/encode/piece-completeness.js +54 -1
  34. package/services/encode/quality/HostLoad.js +19 -73
  35. package/services/encode/quality/QualityOffer.js +1 -1
  36. package/services/encode/run-command.js +1 -589
  37. package/services/encode/segment-formats/mp4-boxes.js +3 -1
  38. package/services/encode/segment-formats/presentation-segment.js +114 -0
  39. package/services/media/ContainerOrchestrator.js +118 -48
  40. package/services/media/KeyframeTables.js +16 -115
  41. package/services/media/MediaReadRequests.js +120 -0
  42. package/services/media/ProbeRequests.js +135 -0
  43. package/services/media/SegmentInputs.js +48 -0
  44. package/services/media/SourceFile.js +6 -0
  45. package/services/media/SubtitleOrchestrator.js +190 -4
  46. package/services/media/await-plan.js +47 -0
  47. package/services/media/container/AsfContainer.js +217 -0
  48. package/services/media/container/AviContainer.js +343 -34
  49. package/services/media/container/CadencedPictureOrder.js +45 -0
  50. package/services/media/container/Container.js +7 -3
  51. package/services/media/container/ContainerFactory.js +10 -3
  52. package/services/media/container/FfprobeContainer.js +222 -0
  53. package/services/media/container/MatroskaContainer.js +72 -46
  54. package/services/media/container/Mp4Container.js +472 -139
  55. package/services/media/container/MpegPsContainer.js +281 -0
  56. package/services/media/container/MpegTsContainer.js +277 -0
  57. package/services/media/container/PacketIndex.js +270 -0
  58. package/services/media/container/PacketRecords.js +237 -0
  59. package/services/media/container/RetainedBytes.js +39 -0
  60. package/services/media/container/RetainedReads.js +51 -0
  61. package/services/media/container/ac3-frame.js +46 -0
  62. package/services/media/container/asf-packets.js +153 -0
  63. package/services/media/container/avc-picture-order.js +50 -0
  64. package/services/media/container/avi-open-dml.js +47 -0
  65. package/services/media/container/dts-frame.js +51 -0
  66. package/services/media/container/ebml-stream.js +12 -1
  67. package/services/media/container/ffprobe-record.js +45 -0
  68. package/services/media/container/flv-packet-ranges.js +27 -0
  69. package/services/media/container/following-packet-time.js +37 -0
  70. package/services/media/container/h264-configuration.js +183 -0
  71. package/services/media/container/h264-elementary-index.js +245 -0
  72. package/services/media/container/hevc-configuration.js +165 -0
  73. package/services/media/container/hevc-elementary-index.js +132 -0
  74. package/services/media/container/hevc-picture-order.js +66 -0
  75. package/services/media/container/latm-frame.js +99 -0
  76. package/services/media/container/matroska-packets.js +207 -0
  77. package/services/media/container/memory-unavailable.js +8 -0
  78. package/services/media/container/mp3-packet-dependencies.js +41 -0
  79. package/services/media/container/mp4-fragments.js +148 -0
  80. package/services/media/container/mpeg-audio-frame.js +19 -0
  81. package/services/media/container/mpeg-audio-reservoir.js +27 -0
  82. package/services/media/container/mpeg-elementary-index.js +196 -0
  83. package/services/media/container/mpeg-sequence-reader.js +69 -0
  84. package/services/media/container/mpeg-ts-packets.js +93 -0
  85. package/services/media/container/mpeg-video-configuration.js +50 -0
  86. package/services/media/container/mpeg4-picture-timing.js +108 -0
  87. package/services/media/container/pes-header.js +34 -0
  88. package/services/media/playback-declarations.js +57 -0
  89. package/services/media/playback-planner.js +72 -230
  90. package/services/media/probe-packets.js +61 -0
  91. package/services/media/read-playback-declarations.js +18 -0
  92. package/services/media/tracks/AudioTrack.js +9 -0
  93. package/services/media/tracks/TextSubtitleTrack.js +4 -1
  94. package/services/server/OutputLifecycle.js +2 -0
  95. package/services/server/SegmentServing.js +40 -33
  96. package/services/server/ViewerRequests.js +22 -41
  97. package/services/server/controllers/PlaybackController.js +4 -0
  98. package/services/server/controllers/SubtitleController.js +8 -29
  99. package/services/server/transcode-session-files.js +58 -99
  100. package/services/server/wire-outputs.js +58 -15
  101. package/services/storage/IndexMemory.js +85 -0
  102. package/services/storage/MachineBudget.js +27 -3
  103. package/services/storage/allowance.js +8 -1
  104. package/services/storage/piece-store/piece-disk-store.js +18 -3
  105. package/services/storage/piece-store/piece-lru.js +8 -4
  106. package/services/storage/piece-store/shared-piece-store.js +26 -11
  107. package/services/storage/segment-store/SegmentStore.js +49 -18
  108. package/services/storage/wire.js +7 -0
  109. package/services/torrent/Contents.js +52 -20
  110. package/services/torrent/demand/DemandRegister.js +13 -4
  111. package/services/torrent/demand/Window.js +13 -2
  112. package/services/torrent/demand-progress.js +25 -0
  113. package/services/torrent/download/DeadlineScheduler.js +167 -0
  114. package/services/torrent/download/FutureDownload.js +166 -0
  115. package/services/torrent/download/SwarmSelection.js +30 -59
  116. package/services/torrent/download/registry.js +45 -5
  117. package/services/torrent/download/withdraw-claim.js +2 -0
  118. package/services/torrent/torrent-pool.js +39 -600
  119. package/services/torrent/worker/channel.js +8 -1
  120. package/services/torrent/worker/client.js +74 -72
  121. package/services/torrent/worker/files-in-use.js +9 -0
  122. package/services/torrent/worker/handle-state.js +12 -0
  123. package/services/torrent/worker/held-bytes.js +82 -99
  124. package/services/torrent/worker/held-file-bytes.js +17 -0
  125. package/services/torrent/worker/piece-reader.js +67 -683
  126. package/services/torrent/worker/pool-adapter.js +37 -123
  127. package/services/torrent/worker/protocol.js +6 -17
  128. package/services/torrent/worker/whole-file-stats.js +2 -0
  129. package/services/torrent/worker/worker.js +86 -156
  130. package/services/transport/data-channel-handler.js +56 -13
  131. package/services/viewer/DownloadMaps.js +168 -0
  132. package/services/viewer/NativeSourceMap.js +33 -0
  133. package/services/viewer/PriorityMap.js +49 -40
  134. package/services/viewer/PriorityOrchestrator.js +42 -13
  135. package/services/viewer/SourcePreparation.js +278 -0
  136. package/services/viewer/Viewer.js +11 -0
  137. package/services/viewer/Viewers.js +109 -2
  138. package/services/viewer/playback-readiness.js +50 -63
  139. package/services/viewer/segment-demands.js +37 -0
  140. package/services/viewer/urgent-output-ready.js +26 -0
  141. package/services/encode/run-inputs.js +0 -53
  142. package/services/media/keyframe-probe.js +0 -157
  143. package/services/media/media-info-probe.js +0 -93
  144. package/services/torrent/worker/background-fill.js +0 -213
  145. package/services/torrent/worker/resume-warm.js +0 -107
@@ -14,6 +14,7 @@ import { waits } from "./WaitLedger.js";
14
14
  import { readMachineState, readProcessCpuSeconds, readProxyCpuSeconds, readSystemCpu, shareOfMachine } from "../encode/host-load.js";
15
15
  import { minimumBufferFrom } from "../torrent/supply-margin.js";
16
16
  import { PriorityOrchestrator } from "../viewer/PriorityOrchestrator.js";
17
+ import { urgentOutputsReady } from "../viewer/urgent-output-ready.js";
17
18
  import { softwareDescriptor } from "../encode/hwaccel.js";
18
19
  import { resolveSegmentFormat } from "../encode/segment-formats/index.js";
19
20
  import { Timelines } from "../encode/output/Timeline.js";
@@ -74,6 +75,7 @@ import { audioRenditionName } from "../media/audio-inventory.js";
74
75
  import { Renditions } from "../encode/Renditions.js";
75
76
  import { CushionReport, LOOKAHEAD_PAUSE_SECONDS } from "../encode/CushionReport.js";
76
77
  import { EncodeRuns } from "../encode/EncodeRuns.js";
78
+ import { EncodeInputs } from "../encode/EncodeInputs.js";
77
79
  import { OutputTimes } from "../encode/OutputTimes.js";
78
80
  import { HostLoad } from "../encode/quality/HostLoad.js";
79
81
  import { HostTimings } from "../encode/quality/HostTimings.js";
@@ -87,8 +89,6 @@ import { OutputSpec } from "../encode/output/OutputSpec.js";
87
89
  import { compareInits } from "../encode/segment-formats/init-compat.js";
88
90
  import { nominalKbpsFor, AUDIO_TRANSCODE_KBPS } from "../encode/args.js";
89
91
  import { audioReadingFor } from "../encode/audio-work-rate.js";
90
- import { probeInputMediaInfo } from "../media/media-info-probe.js";
91
- import { probeVideoKeyframeTimes } from "../media/keyframe-probe.js";
92
92
  import { wireMachineBudget } from "../storage/wire.js";
93
93
  import { Returns } from "../storage/returns.js";
94
94
  import { freeBytesFor } from "../storage/free.js";
@@ -217,7 +217,6 @@ export function wireOutputs({
217
217
  getCachedMediaInfo = null,
218
218
  getCachedAudioTracks = null,
219
219
  getContainerMediaInfo = null,
220
- fetchWholeFile = null,
221
220
  segmentFormatId = undefined,
222
221
  stateDir = "",
223
222
  segmentStore = null,
@@ -228,7 +227,15 @@ export function wireOutputs({
228
227
  diagnostics = null,
229
228
  diagnosticsRoot = "",
230
229
  memoryClaimant = null,
231
- budgetPolicy = null}) {
230
+ resolveEncodeInput = null,
231
+ readSourceMedia = null,
232
+ readMetadataActivity = null,
233
+ sourceInputsFor = null,
234
+ readEncodeRanges = null,
235
+ encodeInputs = null,
236
+ indexMemory = null,
237
+ budgetPolicy = null,
238
+ onViewerChanged = null}) {
232
239
  const parts = {};
233
240
  const audioDescriptionOf = (output) => (getCachedAudioTracks?.({
234
241
  sourceKey: output.file.sourceKey, fileIndex: output.spec.grid?.fileIndex ?? output.file.fileIndex
@@ -284,6 +291,7 @@ export function wireOutputs({
284
291
  parts.limitsFor = (frame) => [nominalKbpsFor(frame)];
285
292
  // What the torrent and the proxy itself spend on this host, and how fast each watched torrent moves.
286
293
  parts.hostLoad = new HostLoad({
294
+ readMetadataActivity,
287
295
  readMachineState,
288
296
  readProcessCpuSeconds,
289
297
  readProxyCpuSeconds,
@@ -326,7 +334,7 @@ export function wireOutputs({
326
334
  });
327
335
  },
328
336
  viewerSecondsOn: (output, consumerId, now) => viewerSecondsOn(parts.viewers, output, consumerId, now),
329
- inputOf: (...args) => parts.renditions.inputOf(...args),
337
+ get encodeInputs() { return parts.encodeInputs; },
330
338
  producedNumbers: (...args) => parts.serving.producedNumbers(...args),
331
339
  servesAudioSeparately: (...args) => parts.renditions.servesAudioSeparately(...args),
332
340
  disposeSession: (...args) => parts.lifecycle.disposeSession(...args),
@@ -352,7 +360,6 @@ export function wireOutputs({
352
360
  logger,
353
361
  producedNumbers: (...args) => parts.serving.producedNumbers(...args),
354
362
  get encodeRuns() { return parts.encodeRuns; },
355
- get fetchWholeFile() { return parts.fetchWholeFile; },
356
363
  get getCachedAudioTracks() { return parts.getCachedAudioTracks; },
357
364
  get hostLoad() { return parts.hostLoad; },
358
365
  get lookaheadSeconds() { return parts.lookaheadSeconds; },
@@ -523,8 +530,8 @@ export function wireOutputs({
523
530
  // carrying here: the only figure an encoder with no bound of its own has.
524
531
  observedPeakMbps: (spec) => parts.localObservations?.peakMbps(spec) ?? null,
525
532
  limitsFor: (frame) => parts.limitsFor(frame),
526
- probeMediaInfo: (url) => probeInputMediaInfo(parts.ffmpegBin, url),
527
- probeKeyframeTimes: (url, budgetMs) => probeVideoKeyframeTimes(parts.ffmpegBin, url, budgetMs),
533
+ readSourceMedia,
534
+ get encodeInputs() { return parts.encodeInputs; },
528
535
  get decodeCostModel() { return parts.decodeCostModel; },
529
536
  get enabled() { return parts.enabled; },
530
537
  get encodeCost() { return parts.encodeCost; },
@@ -565,6 +572,7 @@ export function wireOutputs({
565
572
  // When this viewer's playback can start and run to the end: the viewer
566
573
  // component's forecast, which models the viewer's browser.
567
574
  playbackReadiness: { predict: predictPlaybackReadiness, forecastRate, RateTrend },
575
+ sourceInputsFor: (output, index) => parts.sourceInputsFor?.(output, index),
568
576
  encodeSpeedReadingOf: (output) => parts.encodeCost.latestSpeedReadingOf(output),
569
577
  projectedEncodeSpeedOf: (output) => parts.encodeCost.projectedSpeedOf(output),
570
578
  audioDescriptionOf,
@@ -615,6 +623,7 @@ export function wireOutputs({
615
623
  });
616
624
  parts.enabled = Boolean(enabled);
617
625
  parts.ffmpegBin = ffmpegBin;
626
+ parts.sourceInputsFor = typeof sourceInputsFor === "function" ? sourceInputsFor : null;
618
627
  parts.keyframeTables = keyframeTables;
619
628
  // Where measurements about this host are kept between runs. Empty means
620
629
  // beside the installed proxy; a deployment with somewhere persistent to
@@ -637,12 +646,6 @@ export function wireOutputs({
637
646
  // it meant reading a header this proxy had already read.
638
647
  parts.getContainerMediaInfo =
639
648
  typeof getContainerMediaInfo === "function" ? getContainerMediaInfo : null;
640
- // Fetch one whole file of a source, as a bounded read rather than a
641
- // selection. Used to pull a soundtrack that ships beside the picture onto
642
- // the disk while the swarm has capacity to spare — see
643
- // `#fetchSpareSoundtracks`. Optional: a proxy wired without it simply reads
644
- // such a soundtrack when it is played.
645
- parts.fetchWholeFile = typeof fetchWholeFile === "function" ? fetchWholeFile : null;
646
649
  // Optional async accessor for a source's live download stats, used by the
647
650
  // realtime budget to tell a CPU limit from a download-starved input:
648
651
  // (sourceKey, fileIndex) => Promise<{ downloadSpeed, fileLength, fileProgress } | null>.
@@ -760,7 +763,10 @@ export function wireOutputs({
760
763
  // exist, because that decision reads nothing else about viewers. It used to
761
764
  // be re-taken on a five-second timer instead, which made a just-created
762
765
  // output wait up to five seconds before anything noticed it had a viewer.
763
- parts.viewers = new Viewers({ onChange: () => parts.encodeRuns.planEncodersSoon() });
766
+ parts.viewers = new Viewers({ onChange: () => {
767
+ parts.encodeRuns.planEncodersSoon();
768
+ onViewerChanged?.();
769
+ } });
764
770
  // The outputs that exist, and every question about them: the picture a step
765
771
  // belongs to, the steps, the soundtracks, the height a step is named by.
766
772
  parts.outputs = new OutputCatalog({
@@ -863,6 +869,19 @@ export function wireOutputs({
863
869
  // a step supersedes belongs to the film's shape. Neither layer is handed
864
870
  // the other — one gets a plain id, the other is read for one field.
865
871
  watchedBy: (session, viewer) => !parts.outputs.supersededBy(session, viewer.activeVariantId ?? null),
872
+ urgentReadyFor: (session, viewer, seconds, now) => {
873
+ if (parts.subtitleReadyFor?.(viewer) === false) return false;
874
+ return urgentOutputsReady({
875
+ sourceKey: session.file.sourceKey, fileIndex: session.file.fileIndex,
876
+ durationSeconds: session.file.durationSeconds, atSeconds: viewer.positionSeconds(now) ?? 0, seconds,
877
+ outputs: parts.outputs.values(),
878
+ consumed: (output) => parts.viewers.forOutput(output).has(viewer.id) &&
879
+ !parts.outputs.supersededBy(output, viewer.activeVariantId ?? null),
880
+ segmentIndex: (output, at) => parts.outputTimes.segmentIndexForTime(output, at),
881
+ segmentStart: (output, index) => parts.outputTimes.segmentStartTime(output, index),
882
+ closed: (key, index) => parts.segmentStore.isClosed(key, index)
883
+ });
884
+ },
866
885
  allowanceFor: (session) => minimumBufferFrom({
867
886
  segmentSeconds: parts.segmentDurationSec,
868
887
  worstSupplyWaitSec: parts.hostLoad.supplyFor(session.file)?.worstWaitSec
@@ -910,6 +929,9 @@ export function wireOutputs({
910
929
  availability: () => parts.hostLoad.hostAvailability ?? null
911
930
  });
912
931
  parts.encodeOrchestrator = new EncodeOrchestrator({
932
+ inputDemandChanged: (address, windows) => {
933
+ for (const output of parts.outputs.outputsOn(address)) parts.encodeInputs?.retain(output, windows);
934
+ },
913
935
  admission: parts.admission,
914
936
  // What the viewers actually waited for, by band. The ledger is the
915
937
  // priority layer's; the encoding is handed a way to ask it.
@@ -951,6 +973,25 @@ export function wireOutputs({
951
973
  // one term of the keeping period that is guessed rather than measured, and
952
974
  // the only place it can be measured from.
953
975
  parts.returns = new Returns();
976
+ parts.encodeInputs = encodeInputs ?? (typeof resolveEncodeInput === "function" ? new EncodeInputs({
977
+ resolve: resolveEncodeInput,
978
+ readRanges: readEncodeRanges,
979
+ reviseBudget: () => parts.machineBudget.revise(),
980
+ capacity: () => parts.machineBudget.capacityOf("memory"),
981
+ urgent: (output, index) => parts.encodeOrchestrator.wantedSegmentsOn(output.outputKey).some(window =>
982
+ index >= window.from && index <= window.to && window.withinSeconds === 0),
983
+ changed: (output, result) => {
984
+ if (result.kind === "terminal") {
985
+ logger.warn(`encode input output=${output.outputKey} terminal=${result.reason}: ${result.message ?? ""}`);
986
+ parts.serving.invalidateWaits(output);
987
+ }
988
+ parts.encodeRuns.planEncodersSoon();
989
+ },
990
+ failed: (output, error) => {
991
+ logger.warn(`encode input output=${output.outputKey} failed: ${error.message}`);
992
+ parts.serving.invalidateWaits(output);
993
+ }
994
+ }) : null);
954
995
  // One owner of the disk, and the list of what takes it lives with the owner.
955
996
  parts.machineBudget = wireMachineBudget({
956
997
  segmentStore: parts.segmentStore,
@@ -959,6 +1000,8 @@ export function wireOutputs({
959
1000
  diagnostics: parts.diagnostics,
960
1001
  diagnosticsRoot: parts.diagnosticsRoot,
961
1002
  memory: parts.memoryClaimant,
1003
+ encodeInputs: parts.encodeInputs,
1004
+ indexMemory,
962
1005
  policy: parts.budgetPolicy ?? undefined,
963
1006
  readFree: freeBytesFor,
964
1007
  logger
@@ -0,0 +1,85 @@
1
+ /** Exact binary index ownership and pending growth under the shared budget. */
2
+ export class IndexMemory {
3
+ #files = new Map();
4
+ #held = 0;
5
+ #packetBytes = 0;
6
+ #allowed = 0;
7
+ #revise;
8
+ #changed;
9
+ #revisionQueued = false;
10
+
11
+ constructor({ reviseBudget, changed }) {
12
+ this.#revise = reviseBudget;
13
+ this.#changed = changed;
14
+ }
15
+
16
+ held() { return this.#held; }
17
+ packetBytes() { return this.#packetBytes; }
18
+ wanted() {
19
+ return this.#held + [...this.#files.values()].reduce((sum, file) => sum + Math.max(0, file.needed - file.held), 0);
20
+ }
21
+ required() { return this.wanted(); }
22
+
23
+ allow(bytes) {
24
+ const allowed = Math.max(0, bytes);
25
+ if (allowed === this.#allowed) return;
26
+ this.#allowed = allowed;
27
+ this.#changed?.();
28
+ }
29
+
30
+ #requestRevision() {
31
+ if (this.#revisionQueued) return;
32
+ this.#revisionQueued = true;
33
+ queueMicrotask(() => {
34
+ this.#revisionQueued = false;
35
+ this.#revise?.();
36
+ });
37
+ }
38
+
39
+ forFile(sourceKey, fileIndex) {
40
+ const key = JSON.stringify([sourceKey, fileIndex]);
41
+ let file = this.#files.get(key);
42
+ if (file) return file.memory;
43
+ file = { sourceKey, held: 0, needed: 0, records: new Set(), closed: false };
44
+ file.memory = { forRecord: (record, kind = "packets") => {
45
+ file.records.add(record);
46
+ let disposed = false;
47
+ return {
48
+ reserve: bytes => {
49
+ if (file.closed || disposed) return false;
50
+ if (this.#held + bytes > this.#allowed) {
51
+ const needed = file.held + bytes;
52
+ if (needed > file.needed) { file.needed = needed; this.#requestRevision(); }
53
+ return false;
54
+ }
55
+ file.held += bytes;
56
+ this.#held += bytes;
57
+ if (kind === "packets") this.#packetBytes += bytes;
58
+ if (file.held >= file.needed) file.needed = 0;
59
+ return true;
60
+ },
61
+ release: bytes => {
62
+ file.held -= bytes;
63
+ this.#held -= bytes;
64
+ if (kind === "packets") this.#packetBytes -= bytes;
65
+ this.#requestRevision();
66
+ },
67
+ dispose: () => { disposed = true; file.records.delete(record); }
68
+ };
69
+ } };
70
+ this.#files.set(key, file);
71
+ return file.memory;
72
+ }
73
+
74
+ forget(sourceKey) {
75
+ for (const [key, file] of this.#files) {
76
+ if (file.sourceKey !== sourceKey) continue;
77
+ file.closed = true;
78
+ file.needed = 0;
79
+ for (const record of [...file.records]) record.dispose();
80
+ this.#files.delete(key);
81
+ }
82
+ this.#requestRevision();
83
+ this.#changed?.();
84
+ }
85
+ }
@@ -75,6 +75,8 @@ import { OtherDemand, divideAllowance } from "./allowance.js";
75
75
  */
76
76
 
77
77
  export class MachineBudget {
78
+ #revisionPromise = null;
79
+ #reviseAgain = false;
78
80
  /** Resource name → how to read what is free, and what others have needed. */
79
81
  #resources = new Map();
80
82
 
@@ -135,12 +137,33 @@ export class MachineBudget {
135
137
  this.#claimants = this.#claimants.filter((one) => one.name !== name);
136
138
  }
137
139
 
140
+ /** Total measured policy allowance, independent of its current division. */
141
+ capacityOf(name) {
142
+ const reading = this.#last.get(name);
143
+ return reading?.measured ? reading.allowanceBytes : null;
144
+ }
145
+
138
146
  /**
139
147
  * Read every resource, divide each, and tell each claimant its share.
140
148
  *
141
149
  * @returns {Promise<Map<string, { freeBytes: number, allowanceBytes: number, shares: object[] }>>}
142
150
  */
143
- async revise() {
151
+ revise() {
152
+ if (this.#revisionPromise) {
153
+ this.#reviseAgain = true;
154
+ return this.#revisionPromise;
155
+ }
156
+ this.#revisionPromise = (async () => {
157
+ do {
158
+ this.#reviseAgain = false;
159
+ await this.#readRevision();
160
+ } while (this.#reviseAgain);
161
+ return this.#last;
162
+ })().finally(() => { this.#revisionPromise = null; });
163
+ return this.#revisionPromise;
164
+ }
165
+
166
+ async #readRevision() {
144
167
  for (const [name, resource] of this.#resources) {
145
168
  const claimants = this.#claimants.filter((one) => one.resource === name);
146
169
  if (claimants.length === 0) {
@@ -153,14 +176,15 @@ export class MachineBudget {
153
176
  // there is how the spill file came to have no limit in the first place.
154
177
  const freeBytes = Number.isFinite(free) && free !== null ? Math.max(0, free) : 0;
155
178
  const allowanceBytes = this.#allowanceFor(name, resource, freeBytes, held);
156
- const shares = divideAllowance(claimants.map((one) => Math.max(0, one.wanted())), allowanceBytes);
179
+ const shares = divideAllowance(claimants.map((one) => Math.max(0, one.wanted())), allowanceBytes,
180
+ claimants.map(one => Math.max(0, one.required?.() ?? 0)));
157
181
  const reading = claimants.map((one, position) => ({
158
182
  name: one.name,
159
183
  held: one.held(),
160
184
  allowed: shares[position],
161
185
  short: Number.isFinite(one.minimum) && shares[position] < one.minimum
162
186
  }));
163
- this.#last.set(name, { freeBytes, allowanceBytes, shares: reading });
187
+ this.#last.set(name, { freeBytes, allowanceBytes, measured: Number.isFinite(free) && free !== null, shares: reading });
164
188
  for (const [position, one] of claimants.entries()) {
165
189
  one.allow(shares[position]);
166
190
  }
@@ -98,7 +98,14 @@ export function machineAllowanceBytes(freeBytes, heldBytes, reserveBytes) {
98
98
  * @param {number} allowanceBytes
99
99
  * @returns {number[]} What each store may hold, in the same order.
100
100
  */
101
- export function divideAllowance(wantedBytes, allowanceBytes) {
101
+ export function divideAllowance(wantedBytes, allowanceBytes, requiredBytes = []) {
102
+ const required = wantedBytes.map((wanted, index) => Math.min(Math.max(0, wanted), Math.max(0, requiredBytes[index] ?? 0)));
103
+ const requiredTotal = required.reduce((sum, bytes) => sum + bytes, 0);
104
+ if (requiredTotal > 0) {
105
+ if (requiredTotal >= allowanceBytes) return divideAllowance(required, allowanceBytes);
106
+ const remaining = divideAllowance(wantedBytes.map((wanted, index) => Math.max(0, wanted - required[index])), allowanceBytes - requiredTotal);
107
+ return remaining.map((bytes, index) => bytes + required[index]);
108
+ }
102
109
  const total = wantedBytes.reduce((sum, want) => sum + Math.max(0, want), 0);
103
110
  if (total <= allowanceBytes || total === 0) {
104
111
  return wantedBytes.map((want) => Math.max(0, want));
@@ -80,6 +80,8 @@ export class PieceDiskStore {
80
80
 
81
81
  /** Where the live readers stand, from whoever holds that fact. @type {() => number[]} */
82
82
  #readHeads;
83
+ #canForget;
84
+ #wantedAt;
83
85
 
84
86
  /**
85
87
  * Said whenever this tier stops holding a piece, whatever the reason.
@@ -107,12 +109,16 @@ export class PieceDiskStore {
107
109
  allowanceBytes = null,
108
110
  now = Date.now,
109
111
  readHeads = () => [],
112
+ canForget = () => true,
113
+ wantedAt = null,
110
114
  onForgotten = () => undefined
111
115
  }) {
112
116
  this.#directory = path.join(directory, name);
113
117
  this.#allowanceBytes = Number.isFinite(allowanceBytes) && allowanceBytes >= 0 ? allowanceBytes : null;
114
118
  this.#now = now;
115
119
  this.#readHeads = typeof readHeads === "function" ? readHeads : () => [];
120
+ this.#canForget = canForget;
121
+ this.#wantedAt = wantedAt;
116
122
  this.#onForgotten = typeof onForgotten === "function" ? onForgotten : () => undefined;
117
123
  this.#adoptWhatIsAlreadyHere();
118
124
  }
@@ -211,6 +217,7 @@ export class PieceDiskStore {
211
217
  */
212
218
  reviseAllowance(bytes) {
213
219
  this.#allowanceBytes = Number.isFinite(bytes) && bytes >= 0 ? bytes : null;
220
+ this.#makeRoomFor(0, -1);
214
221
  return this.#allowanceBytes;
215
222
  }
216
223
 
@@ -358,7 +365,7 @@ export class PieceDiskStore {
358
365
  * @returns {void}
359
366
  */
360
367
  forget(index) {
361
- if (this.#reading.has(index)) {
368
+ if (this.#reading.has(index) || !this.#canForget(index)) {
362
369
  return;
363
370
  }
364
371
  const length = this.#stored.get(index);
@@ -482,7 +489,7 @@ export class PieceDiskStore {
482
489
  * @param {number} incomingIndex
483
490
  * @returns {Promise<void>}
484
491
  */
485
- async #makeRoomFor(incomingBytes, incomingIndex) {
492
+ #makeRoomFor(incomingBytes, incomingIndex) {
486
493
  if (this.#allowanceBytes === null) {
487
494
  return;
488
495
  }
@@ -519,7 +526,15 @@ export class PieceDiskStore {
519
526
  let victim = null;
520
527
  let worst = null;
521
528
  for (const [index, at] of this.#touched) {
522
- if (index === except || this.#reading.has(index)) {
529
+ if (index === except || this.#reading.has(index) || !this.#canForget(index)) {
530
+ continue;
531
+ }
532
+ if (this.#wantedAt) {
533
+ const want = this.#wantedAt(index);
534
+ if (worst === null || want > worst.want || (want === worst.want && at < worst.at)) {
535
+ worst = { want, at };
536
+ victim = index;
537
+ }
523
538
  continue;
524
539
  }
525
540
  const behind = earliest !== null && index < earliest;
@@ -251,8 +251,10 @@ export class PieceLru {
251
251
  wantAt(index) {
252
252
  let want = Number.POSITIVE_INFINITY;
253
253
  for (const range of this.#protected.values()) {
254
- if (index >= range.from && index <= range.to && range.urgency < want) {
255
- want = range.urgency;
254
+ const deadline = range.deadlineAt === undefined ? range.urgency :
255
+ Number.isFinite(range.deadlineAt) ? range.deadlineAt : Number.MAX_SAFE_INTEGER - range.priority;
256
+ if (index >= range.from && index <= range.to && deadline < want) {
257
+ want = deadline;
256
258
  }
257
259
  }
258
260
  return want;
@@ -432,14 +434,16 @@ export class PieceLru {
432
434
  * displace a stated one.
433
435
  * @returns {void}
434
436
  */
435
- protect(readerId, from, to, urgency) {
437
+ protect(readerId, from, to, urgency, deadlineAt, priority = 1) {
436
438
  if (!Number.isInteger(from) || !Number.isInteger(to) || to < from) {
437
439
  return;
438
440
  }
439
441
  this.#protected.set(readerId, {
440
442
  from,
441
443
  to,
442
- urgency: Number.isFinite(urgency) ? Number(urgency) : Number.MAX_SAFE_INTEGER
444
+ urgency: Number.isFinite(urgency) ? Number(urgency) : Number.MAX_SAFE_INTEGER,
445
+ deadlineAt,
446
+ priority
443
447
  });
444
448
  }
445
449
 
@@ -530,6 +530,8 @@ export class SharedPieceStore {
530
530
  // Where the live readers stand, so what goes first is decided by them and
531
531
  // not by which piece happened to be touched longest ago.
532
532
  readHeads: () => this.#lru.readHeads(),
533
+ canForget: (index) => !this.#lru.isPinned(index),
534
+ wantedAt: (index) => this.#lru.wantAt(index),
533
535
  // EVERY WAY A PIECE LEAVES THE DISK COMES THROUGH HERE — behind the read
534
536
  // heads, over the allowance, dropped as a duplicate, or forgotten while a
535
537
  // spill finished. One listener instead of a list of call sites to
@@ -631,19 +633,13 @@ export class SharedPieceStore {
631
633
  * @returns {{ name: string, allowanceBytes: number | null, bytes: number }}
632
634
  */
633
635
  reviseSpillCeiling(allowedBytes) {
634
- // THE OTHER RULE, and it does not wait for the disk to be short. A piece
635
- // behind every read head has been read and will not be read again unless
636
- // somebody seeks back, and a seek back re-downloads it — the same bargain
637
- // this tier makes whenever it drops a piece for room. Without it the spill
638
- // is bounded only by a share of free space, which on a roomy host is tens of
639
- // gigabytes against a measured growth of 14 400 MB in one viewing: the
640
- // ceiling never binds and nothing is ever removed until the torrent goes.
641
- const behind = this.#disk.forgetBehind(this.#lru.readHeads());
636
+ // Downloaded material is removed only for capacity. Playback position
637
+ // alone cannot invalidate bytes needed by another viewer or a later seek.
642
638
  return {
643
639
  name: this.#name,
644
640
  allowanceBytes: this.#disk.reviseAllowance(allowedBytes),
645
641
  bytes: this.#disk.bytes,
646
- behind
642
+ behind: 0
647
643
  };
648
644
  }
649
645
 
@@ -816,10 +812,29 @@ export class SharedPieceStore {
816
812
 
817
813
  /** Availability belongs to the store, not to a second cached bitfield. */
818
814
  locationOf(index) {
815
+ if (this.#closed || !Number.isSafeInteger(index) || index < 0 ||
816
+ (this.#lastChunkIndex >= 0 && index > this.#lastChunkIndex)) return "missing";
819
817
  return this.#buffers.has(index) ? "memory" : this.#evicting.has(index) ? "writing" :
820
818
  this.#disk.has(index) ? "disk" : this.isInWholeFiles(index) ? "whole-file" : "missing";
821
819
  }
822
820
 
821
+ /** Atomically acquire holds only when every requested piece is present. */
822
+ holdAvailable(indexes) {
823
+ if (this.#closed || !Array.isArray(indexes)) return null;
824
+ const unique = [...new Set(indexes)];
825
+ if (unique.some((index) => !Number.isInteger(index) || index < 0 || this.locationOf(index) === "missing")) {
826
+ return null;
827
+ }
828
+ for (const index of unique) this.#lru.pin(index);
829
+ let held = true;
830
+ return () => {
831
+ if (!held) return;
832
+ held = false;
833
+ for (const index of unique) this.#lru.unpin(index);
834
+ this.#noteProgress();
835
+ };
836
+ }
837
+
823
838
  pin(index) {
824
839
  this.#lru.pin(index);
825
840
  // A piece reader pins before it waits for the bytes to arrive or return
@@ -1833,8 +1848,8 @@ export class SharedPieceStore {
1833
1848
  * @param {number} [urgency]
1834
1849
  * @returns {void}
1835
1850
  */
1836
- protectRange(readerId, from, to, urgency) {
1837
- this.#lru.protect(readerId, from, to, urgency);
1851
+ protectRange(readerId, from, to, urgency, deadlineAt, priority) {
1852
+ this.#lru.protect(readerId, from, to, urgency, deadlineAt, priority);
1838
1853
  // A READER DECLARES ITSELF IN A MOMENT; the allowance was re-derived once a
1839
1854
  // minute. Between the two a read met whatever the store had shrunk to while
1840
1855
  // nobody was reading, and a claim gives up after five seconds — twelve