@torrent-tv/proxy 2.80.0 → 2.80.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1576 -1558
- package/package.json +1 -1
- package/server.js +16 -0
- package/services/demand/DemandRegister.js +20 -0
- package/services/download/SwarmSelection.js +4 -1
- package/services/encode/EncodePlan.js +159 -31
- package/services/encode/SegmentStore.js +29 -0
- package/services/encode/run-budget.js +8 -0
- package/services/hls-session-manager.js +25 -8
- package/services/orchestrators/EncodeOrchestrator.js +27 -5
- package/services/piece-store/piece-lru.js +91 -31
- package/services/piece-store/shared-piece-store.js +1531 -1512
- package/services/priority/PriorityMap.js +57 -101
- package/services/priority/PriorityOrchestrator.js +149 -0
- package/services/torrent-pool.js +104 -0
- package/services/torrent-worker/client.js +20 -0
- package/services/torrent-worker/piece-reader.js +57 -217
- package/services/torrent-worker/pool-adapter.js +17 -0
- package/services/torrent-worker/protocol.js +10 -0
- package/services/torrent-worker/worker.js +11 -0
- package/test/demand-register.test.js +11 -0
- package/test/encode-plan.test.js +65 -0
- package/test/piece-lru.test.js +61 -0
- package/test/piece-store-eviction.test.js +30 -0
- package/test/priority-map-download.test.js +156 -0
- package/test/priority-map.test.js +50 -77
- package/test/read-window.test.js +46 -29
- package/test/read-bands.test.js +0 -140
|
@@ -58,133 +58,6 @@ let readerSequence = 0;
|
|
|
58
58
|
*/
|
|
59
59
|
const READ_WINDOW_BYTES = 32 * 1024 * 1024;
|
|
60
60
|
|
|
61
|
-
/**
|
|
62
|
-
* How urgently each band is wanted, most urgent first.
|
|
63
|
-
*
|
|
64
|
-
* Distinct numbers, and none of them zero: WebTorrent selects the whole torrent
|
|
65
|
-
* at priority 0 for the background fill, so a band at 0 would be indistinguishable
|
|
66
|
-
* from it. Equal non-zero priorities are deliberately shuffled against each other
|
|
67
|
-
* by the library (`shufflePriority`), so bands that must keep their order have to
|
|
68
|
-
* differ.
|
|
69
|
-
*
|
|
70
|
-
* 4 is what the viewer reaches in seconds; 3 and 2 are the lead being built ahead
|
|
71
|
-
* of them; 1 is what was never downloaded BEHIND the position, which only a
|
|
72
|
-
* backward seek needs.
|
|
73
|
-
*/
|
|
74
|
-
/**
|
|
75
|
-
* Which way this proxy claims what a reader wants, when the deployment names it:
|
|
76
|
-
* `flat` is the single band every release before this used, `bands` is the four
|
|
77
|
-
* described above. Anything else — the default — alternates per read, so the two
|
|
78
|
-
* accumulate side by side from real viewing and the log can compare them.
|
|
79
|
-
*/
|
|
80
|
-
|
|
81
|
-
/**
|
|
82
|
-
* Which level of urgency each band states.
|
|
83
|
-
*
|
|
84
|
-
* Not numbers handed to WebTorrent: measured against the vendored 2.8.5, the
|
|
85
|
-
* library keeps distinct non-zero priorities only until the first wire is
|
|
86
|
-
* served and round-robins them afterwards. The ordering is kept by WHAT is
|
|
87
|
-
* stated, in `services/demand/`, and these say which level each band belongs
|
|
88
|
-
* to (roadmap item 5).
|
|
89
|
-
*/
|
|
90
|
-
const BAND_URGENCY = [Urgency.NEAR, Urgency.AHEAD, Urgency.AHEAD, Urgency.BEHIND];
|
|
91
|
-
|
|
92
|
-
/**
|
|
93
|
-
* Where each band sits, given the urgent one and how far the trailing bands have
|
|
94
|
-
* been allowed to grow.
|
|
95
|
-
*
|
|
96
|
-
* The urgent band is the reader's own window, anchored at the first piece it does
|
|
97
|
-
* not already hold. Behind it the lead is built in two steps rather than one, so
|
|
98
|
-
* the nearer half is asked for before the farther half; behind THOSE, once they
|
|
99
|
-
* have reached the end of the file, comes whatever was never downloaded before the
|
|
100
|
-
* position — needed only if the viewer seeks backwards, and never ahead of the
|
|
101
|
-
* picture they are watching.
|
|
102
|
-
*
|
|
103
|
-
* @param {{ urgent: { from: number, to: number }, pieceIndex: number, firstPiece: number, lastPiece: number, widths: { near: number, far: number } }} params
|
|
104
|
-
* @returns {Array<{ from: number, to: number, urgency: number }>}
|
|
105
|
-
*/
|
|
106
|
-
export function bandsFrom({ urgent, pieceIndex, firstPiece, lastPiece, widths }) {
|
|
107
|
-
const bands = [{ from: urgent.from, to: urgent.to, urgency: BAND_URGENCY[0] }];
|
|
108
|
-
let edge = urgent.to;
|
|
109
|
-
for (let index = 0; index < 2; index += 1) {
|
|
110
|
-
if (edge >= lastPiece) {
|
|
111
|
-
break;
|
|
112
|
-
}
|
|
113
|
-
const width = index === 0 ? widths.near : widths.far;
|
|
114
|
-
if (width <= 0) {
|
|
115
|
-
break;
|
|
116
|
-
}
|
|
117
|
-
const from = edge + 1;
|
|
118
|
-
const to = Math.min(lastPiece, edge + width);
|
|
119
|
-
bands.push({ from, to, urgency: BAND_URGENCY[index + 1] });
|
|
120
|
-
edge = to;
|
|
121
|
-
}
|
|
122
|
-
// Only once the lead has nothing left to cover: asking for the past while the
|
|
123
|
-
// future is still missing would take capacity from the picture being watched.
|
|
124
|
-
if (edge >= lastPiece && pieceIndex > firstPiece) {
|
|
125
|
-
bands.push({ from: firstPiece, to: pieceIndex - 1, urgency: BAND_URGENCY[3] });
|
|
126
|
-
}
|
|
127
|
-
return bands;
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
/**
|
|
131
|
-
* How wide the two lead bands should be, in pieces, from what has been measured
|
|
132
|
-
* about this file on this swarm.
|
|
133
|
-
*
|
|
134
|
-
* Nothing here is chosen. The near band has to cover the worst interruption this
|
|
135
|
-
* reader has actually met, because that is what must already be in hand for one
|
|
136
|
-
* not to reach the viewer: `worstWait × consumeRate`. The far band has to cover
|
|
137
|
-
* what the swarm can put ahead of the viewer between interruptions, which is the
|
|
138
|
-
* surplus it delivers over what the film eats, for as long as it typically runs
|
|
139
|
-
* without stopping: `(downloadRate - consumeRate) × medianInterval`. A swarm
|
|
140
|
-
* with no surplus produces no far band, which is correct — there is nothing to
|
|
141
|
-
* get ahead with.
|
|
142
|
-
*
|
|
143
|
-
* Until a reader has met two interruptions there are no figures, and both bands
|
|
144
|
-
* fall back to the width of the urgent one; the log says so in words.
|
|
145
|
-
*
|
|
146
|
-
* @param {{ worstWaitSec: number | null, medianIntervalSec: number | null, downloadBytesPerSec: number, consumeBytesPerSec: number, pieceLength: number, basePieces: number }} params
|
|
147
|
-
* @returns {{ near: number, far: number, measured: boolean }}
|
|
148
|
-
*/
|
|
149
|
-
export function bandWidthsFrom({
|
|
150
|
-
worstWaitSec,
|
|
151
|
-
medianIntervalSec,
|
|
152
|
-
downloadBytesPerSec,
|
|
153
|
-
consumeBytesPerSec,
|
|
154
|
-
pieceLength,
|
|
155
|
-
basePieces
|
|
156
|
-
}) {
|
|
157
|
-
const usable = Number.isFinite(worstWaitSec) && worstWaitSec > 0 &&
|
|
158
|
-
Number.isFinite(medianIntervalSec) && medianIntervalSec > 0 &&
|
|
159
|
-
Number.isFinite(consumeBytesPerSec) && consumeBytesPerSec > 0 &&
|
|
160
|
-
Number.isFinite(pieceLength) && pieceLength > 0;
|
|
161
|
-
if (!usable) {
|
|
162
|
-
return { near: basePieces, far: basePieces, measured: false };
|
|
163
|
-
}
|
|
164
|
-
const near = Math.max(1, Math.ceil((worstWaitSec * consumeBytesPerSec) / pieceLength));
|
|
165
|
-
const surplus = Math.max(0, (Number(downloadBytesPerSec) || 0) - consumeBytesPerSec);
|
|
166
|
-
const far = Math.max(0, Math.ceil((surplus * medianIntervalSec) / pieceLength));
|
|
167
|
-
return { near, far, measured: true };
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
/**
|
|
171
|
-
* Whether two sets of bands are the same, so an unchanged claim is not released
|
|
172
|
-
* and re-made on every read.
|
|
173
|
-
*
|
|
174
|
-
* @param {Array<{ from: number, to: number, priority: number }>} left
|
|
175
|
-
* @param {Array<{ from: number, to: number, priority: number }>} right
|
|
176
|
-
* @returns {boolean}
|
|
177
|
-
*/
|
|
178
|
-
export function sameBands(left, right) {
|
|
179
|
-
if (left.length !== right.length) {
|
|
180
|
-
return false;
|
|
181
|
-
}
|
|
182
|
-
return left.every((band, index) =>
|
|
183
|
-
band.from === right[index].from &&
|
|
184
|
-
band.to === right[index].to &&
|
|
185
|
-
band.urgency === right[index].urgency);
|
|
186
|
-
}
|
|
187
|
-
|
|
188
61
|
/**
|
|
189
62
|
* The pieces a reader at `pieceIndex` wants next, clamped to its own range.
|
|
190
63
|
*
|
|
@@ -743,32 +616,16 @@ export async function* readFragments({
|
|
|
743
616
|
*/
|
|
744
617
|
let waitBelongsToJump = false;
|
|
745
618
|
|
|
746
|
-
/**
|
|
747
|
-
* The bands currently claimed from the swarm, most urgent first.
|
|
748
|
-
*
|
|
749
|
-
* @type {Array<{ from: number, to: number, priority: number }>}
|
|
750
|
-
*/
|
|
751
|
-
let claimed = [];
|
|
752
|
-
/**
|
|
753
|
-
* What the consumer is taking from this read, in bytes a second, measured as
|
|
754
|
-
* it goes. For a viewer that is the film's own byte rate, which is exactly the
|
|
755
|
-
* quantity the band widths are derived against — and measuring it here needs
|
|
756
|
-
* nothing passed in and no assumption about who is reading.
|
|
757
|
-
*/
|
|
619
|
+
/** What the consumer has taken from this read, in bytes. */
|
|
758
620
|
let deliveredBytes = 0;
|
|
759
621
|
// How often this read had to stop, and for how long in total. Reported at the
|
|
760
622
|
// end whatever the outcome, so a read that never stopped is counted too.
|
|
761
623
|
let waitCount = 0;
|
|
762
624
|
let waitedTotalMs = 0;
|
|
763
625
|
const readStartedAt = Date.now();
|
|
764
|
-
const consumeBytesPerSec = () => {
|
|
765
|
-
const seconds = (Date.now() - readStartedAt) / 1000;
|
|
766
|
-
return seconds > 0 ? deliveredBytes / seconds : 0;
|
|
767
|
-
};
|
|
768
626
|
|
|
769
627
|
/**
|
|
770
|
-
* Where the reader's
|
|
771
|
-
* does not already have. Everything between the read position and that piece
|
|
628
|
+
* Where the reader's claim starts: the first piece it does not already have. Everything between the read position and that piece
|
|
772
629
|
* is on disk or in memory, so claiming it asks the swarm for what we hold.
|
|
773
630
|
*
|
|
774
631
|
* @param {number} pieceIndex
|
|
@@ -784,26 +641,6 @@ export async function* readFragments({
|
|
|
784
641
|
};
|
|
785
642
|
|
|
786
643
|
|
|
787
|
-
/**
|
|
788
|
-
* The widths behind the bands currently claimed, for the log to state.
|
|
789
|
-
*
|
|
790
|
-
* @type {{ near: number, far: number, measured: boolean }}
|
|
791
|
-
*/
|
|
792
|
-
let lastWidths = { near: basePieces, far: basePieces, measured: false };
|
|
793
|
-
|
|
794
|
-
const bandWidths = () => {
|
|
795
|
-
const figures = supplyFiguresFor(torrent?.infoHash ?? "?", file?.name ?? "", SEGMENT_SECONDS_FOR_BUFFER);
|
|
796
|
-
lastWidths = bandWidthsFrom({
|
|
797
|
-
worstWaitSec: figures?.worstWaitSec ?? null,
|
|
798
|
-
medianIntervalSec: figures?.medianIntervalSec ?? null,
|
|
799
|
-
downloadBytesPerSec: Number(torrent?.downloadSpeed) || 0,
|
|
800
|
-
consumeBytesPerSec: consumeBytesPerSec(),
|
|
801
|
-
pieceLength,
|
|
802
|
-
basePieces
|
|
803
|
-
});
|
|
804
|
-
return lastWidths;
|
|
805
|
-
};
|
|
806
|
-
|
|
807
644
|
/**
|
|
808
645
|
* State one band as a need, in bytes.
|
|
809
646
|
*
|
|
@@ -834,39 +671,47 @@ export async function* readFragments({
|
|
|
834
671
|
});
|
|
835
672
|
};
|
|
836
673
|
|
|
674
|
+
/**
|
|
675
|
+
* Move the window the reader is working through.
|
|
676
|
+
*
|
|
677
|
+
* It states NOTHING to the swarm. What should be downloaded ahead of a viewer
|
|
678
|
+
* is the priority map's answer, stated once for the whole file by the side
|
|
679
|
+
* that knows where the viewers are; a read is consumption, not a forecast.
|
|
680
|
+
*
|
|
681
|
+
* A read used to declare a rolling window of its own — the piece it wanted
|
|
682
|
+
* and two bands beyond it — and with fifteen reads on one file that was
|
|
683
|
+
* fifteen windows on a piece store holding sixteen pieces. Half of all
|
|
684
|
+
* evictions then took a piece a reader had declared, two thirds of reads came
|
|
685
|
+
* back from disk, and the bytes handed out stopped being the file's: twenty-
|
|
686
|
+
* two source-parse errors, a segment the player refused, and an empty picture
|
|
687
|
+
* for six minutes (field 2026-09-05).
|
|
688
|
+
*
|
|
689
|
+
* What the window is still for: the pieces to mark critical when the reader
|
|
690
|
+
* is actually stopped, and the range to bring back from disk after a jump.
|
|
691
|
+
*
|
|
692
|
+
* @param {number} pieceIndex
|
|
693
|
+
* @returns {void}
|
|
694
|
+
*/
|
|
695
|
+
/**
|
|
696
|
+
* Where the priority map puts one piece, in the map's own levels.
|
|
697
|
+
*
|
|
698
|
+
* A stated window is file-relative — `bytesOf` subtracts the file's offset
|
|
699
|
+
* within the torrent — so the piece is put in that frame before it is looked
|
|
700
|
+
* up, or every answer belongs to some other part of the torrent.
|
|
701
|
+
*
|
|
702
|
+
* @param {number} pieceIndex
|
|
703
|
+
* @returns {number | null}
|
|
704
|
+
*/
|
|
705
|
+
const mapLevelOf = (pieceIndex) =>
|
|
706
|
+
register.urgencyAt(fileIndex, pieceIndex * pieceLength - Number(file.offset));
|
|
707
|
+
|
|
837
708
|
const moveWindowTo = (pieceIndex) => {
|
|
838
709
|
const anchor = firstMissingFrom(pieceIndex);
|
|
839
710
|
const next = readWindowFor({ pieceIndex: anchor, lastPiece, windowPieces });
|
|
840
|
-
|
|
841
|
-
const isJump = !window || next.from > window.to || next.from < window.from;
|
|
842
|
-
// A seek makes every width behind the urgent band meaningless: they were
|
|
843
|
-
// grown against a position the viewer has left, and what lies beyond the
|
|
844
|
-
// new one has to be earned from a standing start.
|
|
845
|
-
const wanted = bandsFrom({
|
|
846
|
-
urgent: next,
|
|
847
|
-
pieceIndex,
|
|
848
|
-
firstPiece,
|
|
849
|
-
lastPiece,
|
|
850
|
-
widths: bandWidths()
|
|
851
|
-
});
|
|
852
|
-
if (sameUrgent && sameBands(claimed, wanted)) {
|
|
711
|
+
if (window && window.from === next.from && window.to === next.to) {
|
|
853
712
|
return;
|
|
854
713
|
}
|
|
855
|
-
|
|
856
|
-
// stay. The swarm is told nothing here — `reconcile` is the only thing that
|
|
857
|
-
// speaks to the library, and it reads what is stated.
|
|
858
|
-
for (const band of claimed) {
|
|
859
|
-
register.withdraw(`${readerId}:${urgencyName(band.urgency)}`);
|
|
860
|
-
}
|
|
861
|
-
for (const band of wanted) {
|
|
862
|
-
stateBand(band);
|
|
863
|
-
}
|
|
864
|
-
// One call, and it does both: the swarm is told what to fetch and the store
|
|
865
|
-
// is told what will be read soon, from the same stated needs. The store used
|
|
866
|
-
// to be told separately here, which made the same intent two lists that
|
|
867
|
-
// could drift.
|
|
868
|
-
selection.reconcile();
|
|
869
|
-
claimed = wanted;
|
|
714
|
+
const isJump = !window || next.from > window.to || next.from < window.from;
|
|
870
715
|
window = next;
|
|
871
716
|
if (isJump) {
|
|
872
717
|
waitBelongsToJump = true;
|
|
@@ -1019,6 +864,9 @@ export async function* readFragments({
|
|
|
1019
864
|
// whether that is the swarm, the picker, or ffmpeg. Logged only when the
|
|
1020
865
|
// wait is long enough to matter, so ordinary sequential reading is silent.
|
|
1021
866
|
const waitedMs = Date.now() - waitStartedAt;
|
|
867
|
+
// Where the map puts this piece, read once and used by both the
|
|
868
|
+
// attribution below and the line beside it.
|
|
869
|
+
const wantedBy = mapLevelOf(pieceIndex);
|
|
1022
870
|
// The window answers to what just happened: a wait means the lead was too
|
|
1023
871
|
// short, an immediate hit means it is longer than it needs to be. Applied
|
|
1024
872
|
// before the logging below so the line reports the window the next piece
|
|
@@ -1035,14 +883,13 @@ export async function* readFragments({
|
|
|
1035
883
|
} else {
|
|
1036
884
|
const supplyKey = `${torrent?.infoHash ?? "?"}/${file?.name ?? "?"}`;
|
|
1037
885
|
noteSteeringOutcome(supplyKey, waitedMs, pushed.asked > 0 || duplicated > 0);
|
|
1038
|
-
// Which
|
|
1039
|
-
// the level says whether that
|
|
1040
|
-
//
|
|
886
|
+
// Which level of the priority map the reader was stopped in. A wait
|
|
887
|
+
// belongs to a level, and the level says whether that zone is asked for
|
|
888
|
+
// too late — the reader itself no longer has an opinion about it.
|
|
1041
889
|
noteWaitLevel(
|
|
1042
890
|
supplyKey,
|
|
1043
891
|
waitedMs,
|
|
1044
|
-
|
|
1045
|
-
?? Urgency.BLOCKED
|
|
892
|
+
wantedBy ?? Urgency.BLOCKED
|
|
1046
893
|
);
|
|
1047
894
|
waitCount += 1;
|
|
1048
895
|
waitedTotalMs += waitedMs;
|
|
@@ -1111,13 +958,13 @@ export async function* readFragments({
|
|
|
1111
958
|
// What we did about the tail, so the next session says by number
|
|
1112
959
|
// whether a second copy of those blocks shortens the wait.
|
|
1113
960
|
(duplicated > 0 ? `; duplicated ${duplicated} blocks` : "") +
|
|
1114
|
-
//
|
|
1115
|
-
//
|
|
1116
|
-
//
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
?
|
|
1120
|
-
:
|
|
961
|
+
// Where the priority map puts this piece. A wait at the level the
|
|
962
|
+
// viewer is about to reach is a different fault from a wait on the
|
|
963
|
+
// speculative tail, and the reader states nothing of its own that
|
|
964
|
+
// could be read instead.
|
|
965
|
+
`; the map wants it ${wantedBy === null
|
|
966
|
+
? "nowhere — nobody asked for this piece"
|
|
967
|
+
: urgencyName(wantedBy)}`
|
|
1121
968
|
);
|
|
1122
969
|
}
|
|
1123
970
|
|
|
@@ -1181,14 +1028,10 @@ export async function* readFragments({
|
|
|
1181
1028
|
releaseHeldPin();
|
|
1182
1029
|
releaseHeldPin = null;
|
|
1183
1030
|
}
|
|
1184
|
-
// What this read did, said once at its end and under EVERY outcome
|
|
1185
|
-
//
|
|
1186
|
-
//
|
|
1187
|
-
//
|
|
1188
|
-
// measured 2026-08-19 across eight sessions with zero waits, the log could
|
|
1189
|
-
// not say whether `flat` or `bands` had run even once. A comparison that
|
|
1190
|
-
// only records the bad outcomes cannot say that the good ones happened at
|
|
1191
|
-
// all, and "no wait" is exactly the result worth counting.
|
|
1031
|
+
// What this read did, said once at its end and under EVERY outcome: how
|
|
1032
|
+
// much it delivered, and how much of that time it spent stopped. Said even
|
|
1033
|
+
// when it never stopped, because "no wait" is the result worth counting and
|
|
1034
|
+
// a line printed only beside a wait cannot report it.
|
|
1192
1035
|
const readSeconds = (Date.now() - readStartedAt) / 1000;
|
|
1193
1036
|
if (deliveredBytes > 0) {
|
|
1194
1037
|
logger.info(
|
|
@@ -1198,11 +1041,8 @@ export async function* readFragments({
|
|
|
1198
1041
|
);
|
|
1199
1042
|
}
|
|
1200
1043
|
// Reached on completion, on cancellation, on a throw, and when the consumer
|
|
1201
|
-
// stops iterating — a
|
|
1044
|
+
// stops iterating — a claim left behind would keep the swarm fetching for a
|
|
1202
1045
|
// reader that no longer exists.
|
|
1203
|
-
for (const band of claimed) {
|
|
1204
|
-
register.withdraw(`${readerId}:${urgencyName(band.urgency)}`);
|
|
1205
|
-
}
|
|
1206
1046
|
if (blockedStated) {
|
|
1207
1047
|
register.withdraw(`${readerId}:${urgencyName(Urgency.BLOCKED)}`);
|
|
1208
1048
|
}
|
|
@@ -335,6 +335,23 @@ export class WorkerTorrentPool {
|
|
|
335
335
|
.catch(() => undefined);
|
|
336
336
|
}
|
|
337
337
|
|
|
338
|
+
/**
|
|
339
|
+
* Hand the download the priority map for one file.
|
|
340
|
+
*
|
|
341
|
+
* The map is republished whenever it changes, so a call that fails costs a
|
|
342
|
+
* moment rather than correctness — which is why the caller lets it go rather
|
|
343
|
+
* than retrying.
|
|
344
|
+
*
|
|
345
|
+
* @param {{ sourceKey: string, fileIndex: number, durationSeconds: number, zones: object[] }} params
|
|
346
|
+
* @returns {Promise<void>}
|
|
347
|
+
*/
|
|
348
|
+
async setPriorityMap({ sourceKey, fileIndex, durationSeconds, zones }) {
|
|
349
|
+
if (!sourceKey) {
|
|
350
|
+
return;
|
|
351
|
+
}
|
|
352
|
+
await this.#client.setPriorityMap({ sourceKey, fileIndex, durationSeconds, zones });
|
|
353
|
+
}
|
|
354
|
+
|
|
338
355
|
/**
|
|
339
356
|
* Pre-fetch the head and tail the codec probe needs.
|
|
340
357
|
*
|
|
@@ -61,6 +61,16 @@ export const Command = {
|
|
|
61
61
|
HELD_TORRENTS: "held-torrents",
|
|
62
62
|
/** Reorder piece selection around a read position (seek prioritisation). */
|
|
63
63
|
PRIORITIZE: "prioritize",
|
|
64
|
+
/**
|
|
65
|
+
* The priority map for one file: seconds of film against a number.
|
|
66
|
+
*
|
|
67
|
+
* What anybody wants and in what order, stated once by the side that knows
|
|
68
|
+
* where the viewers are. The download decides what to fetch and what to keep
|
|
69
|
+
* from this, instead of from the windows the reads themselves used to
|
|
70
|
+
* declare — fifteen reads declaring fifteen windows on a store that holds
|
|
71
|
+
* sixteen pieces is what tore a film apart on 2026-09-05.
|
|
72
|
+
*/
|
|
73
|
+
PRIORITY_MAP: "priority-map",
|
|
64
74
|
/** Read a byte range; the body arrives as CHUNK messages. */
|
|
65
75
|
READ_RANGE: "read-range",
|
|
66
76
|
/** Abandon an in-flight READ_RANGE (viewer gone, seek superseded). */
|
|
@@ -534,6 +534,17 @@ async function runCommand(command, params, id) {
|
|
|
534
534
|
};
|
|
535
535
|
}
|
|
536
536
|
|
|
537
|
+
case Command.PRIORITY_MAP: {
|
|
538
|
+
const torrent = await requireTorrent(params.sourceKey);
|
|
539
|
+
pool.applyPriorityMap(
|
|
540
|
+
torrent,
|
|
541
|
+
params.fileIndex,
|
|
542
|
+
params.zones,
|
|
543
|
+
params.durationSeconds
|
|
544
|
+
);
|
|
545
|
+
return true;
|
|
546
|
+
}
|
|
547
|
+
|
|
537
548
|
case Command.PRIORITIZE: {
|
|
538
549
|
const torrent = await requireTorrent(params.sourceKey);
|
|
539
550
|
pool.prioritizeByteRange(torrent, params.fileIndex, params.byteStart, params.windowBytes, {
|
|
@@ -193,3 +193,14 @@ test("the gap behind the playhead is stated nearest first", () => {
|
|
|
193
193
|
}
|
|
194
194
|
assert.deepEqual(nearestFirst({ byteStart: 10, byteEnd: 5, parts: 2 }), []);
|
|
195
195
|
});
|
|
196
|
+
|
|
197
|
+
test("a byte is as urgent as the most urgent thing that wants it", () => {
|
|
198
|
+
const register = new DemandRegister();
|
|
199
|
+
register.state({ claimant: "map:tail", fileIndex: 0, byteStart: 0, byteEnd: 999, urgency: Urgency.TAIL });
|
|
200
|
+
register.state({ claimant: "map:near", fileIndex: 0, byteStart: 400, byteEnd: 599, urgency: Urgency.NEAR });
|
|
201
|
+
|
|
202
|
+
assert.equal(register.urgencyAt(0, 100), Urgency.TAIL);
|
|
203
|
+
assert.equal(register.urgencyAt(0, 500), Urgency.NEAR, "the overlap took the less urgent of the two");
|
|
204
|
+
assert.equal(register.urgencyAt(0, 5000), null, "a byte nobody wants reported a level anyway");
|
|
205
|
+
assert.equal(register.urgencyAt(1, 500), null, "byte 500 of another file is not this byte");
|
|
206
|
+
});
|
package/test/encode-plan.test.js
CHANGED
|
@@ -380,3 +380,68 @@ test("a run with no end is making what the viewers ahead of it are waiting for",
|
|
|
380
380
|
"and nothing new is started over ground it already holds"
|
|
381
381
|
);
|
|
382
382
|
});
|
|
383
|
+
|
|
384
|
+
test("the machine's whole budget is used, not one encoder per viewer", () => {
|
|
385
|
+
// Nothing about a viewer says how many encoders there should be. What says it
|
|
386
|
+
// is what the machine affords, and the film is divided between them.
|
|
387
|
+
const coverage = new CoverageMap({ segmentCount: 400 });
|
|
388
|
+
const actions = planEncoders({
|
|
389
|
+
coverage,
|
|
390
|
+
windows: [
|
|
391
|
+
{ from: 0, to: 9, priority: 32 },
|
|
392
|
+
{ from: 10, to: 399, priority: 20 }
|
|
393
|
+
],
|
|
394
|
+
runs: [],
|
|
395
|
+
...HOST,
|
|
396
|
+
maxRuns: 4
|
|
397
|
+
});
|
|
398
|
+
const started = actions.filter((action) => action.type === "start");
|
|
399
|
+
|
|
400
|
+
assert.equal(started.length, 4, "four encoders where the machine allows four");
|
|
401
|
+
const from = started.map((action) => action.from).sort((left, right) => left - right);
|
|
402
|
+
assert.equal(from[0], 0, "the first where the viewer is stopped");
|
|
403
|
+
assert.ok(from[3] > from[0], "and the rest spread over the film");
|
|
404
|
+
});
|
|
405
|
+
|
|
406
|
+
test("below realtime the first stretches are what one encoder can hold", () => {
|
|
407
|
+
// At half speed an encoder holds `(q - p)` of film: the first covers ten
|
|
408
|
+
// segments, and the next has to be standing where it stops holding.
|
|
409
|
+
const coverage = new CoverageMap({ segmentCount: 400 });
|
|
410
|
+
const actions = planEncoders({
|
|
411
|
+
coverage,
|
|
412
|
+
windows: [{ from: 10, to: 399, priority: 32 }],
|
|
413
|
+
runs: [{ from: 0, to: 0, head: 0, speedX: 0.5 }],
|
|
414
|
+
...HOST,
|
|
415
|
+
maxRuns: 3
|
|
416
|
+
});
|
|
417
|
+
const started = actions
|
|
418
|
+
.filter((action) => action.type === "start")
|
|
419
|
+
.map((action) => action.from)
|
|
420
|
+
.sort((left, right) => left - right);
|
|
421
|
+
|
|
422
|
+
assert.ok(started.length >= 2, "more than one, because one cannot hold the film");
|
|
423
|
+
assert.ok(started[1] > started[0], "and they grow apart rather than sitting together");
|
|
424
|
+
});
|
|
425
|
+
|
|
426
|
+
test("two encoders never share a segment number", () => {
|
|
427
|
+
// The whole of what went wrong in the field: two encoders writing one name.
|
|
428
|
+
const coverage = new CoverageMap({ segmentCount: 400 });
|
|
429
|
+
const actions = planEncoders({
|
|
430
|
+
coverage,
|
|
431
|
+
windows: [{ from: 0, to: 399, priority: 32 }],
|
|
432
|
+
runs: [],
|
|
433
|
+
...HOST,
|
|
434
|
+
maxRuns: 4
|
|
435
|
+
});
|
|
436
|
+
const spans = actions
|
|
437
|
+
.filter((action) => action.type === "start")
|
|
438
|
+
.map((action) => ({ from: action.from, to: action.to }))
|
|
439
|
+
.sort((left, right) => left.from - right.from);
|
|
440
|
+
|
|
441
|
+
for (let index = 0; index < spans.length - 1; index += 1) {
|
|
442
|
+
assert.ok(
|
|
443
|
+
spans[index].to < spans[index + 1].from,
|
|
444
|
+
`#${spans[index].from}..#${spans[index].to} overlaps #${spans[index + 1].from}`
|
|
445
|
+
);
|
|
446
|
+
}
|
|
447
|
+
});
|
package/test/piece-lru.test.js
CHANGED
|
@@ -252,3 +252,64 @@ test("with nothing declared there is no distance to report", () => {
|
|
|
252
252
|
"a pinned piece is never a candidate, and says so without a distance"
|
|
253
253
|
);
|
|
254
254
|
});
|
|
255
|
+
|
|
256
|
+
test("the least wanted piece goes, however recently it was touched", () => {
|
|
257
|
+
// Field 2026-09-05: fifteen reads declared fifteen windows on a store holding
|
|
258
|
+
// sixteen pieces, and with everything resident declared, eviction fell back
|
|
259
|
+
// to recency — 392 of 780 evictions took a piece a reader had said it wanted.
|
|
260
|
+
// Recency cannot separate them; the priority map's number can.
|
|
261
|
+
const lru = new PieceLru(3);
|
|
262
|
+
for (const index of [10, 20, 30]) {
|
|
263
|
+
lru.touch(index);
|
|
264
|
+
}
|
|
265
|
+
// 10 is the stalest, and the map wants it most: it is where a viewer is.
|
|
266
|
+
lru.protect("map:blocked", 10, 10, 0);
|
|
267
|
+
lru.protect("map:ahead", 20, 20, 2);
|
|
268
|
+
lru.protect("map:tail", 30, 30, 3);
|
|
269
|
+
|
|
270
|
+
const choice = lru.evictionChoice();
|
|
271
|
+
assert.equal(choice.index, 30, "eviction took the piece the map wants most");
|
|
272
|
+
assert.equal(choice.protectionYielded, true, "every resident piece was wanted, and it said so");
|
|
273
|
+
});
|
|
274
|
+
|
|
275
|
+
test("recency decides only between pieces the map wants equally", () => {
|
|
276
|
+
const lru = new PieceLru(3);
|
|
277
|
+
for (const index of [10, 20, 30]) {
|
|
278
|
+
lru.touch(index);
|
|
279
|
+
}
|
|
280
|
+
lru.protect("map:blocked", 10, 10, 0);
|
|
281
|
+
lru.protect("map:ahead", 20, 30, 2);
|
|
282
|
+
|
|
283
|
+
// 20 and 30 sit in one zone, so the stalest of the two goes.
|
|
284
|
+
assert.equal(lru.evictionCandidate(), 20);
|
|
285
|
+
lru.touch(20);
|
|
286
|
+
assert.equal(lru.evictionCandidate(), 30);
|
|
287
|
+
});
|
|
288
|
+
|
|
289
|
+
test("a piece no zone covers goes before any piece a zone covers", () => {
|
|
290
|
+
const lru = new PieceLru(4);
|
|
291
|
+
for (const index of [10, 20, 30, 40]) {
|
|
292
|
+
lru.touch(index);
|
|
293
|
+
}
|
|
294
|
+
// Only the last-touched piece is outside every zone — and it still goes
|
|
295
|
+
// first, because nobody has said they will read it.
|
|
296
|
+
lru.protect("map:tail", 10, 30, 4);
|
|
297
|
+
|
|
298
|
+
const choice = lru.evictionChoice();
|
|
299
|
+
assert.equal(choice.index, 40);
|
|
300
|
+
assert.equal(choice.protectionYielded, false, "nothing the map wanted was taken");
|
|
301
|
+
});
|
|
302
|
+
|
|
303
|
+
test("a claimant that states no number cannot displace one that did", () => {
|
|
304
|
+
const lru = new PieceLru(2);
|
|
305
|
+
lru.touch(10);
|
|
306
|
+
lru.touch(20);
|
|
307
|
+
lru.protect("map:tail", 10, 10, 4);
|
|
308
|
+
lru.protect("nameless", 20, 20);
|
|
309
|
+
|
|
310
|
+
assert.equal(
|
|
311
|
+
lru.evictionCandidate(),
|
|
312
|
+
20,
|
|
313
|
+
"a range with no stated number was treated as more wanted than the map's own"
|
|
314
|
+
);
|
|
315
|
+
});
|
|
@@ -489,3 +489,33 @@ test("a piece wanted later than the one it would displace goes to disk instead",
|
|
|
489
489
|
await fs.rm(directory, { recursive: true, force: true });
|
|
490
490
|
}
|
|
491
491
|
});
|
|
492
|
+
|
|
493
|
+
test("a piece the map wants more displaces one it wants less, however far away it is", async () => {
|
|
494
|
+
const capacity = 4;
|
|
495
|
+
const { store, directory } = await makeStore(capacity);
|
|
496
|
+
try {
|
|
497
|
+
// The store is full of the tail — wanted, but last of everything wanted.
|
|
498
|
+
store.protectRange("map:tail", 0, 3, 3);
|
|
499
|
+
for (let index = 0; index < capacity; index += 1) {
|
|
500
|
+
await put(store, index, pieceOf(index));
|
|
501
|
+
}
|
|
502
|
+
const before = store.stats().admittedToDisk;
|
|
503
|
+
|
|
504
|
+
// A piece arrives from far ahead in the file, so by distance to a read head
|
|
505
|
+
// it loses to everything resident — and it is where a viewer is stopped, so
|
|
506
|
+
// by the map it wins. The map decides; the distance separates only pieces
|
|
507
|
+
// the map wants equally.
|
|
508
|
+
store.protectRange("map:blocked", 90, 99, 0);
|
|
509
|
+
await put(store, 95, pieceOf(95));
|
|
510
|
+
|
|
511
|
+
assert.equal(
|
|
512
|
+
store.stats().admittedToDisk,
|
|
513
|
+
before,
|
|
514
|
+
"the piece a viewer is stopped on was written to disk to keep the tail in memory"
|
|
515
|
+
);
|
|
516
|
+
assert.ok((await get(store, 95)).equals(pieceOf(95)));
|
|
517
|
+
} finally {
|
|
518
|
+
store.destroy(() => undefined);
|
|
519
|
+
await fs.rm(directory, { recursive: true, force: true });
|
|
520
|
+
}
|
|
521
|
+
});
|