@torrent-tv/proxy 2.83.3 → 2.83.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/package.json +1 -1
- package/research/handover-reader-claims-removal-2026-09-12.md +166 -0
- package/research/piece-withdrawn-but-still-claimed-2026-09-12.md +175 -0
- package/research/priority-map-is-the-truth-2026-09-12.md +225 -0
- package/routes/api/sources/files/get.js +23 -9
- package/routes/api/sources/warm/post.js +26 -22
- package/routes/api/transcode-sessions/post.js +1 -49
- package/routes/stream/get.js +2 -36
- package/services/controllers/SubtitleController.js +0 -3
- package/services/download/withdraw-claim.js +80 -0
- package/services/hls-session-manager.js +17 -110
- package/services/orchestrators/EncodeOrchestrator.js +133 -0
- package/services/piece-store/piece-disk-store.js +24 -1
- package/services/piece-store/shared-piece-store.js +71 -1
- package/services/playback-planner.js +12 -13
- package/services/priority/PriorityOrchestrator.js +40 -6
- package/services/torrent/Contents.js +324 -0
- package/services/torrent/files.js +8 -1
- package/services/torrent-pool.js +378 -102
- package/services/torrent-worker/client.js +0 -24
- package/services/torrent-worker/piece-reader.js +30 -1
- package/services/torrent-worker/pool-adapter.js +3 -33
- package/services/torrent-worker/protocol.js +0 -4
- package/services/torrent-worker/worker.js +60 -60
- package/test/file-edges.test.js +191 -0
- package/test/input-lost-quiets-the-plan.test.js +261 -0
- package/test/logger-repeats.test.js +120 -0
- package/test/priority-map-emptied.test.js +207 -0
- package/test/read-survives-withdrawal.test.js +164 -0
- package/test/source-files-route.test.js +103 -0
- package/test/stream-route.test.js +4 -8
- package/test/swarm-follows-readers.test.js +96 -14
- package/test/torrent-contents.test.js +235 -0
- package/test/upload-hurry.test.js +66 -28
- package/test/withdraw-piece-claim.test.js +212 -0
- package/utils/logger.js +105 -7
- package/services/torrent-worker/file-claims.js +0 -91
- package/test/file-claims.test.js +0 -64
package/services/torrent-pool.js
CHANGED
|
@@ -18,12 +18,16 @@ import { logger } from "../utils/logger.js";
|
|
|
18
18
|
import { SharedPieceStore, findSharedStore } from "./piece-store/shared-piece-store.js";
|
|
19
19
|
import { Urgency, urgencyName } from "./demand/index.js";
|
|
20
20
|
import { demandFor, forgetTorrent, reconcileAll, hasUnmetDemand } from "./download/registry.js";
|
|
21
|
+
import { withdrawClaim } from "./download/withdraw-claim.js";
|
|
21
22
|
import { isAtAWatchingViewer, isBehindEverybody, isNobodyComingNow } from "./priority/PriorityMap.js";
|
|
22
23
|
import { deriveSourceKey } from "./torrent-source-key.js";
|
|
23
24
|
|
|
24
25
|
/** How a window stated from the priority map names itself. */
|
|
25
26
|
const MAP_CLAIMANT = "priority-map";
|
|
26
27
|
|
|
28
|
+
/** How the two ends of a file name themselves. */
|
|
29
|
+
const EDGES_CLAIMANT = "file-edges";
|
|
30
|
+
|
|
27
31
|
// The DHT's entry points. Two of the three the library ships answer nothing —
|
|
28
32
|
// measured 2026-08-21 from the addon host: `router.bittorrent.com` and
|
|
29
33
|
// `router.utorrent.com` did not reply to a hand-written `ping` at all, while a
|
|
@@ -331,29 +335,25 @@ function describeSwarmDemand(torrent) {
|
|
|
331
335
|
/**
|
|
332
336
|
* The torrents the upload policy is allowed to see.
|
|
333
337
|
*
|
|
334
|
-
* A torrent
|
|
335
|
-
* HURRY qualifies even without
|
|
336
|
-
* first
|
|
337
|
-
*
|
|
338
|
-
*
|
|
339
|
-
*
|
|
340
|
-
* measured 2026-08-04 — so the upload stayed at the near-silent idle floor
|
|
341
|
-
* during the exact seconds peers were deciding whether to serve us.
|
|
338
|
+
* A torrent something is wanted of qualifies for the obvious reason. A torrent
|
|
339
|
+
* in a HURRY qualifies even without that, and that case is the important one:
|
|
340
|
+
* the first seconds of a new torrent are when peers decide whether to serve us,
|
|
341
|
+
* and an upload held at the near-silent idle floor through them costs the whole
|
|
342
|
+
* ramp — 8.36 s of the 11.46 s before playback in the session measured
|
|
343
|
+
* 2026-08-04.
|
|
342
344
|
*
|
|
343
345
|
* @param {Iterable<{ hurryUntil?: number }>} torrents
|
|
344
|
-
* @param {Map<object, { size: number }>} usageByTorrent - fileIndex sets, keyed by torrent.
|
|
345
346
|
* @param {number} now
|
|
346
347
|
* @returns {object[]}
|
|
347
348
|
*/
|
|
348
|
-
export function torrentsForUploadPolicy(torrents,
|
|
349
|
+
export function torrentsForUploadPolicy(torrents, now) {
|
|
349
350
|
const chosen = [];
|
|
350
351
|
for (const torrent of torrents) {
|
|
351
|
-
const
|
|
352
|
-
|
|
353
|
-
if (hasReader || (torrent?.hurryUntil ?? 0) > now) {
|
|
352
|
+
const wanted = isWanted(torrent);
|
|
353
|
+
if (wanted || (torrent?.hurryUntil ?? 0) > now) {
|
|
354
354
|
// Recorded so the policy can tell "nothing is arriving and somebody is
|
|
355
355
|
// waiting" from "nothing is arriving because nobody asked".
|
|
356
|
-
torrent.
|
|
356
|
+
torrent.isWanted = wanted;
|
|
357
357
|
// Whether anybody is still short of bytes of it. Upload is bought with
|
|
358
358
|
// reciprocity and reciprocity is only worth buying while something is
|
|
359
359
|
// missing; a torrent whose declared windows are all present wants nothing
|
|
@@ -398,13 +398,13 @@ export function decideUploadLimit(activeTorrents, opts = {}) {
|
|
|
398
398
|
// iterate the piece array and throw on webtorrent 3.x when a piece is null
|
|
399
399
|
// (deselected / mid-verify), which would crash this timer every cycle.
|
|
400
400
|
const notDone = torrent?.done !== true;
|
|
401
|
-
// A torrent
|
|
402
|
-
// looks. The encoder is held back once it is far enough ahead of the
|
|
401
|
+
// A torrent nothing is wanted of is not starving, however still its
|
|
402
|
+
// download looks. The encoder is held back once it is far enough ahead of the
|
|
403
403
|
// viewer, and while it is held nothing is requested — measured
|
|
404
404
|
// 2026-08-04: four cycles of 512 -> 50 KB/s in three minutes, each
|
|
405
405
|
// reported as `earn unchoke ... down=0KB/s`, all of them raising the
|
|
406
406
|
// upload at moments when no byte was wanted by anyone.
|
|
407
|
-
const starving = notDone && torrent?.
|
|
407
|
+
const starving = notDone && torrent?.isWanted !== false
|
|
408
408
|
&& torrent?.hasUnmetDemand !== false && downloadSpeed < starvingSpeed;
|
|
409
409
|
if (starving && chokedInterested >= chokedThreshold) {
|
|
410
410
|
const name = typeof torrent?.name === "string" ? torrent.name : "?";
|
|
@@ -789,6 +789,176 @@ export function dhtNodeCount(client) {
|
|
|
789
789
|
// without building a pool, a torrent client or a thread.
|
|
790
790
|
const lastMapSaid = new WeakMap();
|
|
791
791
|
|
|
792
|
+
/**
|
|
793
|
+
* Take back what the priority map had stated for one file, from a band onwards.
|
|
794
|
+
*
|
|
795
|
+
* Every band of a map is a claimant of its own, numbered, so a map that has
|
|
796
|
+
* shrunk — or emptied — is applied by withdrawing the numbers it no longer
|
|
797
|
+
* reaches. Written once because both callers need exactly it: the ordinary
|
|
798
|
+
* application, where the count is how many bands there now are, and a file
|
|
799
|
+
* nobody wants any more, where it is none.
|
|
800
|
+
*
|
|
801
|
+
* A function of a torrent rather than of the pool, like `leaveSwarm` beside it:
|
|
802
|
+
* it needs nothing the pool holds, and it can be exercised without building one.
|
|
803
|
+
*
|
|
804
|
+
* @param {object} torrent
|
|
805
|
+
* @param {number} fileIndex
|
|
806
|
+
* @param {number} [keepBelow] - Bands numbered below this are left alone.
|
|
807
|
+
* @returns {number} How many were withdrawn.
|
|
808
|
+
*/
|
|
809
|
+
function withdrawPriorityMap(torrent, fileIndex, keepBelow = 0) {
|
|
810
|
+
const { register } = demandFor(torrent);
|
|
811
|
+
const prefix = `${MAP_CLAIMANT}:${fileIndex}:`;
|
|
812
|
+
let withdrawn = 0;
|
|
813
|
+
for (const window of register.windows()) {
|
|
814
|
+
const claimant = String(window.claimant);
|
|
815
|
+
if (!claimant.startsWith(prefix)) {
|
|
816
|
+
continue;
|
|
817
|
+
}
|
|
818
|
+
const index = Number(claimant.split(":").pop());
|
|
819
|
+
if (!(index < keepBelow)) {
|
|
820
|
+
register.withdraw(claimant);
|
|
821
|
+
withdrawn += 1;
|
|
822
|
+
}
|
|
823
|
+
}
|
|
824
|
+
if (withdrawn > 0 && keepBelow === 0) {
|
|
825
|
+
// Said, because it is the one statement that cannot be read from what
|
|
826
|
+
// follows it: nothing further is stated about this file, so from here on
|
|
827
|
+
// silence is exactly what it looks like.
|
|
828
|
+
logger.info(
|
|
829
|
+
`torrent-pool: [${String(torrent?.infoHash ?? "?").slice(0, 8)}] nobody wants file ${fileIndex} ` +
|
|
830
|
+
`any more — ${withdrawn} band(s) of the map withdrawn`
|
|
831
|
+
);
|
|
832
|
+
}
|
|
833
|
+
return withdrawn;
|
|
834
|
+
}
|
|
835
|
+
|
|
836
|
+
/**
|
|
837
|
+
* WHERE A FILE SAYS WHAT IT IS: the piece at each of its ends.
|
|
838
|
+
*
|
|
839
|
+
* A container keeps its directory at one end or the other — `ftyp` and an EBML
|
|
840
|
+
* header at the front, and for an MP4 that was not written for streaming the
|
|
841
|
+
* `moov` at the very back. Nothing can be read of such a file until those have
|
|
842
|
+
* arrived, and they stay wanted for as long as it is open: a player asks for
|
|
843
|
+
* the file's shape again at every seek.
|
|
844
|
+
*
|
|
845
|
+
* **The size is not chosen, and it cannot be.** One byte is claimed at each
|
|
846
|
+
* end, and the piece is what the swarm delivers — so one byte at each end IS
|
|
847
|
+
* one piece at each end, with nothing rounded up by a number anybody picked.
|
|
848
|
+
* The 256 KB and 2 MB the prefetch reads are its own affair; where a directory
|
|
849
|
+
* is bigger than the piece it starts in, the read that needs it says so itself,
|
|
850
|
+
* at the level of a reader that is stopped.
|
|
851
|
+
*
|
|
852
|
+
* **Stated by the FILE, not by the read that happens to want it.** A read
|
|
853
|
+
* withdraws what it stated the moment it finishes, so until now the two ends of
|
|
854
|
+
* an open film were held by nothing at all once the codec probe was done — and
|
|
855
|
+
* they are exactly the pieces a seek needs and eviction is free to take.
|
|
856
|
+
*
|
|
857
|
+
* A function of a torrent rather than of the pool, like `leaveSwarm` beside it,
|
|
858
|
+
* so both can be exercised without building one.
|
|
859
|
+
*
|
|
860
|
+
* @param {object} torrent
|
|
861
|
+
* @param {number} fileIndex
|
|
862
|
+
* @param {number} urgency - NEAR while somebody is waiting for them, TAIL to
|
|
863
|
+
* keep them afterwards.
|
|
864
|
+
* @returns {boolean} Whether anything was stated.
|
|
865
|
+
*/
|
|
866
|
+
export function stateFileEdges(torrent, fileIndex, urgency, { lower = false } = {}) {
|
|
867
|
+
const file = Array.isArray(torrent?.files) ? torrent.files[fileIndex] : null;
|
|
868
|
+
const length = Number(file?.length);
|
|
869
|
+
if (!file || !(length > 0)) {
|
|
870
|
+
return false;
|
|
871
|
+
}
|
|
872
|
+
const { register } = demandFor(torrent);
|
|
873
|
+
const ends = [
|
|
874
|
+
{ claimant: `${EDGES_CLAIMANT}:${fileIndex}:head`, byteStart: 0, byteEnd: 0 },
|
|
875
|
+
{ claimant: `${EDGES_CLAIMANT}:${fileIndex}:tail`, byteStart: length - 1, byteEnd: length - 1 }
|
|
876
|
+
];
|
|
877
|
+
for (const end of ends) {
|
|
878
|
+
// RAISED, NEVER LOWERED, unless the caller is the one putting them back
|
|
879
|
+
// down. Two things ask for the same ends — a warm-up that nobody is waiting
|
|
880
|
+
// for and a playback plan that somebody is — and the warm-up's sidecars are
|
|
881
|
+
// fired off without being awaited, so it can arrive second. Taking the
|
|
882
|
+
// plan's urgency away there would leave a person watching a loading screen
|
|
883
|
+
// behind a film somebody else is watching.
|
|
884
|
+
const stated = register.windows().find((window) => String(window.claimant) === end.claimant);
|
|
885
|
+
const level = !lower && stated ? Math.min(stated.urgency, urgency) : urgency;
|
|
886
|
+
register.state({ ...end, fileIndex, urgency: level });
|
|
887
|
+
}
|
|
888
|
+
return true;
|
|
889
|
+
}
|
|
890
|
+
|
|
891
|
+
/**
|
|
892
|
+
* Give up the ends of a file nobody has any use for.
|
|
893
|
+
*
|
|
894
|
+
* @param {object} torrent
|
|
895
|
+
* @param {number} fileIndex
|
|
896
|
+
* @returns {number} How many were withdrawn.
|
|
897
|
+
*/
|
|
898
|
+
export function withdrawFileEdges(torrent, fileIndex) {
|
|
899
|
+
const { register } = demandFor(torrent);
|
|
900
|
+
let withdrawn = 0;
|
|
901
|
+
for (const end of ["head", "tail"]) {
|
|
902
|
+
const claimant = `${EDGES_CLAIMANT}:${fileIndex}:${end}`;
|
|
903
|
+
if (register.windows().some((window) => String(window.claimant) === claimant)) {
|
|
904
|
+
register.withdraw(claimant);
|
|
905
|
+
withdrawn += 1;
|
|
906
|
+
}
|
|
907
|
+
}
|
|
908
|
+
return withdrawn;
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
/**
|
|
912
|
+
* IS ANYTHING WANTED OF THIS TORRENT — the one question asked about a torrent's
|
|
913
|
+
* life, and it is answered by what has been stated, never by counting who is
|
|
914
|
+
* reading.
|
|
915
|
+
*
|
|
916
|
+
* Everything that wants bytes says so in the register: the priority map for
|
|
917
|
+
* every file somebody is watching, the ends of a file that is open, the
|
|
918
|
+
* background fill, and a read that is stopped on a piece. So "nothing is
|
|
919
|
+
* stated" is the whole of "nobody wants this", and it becomes true exactly when
|
|
920
|
+
* the last viewer leaves — because that is when the map is published with
|
|
921
|
+
* nothing in it.
|
|
922
|
+
*
|
|
923
|
+
* **A torrent that cannot yet be stated about is wanted.** Until its metadata
|
|
924
|
+
* arrives it has no files, so nothing can name a byte of it, and it is being
|
|
925
|
+
* fetched precisely because somebody asked for it. Judged by the register alone
|
|
926
|
+
* it would look abandoned three seconds after it was added, which is the
|
|
927
|
+
* failure of 2.83.1 exactly.
|
|
928
|
+
*
|
|
929
|
+
* @param {object} torrent
|
|
930
|
+
* @returns {boolean}
|
|
931
|
+
*/
|
|
932
|
+
export function isWanted(torrent) {
|
|
933
|
+
if (!torrent || torrent.destroyed) {
|
|
934
|
+
return false;
|
|
935
|
+
}
|
|
936
|
+
if (!Array.isArray(torrent.files) || torrent.files.length === 0) {
|
|
937
|
+
return true;
|
|
938
|
+
}
|
|
939
|
+
return demandFor(torrent).register.size > 0;
|
|
940
|
+
}
|
|
941
|
+
|
|
942
|
+
/**
|
|
943
|
+
* WHAT TO DO ABOUT A TORRENT'S SWARM, from the two facts that decide it.
|
|
944
|
+
*
|
|
945
|
+
* Separated from the doing because this is the rule that has failed twice in
|
|
946
|
+
* the field, and a rule that can only be exercised by building a pool with a
|
|
947
|
+
* live WebTorrent client is a rule nothing checks. Both failures are cases of
|
|
948
|
+
* it: 2.83.1 let a swarm go for a torrent that had never been wanted, and the
|
|
949
|
+
* version before that never let one go at all.
|
|
950
|
+
*
|
|
951
|
+
* @param {{ wanted: boolean, everWanted: boolean }} facts
|
|
952
|
+
* @returns {{ swarm: "take" | "let go" | "leave alone", onTheClock: boolean }}
|
|
953
|
+
* What to do with the swarm, and whether the idle clock should be running.
|
|
954
|
+
*/
|
|
955
|
+
export function swarmDecisionFor({ wanted, everWanted }) {
|
|
956
|
+
if (wanted) {
|
|
957
|
+
return { swarm: "take", onTheClock: false };
|
|
958
|
+
}
|
|
959
|
+
return { swarm: everWanted ? "let go" : "leave alone", onTheClock: true };
|
|
960
|
+
}
|
|
961
|
+
|
|
792
962
|
export class TorrentPool {
|
|
793
963
|
/**
|
|
794
964
|
* In-flight `client.add()` promises keyed by the same key as `torrents`.
|
|
@@ -929,6 +1099,79 @@ export class TorrentPool {
|
|
|
929
1099
|
this.#storeExtras = extras && typeof extras === "object" ? extras : {};
|
|
930
1100
|
}
|
|
931
1101
|
|
|
1102
|
+
/**
|
|
1103
|
+
* Everything a store of this pool is built with, in ONE place.
|
|
1104
|
+
*
|
|
1105
|
+
* It was assembled at both `client.add` sites — the ordinary one and the one
|
|
1106
|
+
* that replaces a torrent the client no longer has — and a store built by the
|
|
1107
|
+
* second was missing whatever the first had gained. That is how a store came
|
|
1108
|
+
* to exist without the whole-file reader it needs, and it is why this is a
|
|
1109
|
+
* method rather than a literal.
|
|
1110
|
+
*
|
|
1111
|
+
* @returns {object}
|
|
1112
|
+
*/
|
|
1113
|
+
#storeOptions() {
|
|
1114
|
+
return {
|
|
1115
|
+
memoryBytes: this.#memoryBytes,
|
|
1116
|
+
...this.#storeExtras,
|
|
1117
|
+
// WIRED BY THE POOL ITSELF, not handed in: withdrawing the claim needs
|
|
1118
|
+
// only a torrent, and this class is the one that owns them. Last, so a
|
|
1119
|
+
// caller's extras cannot displace the one thing that keeps the library's
|
|
1120
|
+
// completion bitfield in step with the bytes.
|
|
1121
|
+
onPieceGone: (what) => this.withdrawPieceClaim(what)
|
|
1122
|
+
};
|
|
1123
|
+
}
|
|
1124
|
+
|
|
1125
|
+
/**
|
|
1126
|
+
* Withdraw the claim that this proxy has a piece, because the store that held
|
|
1127
|
+
* it no longer can produce it.
|
|
1128
|
+
*
|
|
1129
|
+
* **Whose job this is.** The store owns the bytes and therefore owns the fact;
|
|
1130
|
+
* the library keeps a second copy of that fact in its completion bitfield, and
|
|
1131
|
+
* this pool is the only thing here that owns a torrent, so reconciling the two
|
|
1132
|
+
* is this class's and nothing else's. The store announces and does not know
|
|
1133
|
+
* who listens.
|
|
1134
|
+
*
|
|
1135
|
+
* **What it costs.** The piece is re-created as incomplete, so the next read
|
|
1136
|
+
* that wants it waits for a download instead of failing. Field 2026-09-12,
|
|
1137
|
+
* which is what this is for: 565 spilled pieces were dropped behind the read
|
|
1138
|
+
* heads — correctly, they were behind every reader — the bitfield went on
|
|
1139
|
+
* saying they were verified, the encoder restarted and re-opened its input at
|
|
1140
|
+
* byte 0, and `Piece 0 is verified but absent from the store` then answered
|
|
1141
|
+
* every read for 92 minutes. Nothing re-downloaded it, because the library
|
|
1142
|
+
* does not fetch what it believes it owns.
|
|
1143
|
+
*
|
|
1144
|
+
* **What it deliberately does not do.** `_markUnverified` would also re-select
|
|
1145
|
+
* the piece, and does not here: every torrent is added with `deselect: true`,
|
|
1146
|
+
* which sets the library's own `_startAsDeselected` and makes it skip that
|
|
1147
|
+
* call. The download set has one owner — `SwarmSelection`, from the priority
|
|
1148
|
+
* map — and a piece withdrawn here is fetched again when a read states it,
|
|
1149
|
+
* which is the same statement every other piece waits on.
|
|
1150
|
+
*
|
|
1151
|
+
* @param {object} what
|
|
1152
|
+
* @param {number} what.index
|
|
1153
|
+
* @param {object[]} what.files - The store's own, from which the torrent that
|
|
1154
|
+
* owns them is found; the store has no idea what a torrent is.
|
|
1155
|
+
* @returns {void}
|
|
1156
|
+
*/
|
|
1157
|
+
withdrawPieceClaim({ index, files }) {
|
|
1158
|
+
const outcome = withdrawClaim({
|
|
1159
|
+
index,
|
|
1160
|
+
files,
|
|
1161
|
+
warn: (line) => logger.warn(`torrent-pool: ${line}`)
|
|
1162
|
+
});
|
|
1163
|
+
if (outcome === "withdrawn") {
|
|
1164
|
+
this.#claimsWithdrawn += 1;
|
|
1165
|
+
}
|
|
1166
|
+
}
|
|
1167
|
+
|
|
1168
|
+
/** How many claims this pool has withdrawn, over its whole life. */
|
|
1169
|
+
#claimsWithdrawn = 0;
|
|
1170
|
+
|
|
1171
|
+
get claimsWithdrawn() {
|
|
1172
|
+
return this.#claimsWithdrawn;
|
|
1173
|
+
}
|
|
1174
|
+
|
|
932
1175
|
constructor({ maxDiskBytes, memoryBytes, dhtBootstrap } = {}) {
|
|
933
1176
|
this.#memoryBytes = Number.isFinite(memoryBytes) && memoryBytes > 0 ? memoryBytes : undefined;
|
|
934
1177
|
|
|
@@ -984,7 +1227,6 @@ export class TorrentPool {
|
|
|
984
1227
|
*
|
|
985
1228
|
* @type {WeakMap<import("webtorrent").Torrent, Map<number, number>>}
|
|
986
1229
|
*/
|
|
987
|
-
this.fileUsageByTorrent = new WeakMap();
|
|
988
1230
|
|
|
989
1231
|
this.client.on("error", (error) => {
|
|
990
1232
|
logger.error(`WebTorrent client error: ${error.message}`);
|
|
@@ -1115,13 +1357,14 @@ export class TorrentPool {
|
|
|
1115
1357
|
*/
|
|
1116
1358
|
#stateBackgroundFill(torrent) {
|
|
1117
1359
|
const { register } = demandFor(torrent);
|
|
1118
|
-
const usage = this.fileUsageByTorrent.get(torrent);
|
|
1119
1360
|
const pieceLength = Number(torrent.pieceLength);
|
|
1120
1361
|
if (!Number.isFinite(pieceLength) || pieceLength <= 0) {
|
|
1121
1362
|
return;
|
|
1122
1363
|
}
|
|
1123
|
-
for
|
|
1124
|
-
|
|
1364
|
+
// The files anything is stated for — which is the same list the reader
|
|
1365
|
+
// counts used to give and is one fact rather than two.
|
|
1366
|
+
for (const fileIndex of register.files()) {
|
|
1367
|
+
const file = torrent.files?.[fileIndex] ?? null;
|
|
1125
1368
|
const claimant = `background-fill:${fileIndex}`;
|
|
1126
1369
|
if (!file) {
|
|
1127
1370
|
register.withdraw(claimant);
|
|
@@ -1169,6 +1412,22 @@ export class TorrentPool {
|
|
|
1169
1412
|
* @param {number} durationSeconds
|
|
1170
1413
|
*/
|
|
1171
1414
|
applyPriorityMap(torrent, fileIndex, zones, durationSeconds) {
|
|
1415
|
+
// A MAP WITH NOTHING IN IT IS A STATEMENT, and it is answered before
|
|
1416
|
+
// anything else is looked at. It says nobody wants any of this file, and
|
|
1417
|
+
// that is true whatever the file's length is or whether the torrent still
|
|
1418
|
+
// holds a list — so making it wait on either would be making the departure
|
|
1419
|
+
// depend on facts that are gone by the time it is said.
|
|
1420
|
+
if (Array.isArray(zones) && zones.length === 0) {
|
|
1421
|
+
withdrawPriorityMap(torrent, fileIndex);
|
|
1422
|
+
// The ends of the file go with it. They are kept for as long as the file
|
|
1423
|
+
// is open, and this is what says it is not.
|
|
1424
|
+
withdrawFileEdges(torrent, fileIndex);
|
|
1425
|
+
// And this may have been the last thing anybody wanted of this torrent.
|
|
1426
|
+
// Optional because these two are functions of a torrent and are exercised
|
|
1427
|
+
// as such; called on the pool, the pool also acts on what they said.
|
|
1428
|
+
this?.followTheDemand?.(torrent);
|
|
1429
|
+
return;
|
|
1430
|
+
}
|
|
1172
1431
|
const file = Array.isArray(torrent?.files) ? torrent.files[fileIndex] : null;
|
|
1173
1432
|
const length = Number(file?.length);
|
|
1174
1433
|
const duration = Number(durationSeconds);
|
|
@@ -1243,15 +1502,7 @@ export class TorrentPool {
|
|
|
1243
1502
|
});
|
|
1244
1503
|
// Bands the map no longer has: a viewer moved on, and what they had wanted
|
|
1245
1504
|
// is not wanted by anybody now.
|
|
1246
|
-
|
|
1247
|
-
if (!String(window.claimant).startsWith(`${MAP_CLAIMANT}:${fileIndex}:`)) {
|
|
1248
|
-
continue;
|
|
1249
|
-
}
|
|
1250
|
-
const index = Number(String(window.claimant).split(":").pop());
|
|
1251
|
-
if (!(index < ordered.length)) {
|
|
1252
|
-
register.withdraw(window.claimant);
|
|
1253
|
-
}
|
|
1254
|
-
}
|
|
1505
|
+
withdrawPriorityMap(torrent, fileIndex, ordered.length);
|
|
1255
1506
|
// WHAT THE SWARM WAS ACTUALLY TOLD, said on change and never on a timer.
|
|
1256
1507
|
// The map was applied in silence: that it had been BUILT was visible in the
|
|
1257
1508
|
// encoding's own line, and that the download had received it was visible
|
|
@@ -1275,6 +1526,9 @@ export class TorrentPool {
|
|
|
1275
1526
|
.sort((left, right) => right[0] - left[0])
|
|
1276
1527
|
.map(([level, held]) => `${urgencyName(level)} ${held.zones} zone(s) ${held.megabytes.toFixed(0)}MB`)
|
|
1277
1528
|
.join(", ");
|
|
1529
|
+
// Something is wanted of this torrent, which may be news: a map published
|
|
1530
|
+
// for a file of a torrent whose swarm was let go is what takes it back.
|
|
1531
|
+
this?.followTheDemand?.(torrent);
|
|
1278
1532
|
const said = `${fileIndex}:${shape}`;
|
|
1279
1533
|
if (lastMapSaid.get(torrent) !== said) {
|
|
1280
1534
|
lastMapSaid.set(torrent, said);
|
|
@@ -1288,8 +1542,7 @@ export class TorrentPool {
|
|
|
1288
1542
|
#reportStalledDownloads() {
|
|
1289
1543
|
const now = Date.now();
|
|
1290
1544
|
for (const torrent of this.torrents.values()) {
|
|
1291
|
-
|
|
1292
|
-
if (!usage || usage.size === 0 || torrent?.done === true) {
|
|
1545
|
+
if (!isWanted(torrent) || torrent?.done === true) {
|
|
1293
1546
|
continue;
|
|
1294
1547
|
}
|
|
1295
1548
|
const speed = typeof torrent.downloadSpeed === "number" ? torrent.downloadSpeed : 0;
|
|
@@ -1329,22 +1582,23 @@ export class TorrentPool {
|
|
|
1329
1582
|
if (!this.client || this.client.destroyed || typeof this.client.throttleUpload !== "function") {
|
|
1330
1583
|
return;
|
|
1331
1584
|
}
|
|
1332
|
-
const active = torrentsForUploadPolicy(
|
|
1333
|
-
this.torrents.values(),
|
|
1334
|
-
this.fileUsageByTorrent,
|
|
1335
|
-
Date.now()
|
|
1336
|
-
);
|
|
1585
|
+
const active = torrentsForUploadPolicy(this.torrents.values(), Date.now());
|
|
1337
1586
|
// The one place the swarm is told anything: it reads what everybody has
|
|
1338
1587
|
// stated and works out for itself what to ask for, including whether the
|
|
1339
1588
|
// speculative levels may be stated at all — which is a question about every
|
|
1340
1589
|
// torrent at once, because they share the link.
|
|
1341
1590
|
reconcileAll();
|
|
1342
1591
|
for (const torrent of this.torrents.values()) {
|
|
1343
|
-
|
|
1344
|
-
if (!usage || usage.size === 0 || torrent?.done === true) {
|
|
1592
|
+
if (!isWanted(torrent) || torrent?.done === true) {
|
|
1345
1593
|
continue;
|
|
1346
1594
|
}
|
|
1347
1595
|
this.#stateBackgroundFill(torrent);
|
|
1596
|
+
// And act on what that just said. The fill withdraws itself once nothing
|
|
1597
|
+
// else wants the file, so a torrent everybody has left can be held by the
|
|
1598
|
+
// fill's own claim until this pass — and without this line the moment it
|
|
1599
|
+
// goes would be noticed by nobody, leaving the torrent in its swarm with
|
|
1600
|
+
// no idle clock running.
|
|
1601
|
+
this.followTheDemand(torrent);
|
|
1348
1602
|
}
|
|
1349
1603
|
this.#reportStalledDownloads();
|
|
1350
1604
|
const { bytesPerSec, reason } = decideUploadLimit(active);
|
|
@@ -1385,12 +1639,10 @@ export class TorrentPool {
|
|
|
1385
1639
|
if (used <= this.#maxDiskBytes) {
|
|
1386
1640
|
return;
|
|
1387
1641
|
}
|
|
1388
|
-
// Candidates: pooled torrents
|
|
1642
|
+
// Candidates: pooled torrents nothing is wanted of, the longest unwanted
|
|
1643
|
+
// first.
|
|
1389
1644
|
const candidates = [...this.torrents.values()]
|
|
1390
|
-
.filter((torrent) =>
|
|
1391
|
-
const usage = this.fileUsageByTorrent.get(torrent);
|
|
1392
|
-
return !usage || usage.size === 0;
|
|
1393
|
-
})
|
|
1645
|
+
.filter((torrent) => !isWanted(torrent))
|
|
1394
1646
|
.sort(
|
|
1395
1647
|
(earlier, later) =>
|
|
1396
1648
|
(this.#lastAccess.get(earlier) ?? 0) - (this.#lastAccess.get(later) ?? 0)
|
|
@@ -1658,14 +1910,13 @@ export class TorrentPool {
|
|
|
1658
1910
|
this.torrents.delete(otherKey);
|
|
1659
1911
|
}
|
|
1660
1912
|
}
|
|
1661
|
-
this.fileUsageByTorrent.delete(existing);
|
|
1662
1913
|
this.#lastAccess.delete(existing);
|
|
1663
1914
|
this.#readPositionByTorrent.delete(existing);
|
|
1664
1915
|
this.client.remove(existing, { destroyStore: true }, () => {
|
|
1665
1916
|
const addedReplacement = this.client.add(torrentId, {
|
|
1666
1917
|
store: SharedPieceStore,
|
|
1667
1918
|
storeCacheSlots: 0,
|
|
1668
|
-
storeOpts:
|
|
1919
|
+
storeOpts: this.#storeOptions(),
|
|
1669
1920
|
deselect: true
|
|
1670
1921
|
}, (replacement) => {
|
|
1671
1922
|
this.torrents.set(key, replacement);
|
|
@@ -1698,7 +1949,7 @@ export class TorrentPool {
|
|
|
1698
1949
|
const added = this.client.add(torrentId, {
|
|
1699
1950
|
store: SharedPieceStore,
|
|
1700
1951
|
storeCacheSlots: 0,
|
|
1701
|
-
storeOpts:
|
|
1952
|
+
storeOpts: this.#storeOptions(),
|
|
1702
1953
|
// Nothing is fetched until somebody says they want it. WebTorrent's own
|
|
1703
1954
|
// default is `this.select(0, this.pieces.length - 1)` — the whole
|
|
1704
1955
|
// torrent — and this proxy used to undo that afterwards by deselecting
|
|
@@ -1722,6 +1973,11 @@ export class TorrentPool {
|
|
|
1722
1973
|
// leave the survivor unwatched. Attaching twice costs nothing — the
|
|
1723
1974
|
// guard makes the second call a no-op when it is the same object.
|
|
1724
1975
|
this.#attachSwarmDiagnostics(readyTorrent);
|
|
1976
|
+
// On the clock from the moment it exists. A torrent nobody ever states
|
|
1977
|
+
// anything about — a file list fetched and never played — used to be
|
|
1978
|
+
// held for the life of the process, because the only thing that ever
|
|
1979
|
+
// started the idle timer was a reader letting go.
|
|
1980
|
+
this.followTheDemand(readyTorrent);
|
|
1725
1981
|
resolve(readyTorrent);
|
|
1726
1982
|
});
|
|
1727
1983
|
// Attached to what `add` returns, NOT inside its callback. That callback
|
|
@@ -1740,59 +1996,61 @@ export class TorrentPool {
|
|
|
1740
1996
|
}
|
|
1741
1997
|
|
|
1742
1998
|
/**
|
|
1743
|
-
*
|
|
1744
|
-
*
|
|
1745
|
-
*
|
|
1999
|
+
* Torrents something has been wanted of at least once, which is what tells a
|
|
2000
|
+
* departure from a beginning.
|
|
2001
|
+
*
|
|
2002
|
+
* @type {WeakSet<object>}
|
|
2003
|
+
*/
|
|
2004
|
+
#wasWanted = new WeakSet();
|
|
2005
|
+
|
|
2006
|
+
/**
|
|
2007
|
+
* ACT ON WHAT IS WANTED OF THIS TORRENT, and on nothing else.
|
|
2008
|
+
*
|
|
2009
|
+
* Called wherever what is wanted changes — a priority map applied or emptied,
|
|
2010
|
+
* the ends of a file stated — because those are the moments, and a pass that
|
|
2011
|
+
* asks every torrent every few seconds has the wrong answer between two of
|
|
2012
|
+
* them. That is not a theory: 2.83.1 asked such a question every five seconds
|
|
2013
|
+
* and a torrent added three seconds earlier answered "nobody", so it left its
|
|
2014
|
+
* swarm with 741 connections let go and playback never started.
|
|
2015
|
+
*
|
|
2016
|
+
* What it does NOT do is decide whether a torrent is wanted. That is
|
|
2017
|
+
* {@link isWanted}, which reads what has been stated, and the only reason
|
|
2018
|
+
* this is a method at all is that the idle timer and the eviction order are
|
|
2019
|
+
* the pool's own.
|
|
1746
2020
|
*
|
|
1747
2021
|
* @param {import("webtorrent").Torrent} torrent
|
|
1748
|
-
* @
|
|
1749
|
-
* @returns {() => void} Release function — call it once when done streaming.
|
|
2022
|
+
* @returns {void}
|
|
1750
2023
|
*/
|
|
1751
|
-
|
|
1752
|
-
if (!torrent ||
|
|
1753
|
-
return
|
|
2024
|
+
followTheDemand(torrent) {
|
|
2025
|
+
if (!torrent || torrent.destroyed) {
|
|
2026
|
+
return;
|
|
1754
2027
|
}
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
2028
|
+
const { swarm, onTheClock } = swarmDecisionFor({
|
|
2029
|
+
wanted: isWanted(torrent),
|
|
2030
|
+
everWanted: this.#wasWanted.has(torrent)
|
|
2031
|
+
});
|
|
2032
|
+
if (swarm === "take") {
|
|
2033
|
+
this.#wasWanted.add(torrent);
|
|
2034
|
+
this.#cancelIdleRemoval(torrent);
|
|
2035
|
+
// When it was last wanted, which is the order the disk cap evicts in.
|
|
2036
|
+
this.#lastAccess.set(torrent, Date.now());
|
|
2037
|
+
rejoinSwarm(torrent);
|
|
2038
|
+
}
|
|
2039
|
+
// A PROXY THAT NEEDS NOTHING FROM A SWARM SHOULD NOT BE IN THAT SWARM. The
|
|
2040
|
+
// data stays on disk; the next thing that wants a byte of it takes the
|
|
2041
|
+
// swarm back.
|
|
2042
|
+
if (swarm === "let go") {
|
|
2043
|
+
leaveSwarm(torrent);
|
|
2044
|
+
}
|
|
2045
|
+
if (onTheClock) {
|
|
2046
|
+
this.#scheduleIdleRemoval(torrent);
|
|
1759
2047
|
}
|
|
1760
|
-
// The torrent is in use again — cancel any pending idle removal and mark
|
|
1761
|
-
// it recently accessed so LRU eviction keeps it.
|
|
1762
|
-
this.#cancelIdleRemoval(torrent);
|
|
1763
|
-
this.#lastAccess.set(torrent, Date.now());
|
|
1764
|
-
// AND REJOIN ITS SWARM, which is the other half of the pair: a departure
|
|
1765
|
-
// lets a swarm go, an arrival takes it back. Both are acted on at the
|
|
1766
|
-
// moment they happen, and neither is discovered by asking.
|
|
1767
|
-
rejoinSwarm(torrent);
|
|
1768
|
-
usage.set(fileIndex, (usage.get(fileIndex) ?? 0) + 1);
|
|
1769
|
-
|
|
1770
|
-
let released = false;
|
|
1771
|
-
return () => {
|
|
1772
|
-
if (released) {
|
|
1773
|
-
return;
|
|
1774
|
-
}
|
|
1775
|
-
released = true;
|
|
1776
|
-
const nextCount = (usage.get(fileIndex) ?? 0) - 1;
|
|
1777
|
-
if (nextCount > 0) {
|
|
1778
|
-
usage.set(fileIndex, nextCount);
|
|
1779
|
-
} else {
|
|
1780
|
-
usage.delete(fileIndex);
|
|
1781
|
-
}
|
|
1782
|
-
if (usage.size === 0) {
|
|
1783
|
-
this.fileUsageByTorrent.delete(torrent);
|
|
1784
|
-
// No active readers — schedule removal (with store) after an idle TTL.
|
|
1785
|
-
this.#scheduleIdleRemoval(torrent);
|
|
1786
|
-
// THE READER LEFT, AND THAT IS THE ONLY MOMENT A SWARM MAY BE LET GO.
|
|
1787
|
-
leaveSwarm(torrent);
|
|
1788
|
-
}
|
|
1789
|
-
};
|
|
1790
2048
|
}
|
|
1791
2049
|
|
|
1792
2050
|
/**
|
|
1793
2051
|
* Schedule removal of a torrent (with its on-disk store) after
|
|
1794
|
-
* {@link TORRENT_IDLE_TTL_MS} of
|
|
1795
|
-
* any existing timer for the torrent.
|
|
2052
|
+
* {@link TORRENT_IDLE_TTL_MS} of nothing being wanted of it. Idempotent —
|
|
2053
|
+
* replaces any existing timer for the torrent.
|
|
1796
2054
|
*
|
|
1797
2055
|
* @param {import("webtorrent").Torrent} torrent
|
|
1798
2056
|
* @returns {void}
|
|
@@ -1807,11 +2065,13 @@ export class TorrentPool {
|
|
|
1807
2065
|
logger.info(`torrent-pool: scheduling idle removal for "${name}" [${infoHashShort}] in ${TORRENT_IDLE_TTL_MS / 1000}s`);
|
|
1808
2066
|
const timer = setTimeout(() => {
|
|
1809
2067
|
this.#idleTimers.delete(torrent);
|
|
1810
|
-
// Re-check:
|
|
1811
|
-
//
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
2068
|
+
// Re-check: anything stated since would have cancelled this timer, but
|
|
2069
|
+
// guard anyway against a race.
|
|
2070
|
+
if (isWanted(torrent)) {
|
|
2071
|
+
logger.info(
|
|
2072
|
+
`torrent-pool: idle timer fired for "${name}" [${infoHashShort}] but ` +
|
|
2073
|
+
`${demandFor(torrent).register.size} thing(s) are stated for it — keep`
|
|
2074
|
+
);
|
|
1815
2075
|
return;
|
|
1816
2076
|
}
|
|
1817
2077
|
logger.warn(`torrent-pool: idle TTL fired for "${name}" [${infoHashShort}] — removing torrent (reason=idle-ttl caller=scheduleIdleRemoval)`);
|
|
@@ -1871,7 +2131,6 @@ export class TorrentPool {
|
|
|
1871
2131
|
return;
|
|
1872
2132
|
}
|
|
1873
2133
|
forgetTorrent(torrent);
|
|
1874
|
-
this.fileUsageByTorrent.delete(torrent);
|
|
1875
2134
|
this.#lastAccess.delete(torrent);
|
|
1876
2135
|
this.#readPositionByTorrent.delete(torrent);
|
|
1877
2136
|
const name = typeof torrent.name === "string" ? torrent.name : "(unknown)";
|
|
@@ -1904,8 +2163,7 @@ export class TorrentPool {
|
|
|
1904
2163
|
}
|
|
1905
2164
|
const infoHash = String(torrent.infoHash ?? "?").slice(0, 8);
|
|
1906
2165
|
const name = typeof torrent.name === "string" ? torrent.name : "(unknown)";
|
|
1907
|
-
const
|
|
1908
|
-
const refcount = usage ? usage.size : 0;
|
|
2166
|
+
const stated = demandFor(torrent).register.size;
|
|
1909
2167
|
const hasData = (() => { try { return torrentDownloadedBytes(torrent); } catch { return -1; } })();
|
|
1910
2168
|
// Capture caller for diagnostics — not for control flow.
|
|
1911
2169
|
const caller = new Error().stack?.split("\n")[2]?.trim() ?? "";
|
|
@@ -1916,10 +2174,9 @@ export class TorrentPool {
|
|
|
1916
2174
|
break;
|
|
1917
2175
|
}
|
|
1918
2176
|
}
|
|
1919
|
-
this.fileUsageByTorrent.delete(torrent);
|
|
1920
2177
|
this.#lastAccess.delete(torrent);
|
|
1921
2178
|
this.#readPositionByTorrent.delete(torrent);
|
|
1922
|
-
logger.warn(`torrent-pool: removing torrent "${name}" [${infoHash}] reason=${reason}
|
|
2179
|
+
logger.warn(`torrent-pool: removing torrent "${name}" [${infoHash}] reason=${reason} stated=${stated} downloaded=${hasData}B caller=${caller}`);
|
|
1923
2180
|
try {
|
|
1924
2181
|
torrent.destroy({ destroyStore: true }, () => {
|
|
1925
2182
|
logger.info(`torrent-pool: removed torrent "${name}" [${infoHash}] reason=${reason} and its store`);
|
|
@@ -2147,16 +2404,31 @@ export class TorrentPool {
|
|
|
2147
2404
|
* @param {number} [options.headBytes=262144] - Leading bytes to fetch (default 256 KB).
|
|
2148
2405
|
* @param {number} [options.tailBytes=2097152] - Trailing bytes to fetch (default 2 MB).
|
|
2149
2406
|
* @param {number} [options.timeoutMs=300000] - Maximum wait time in milliseconds (default 5 min).
|
|
2407
|
+
* @param {boolean} [options.awaited=false] - Whether somebody is waiting for
|
|
2408
|
+
* these bytes right now. The playback plan is: it cannot answer until the
|
|
2409
|
+
* file has said what is in it, and a person is watching a loading screen
|
|
2410
|
+
* meanwhile. The warm-up is NOT, by its whole purpose — it happens while
|
|
2411
|
+
* the viewer is still choosing, so it must not outrank another film that
|
|
2412
|
+
* somebody is watching on this proxy this minute.
|
|
2150
2413
|
* @returns {Promise<void>}
|
|
2151
2414
|
*/
|
|
2152
2415
|
async prefetchFileEdges(
|
|
2153
2416
|
torrent,
|
|
2154
2417
|
fileIndex,
|
|
2155
|
-
{ headBytes = 256 * 1024, tailBytes = 2 * 1024 * 1024, timeoutMs = 300_000 } = {}
|
|
2418
|
+
{ headBytes = 256 * 1024, tailBytes = 2 * 1024 * 1024, timeoutMs = 300_000, awaited = false } = {}
|
|
2156
2419
|
) {
|
|
2157
2420
|
if (!torrent || !Array.isArray(torrent.files)) {
|
|
2158
2421
|
return;
|
|
2159
2422
|
}
|
|
2423
|
+
// THE ENDS OF THIS FILE ARE WANTED, and they go on being wanted after this
|
|
2424
|
+
// read is over: the file's own directory lives there, and a seek asks for
|
|
2425
|
+
// it again. Stated before the read rather than by it, and kept afterwards
|
|
2426
|
+
// at the level of something nobody is waiting for.
|
|
2427
|
+
stateFileEdges(torrent, fileIndex, awaited ? Urgency.NEAR : Urgency.TAIL);
|
|
2428
|
+
// The first thing ever stated about a torrent being opened, and therefore
|
|
2429
|
+
// what keeps it in its swarm through the seconds when nothing else can say
|
|
2430
|
+
// anything about it.
|
|
2431
|
+
this.followTheDemand(torrent);
|
|
2160
2432
|
// Two callers can ask for the same edges at once: the warm-up that starts
|
|
2161
2433
|
// when a torrent is picked, and the playback plan a moment later. Reading
|
|
2162
2434
|
// the same two pieces twice costs nothing in bandwidth — the torrent
|
|
@@ -2173,6 +2445,10 @@ export class TorrentPool {
|
|
|
2173
2445
|
return await prefetch;
|
|
2174
2446
|
} finally {
|
|
2175
2447
|
this.#edgePrefetches.delete(inFlightKey);
|
|
2448
|
+
// Nobody is waiting for them now. Said with `lower`, because this is the
|
|
2449
|
+
// one caller entitled to put them back down: the read it belongs to is
|
|
2450
|
+
// over, and there is no other — a file's edges are fetched once at a time.
|
|
2451
|
+
stateFileEdges(torrent, fileIndex, Urgency.TAIL, { lower: true });
|
|
2176
2452
|
}
|
|
2177
2453
|
}
|
|
2178
2454
|
|