@torrent-tv/proxy 2.83.0 → 2.83.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +35 -0
- package/package.json +1 -1
- package/routes/api/sources/stats/get.js +11 -2
- package/routes/stream/get.js +74 -3
- package/services/delivery-probe.js +248 -43
- package/services/download/SwarmSelection.js +5 -5
- package/services/download/registry.js +20 -0
- package/services/encode/EncodeRun.js +1 -0
- package/services/encode/encode-exit.js +17 -0
- package/services/files/CompletedFiles.js +276 -0
- package/services/files/piece-from-whole-file.js +118 -0
- package/services/output/cut-grid.js +13 -3
- package/services/piece-store/piece-disk-store.js +72 -2
- package/services/piece-store/shared-piece-store.js +274 -27
- package/services/torrent-pool.js +238 -19
- package/services/torrent-worker/client.js +21 -0
- package/services/torrent-worker/protocol.js +9 -1
- package/services/torrent-worker/worker.js +183 -2
- package/test/completed-files.test.js +115 -0
- package/test/cuts-follow-published-grid.test.js +35 -0
- package/test/delivery-probe.test.js +114 -1
- package/test/encode-exit.test.js +18 -0
- package/test/piece-disk-store.test.js +26 -0
- package/test/piece-from-whole-file.test.js +129 -0
- package/test/piece-store-eviction.test.js +28 -15
- package/test/piece-store-never-refuses.test.js +153 -0
- package/test/piece-store-reservations.test.js +16 -3
- package/test/probe-wedge-certainty.test.js +3 -3
- package/test/shared-piece-store.test.js +27 -13
- package/test/stream-route.test.js +41 -0
- package/test/swarm-follows-readers.test.js +126 -0
- package/test/swarm-reach.test.js +5 -0
- package/test/upload-hurry.test.js +27 -0
package/services/torrent-pool.js
CHANGED
|
@@ -17,7 +17,7 @@ import WebTorrent from "webtorrent";
|
|
|
17
17
|
import { logger } from "../utils/logger.js";
|
|
18
18
|
import { SharedPieceStore, findSharedStore } from "./piece-store/shared-piece-store.js";
|
|
19
19
|
import { Urgency, urgencyName } from "./demand/index.js";
|
|
20
|
-
import { demandFor, forgetTorrent, reconcileAll } from "./download/registry.js";
|
|
20
|
+
import { demandFor, forgetTorrent, reconcileAll, hasUnmetDemand } from "./download/registry.js";
|
|
21
21
|
import { isAtAWatchingViewer, isBehindEverybody, isNobodyComingNow } from "./priority/PriorityMap.js";
|
|
22
22
|
import { deriveSourceKey } from "./torrent-source-key.js";
|
|
23
23
|
|
|
@@ -164,10 +164,19 @@ const STALL_REPORT_INTERVAL_MS = 30_000;
|
|
|
164
164
|
* makes every two seconds.
|
|
165
165
|
*
|
|
166
166
|
* @param {import("webtorrent").Torrent} torrent
|
|
167
|
-
* @returns {{ connectedPeers: number,
|
|
167
|
+
* @returns {{ connectedPeers: number, deliveringPeers: number, knownPeers: number | null,
|
|
168
|
+
* queuedPeers: number | null }}
|
|
168
169
|
*/
|
|
169
170
|
export function describeSwarmReach(torrent) {
|
|
170
171
|
const wires = Array.isArray(torrent?.wires) ? torrent.wires.length : 0;
|
|
172
|
+
// HOW MANY OF THEM ARE DOING ANYTHING. Connections accumulated without bound
|
|
173
|
+
// in the field — 249 to 596 over one viewing, 15 862 more queued — and
|
|
174
|
+
// whether that helped or merely cost memory and requests is not answerable
|
|
175
|
+
// from a count of wires. A wire that has delivered a byte is the measured
|
|
176
|
+
// unit; the rest are held open for nothing.
|
|
177
|
+
const delivering = Array.isArray(torrent?.wires)
|
|
178
|
+
? torrent.wires.filter((wire) => Number(wire?.downloaded) > 0).length
|
|
179
|
+
: 0;
|
|
171
180
|
const read = (value) => (typeof value === "number" && Number.isFinite(value) ? value : null);
|
|
172
181
|
let knownPeers = null;
|
|
173
182
|
let queuedPeers = null;
|
|
@@ -179,7 +188,7 @@ export function describeSwarmReach(torrent) {
|
|
|
179
188
|
// destroyed torrent, and the reading is a diagnostic. Nothing here is
|
|
180
189
|
// worth failing a stats poll for.
|
|
181
190
|
}
|
|
182
|
-
return { connectedPeers: wires, knownPeers, queuedPeers };
|
|
191
|
+
return { connectedPeers: wires, deliveringPeers: delivering, knownPeers, queuedPeers };
|
|
183
192
|
}
|
|
184
193
|
|
|
185
194
|
/**
|
|
@@ -345,6 +354,11 @@ export function torrentsForUploadPolicy(torrents, usageByTorrent, now) {
|
|
|
345
354
|
// Recorded so the policy can tell "nothing is arriving and somebody is
|
|
346
355
|
// waiting" from "nothing is arriving because nobody asked".
|
|
347
356
|
torrent.hasActiveReader = hasReader;
|
|
357
|
+
// Whether anybody is still short of bytes of it. Upload is bought with
|
|
358
|
+
// reciprocity and reciprocity is only worth buying while something is
|
|
359
|
+
// missing; a torrent whose declared windows are all present wants nothing
|
|
360
|
+
// from the swarm, and what it gives it gives for nobody.
|
|
361
|
+
torrent.hasUnmetDemand = hasUnmetDemand(torrent);
|
|
348
362
|
chosen.push(torrent);
|
|
349
363
|
}
|
|
350
364
|
}
|
|
@@ -390,7 +404,8 @@ export function decideUploadLimit(activeTorrents, opts = {}) {
|
|
|
390
404
|
// 2026-08-04: four cycles of 512 -> 50 KB/s in three minutes, each
|
|
391
405
|
// reported as `earn unchoke ... down=0KB/s`, all of them raising the
|
|
392
406
|
// upload at moments when no byte was wanted by anyone.
|
|
393
|
-
const starving = notDone && torrent?.hasActiveReader !== false
|
|
407
|
+
const starving = notDone && torrent?.hasActiveReader !== false
|
|
408
|
+
&& torrent?.hasUnmetDemand !== false && downloadSpeed < starvingSpeed;
|
|
394
409
|
if (starving && chokedInterested >= chokedThreshold) {
|
|
395
410
|
const name = typeof torrent?.name === "string" ? torrent.name : "?";
|
|
396
411
|
return {
|
|
@@ -402,6 +417,14 @@ export function decideUploadLimit(activeTorrents, opts = {}) {
|
|
|
402
417
|
}
|
|
403
418
|
}
|
|
404
419
|
|
|
420
|
+
// NOTHING TO BUY. Every torrent here has a reader, and not one of them is
|
|
421
|
+
// short of a byte anybody asked for: the upload would be given for nobody.
|
|
422
|
+
// Field 2026-09-11 — 512 KB/s held for forty-eight minutes on a file that was
|
|
423
|
+
// complete, 596 peers connected and climbing, and every 16 KB served read a
|
|
424
|
+
// 4 MB piece back off the disk.
|
|
425
|
+
if (activeTorrents.every((torrent) => torrent?.hasUnmetDemand === false)) {
|
|
426
|
+
return { bytesPerSec: idleFloor, reason: "nothing anybody asked for is missing — upload buys nothing" };
|
|
427
|
+
}
|
|
405
428
|
return { bytesPerSec: floor, reason: "active readers, not choke-starved" };
|
|
406
429
|
}
|
|
407
430
|
|
|
@@ -783,6 +806,52 @@ export class TorrentPool {
|
|
|
783
806
|
* default is computed from free disk (min(10 GB, half free)). Pass 0 to
|
|
784
807
|
* disable the cap.
|
|
785
808
|
*/
|
|
809
|
+
/**
|
|
810
|
+
* Sources every file of which this proxy holds whole.
|
|
811
|
+
*
|
|
812
|
+
* A torrent added again for one of these has nothing to fetch and nothing to
|
|
813
|
+
* check: what it would verify was written out of pieces this client had
|
|
814
|
+
* already hashed, and the file's size was checked against what the torrent
|
|
815
|
+
* says. So it is added with verification skipped — otherwise a viewer
|
|
816
|
+
* returning to a film waits for a gigabyte and a half to be hashed to learn
|
|
817
|
+
* what we wrote down.
|
|
818
|
+
*
|
|
819
|
+
* @type {Set<string>}
|
|
820
|
+
*/
|
|
821
|
+
#wholeSources = new Set();
|
|
822
|
+
|
|
823
|
+
/**
|
|
824
|
+
* Say that every file of a source is held whole.
|
|
825
|
+
*
|
|
826
|
+
* @param {string} sourceKey
|
|
827
|
+
* @returns {void}
|
|
828
|
+
*/
|
|
829
|
+
addWholeSource(sourceKey) {
|
|
830
|
+
this.#wholeSources.add(String(sourceKey));
|
|
831
|
+
}
|
|
832
|
+
|
|
833
|
+
/**
|
|
834
|
+
* Anything else every store of this pool is built with.
|
|
835
|
+
*
|
|
836
|
+
* Today: where a piece can be had when neither of the store's own tiers has
|
|
837
|
+
* it — the files this proxy has assembled. Given to the pool rather than
|
|
838
|
+
* discovered by the store, which knows nothing about files.
|
|
839
|
+
*
|
|
840
|
+
* @type {object}
|
|
841
|
+
*/
|
|
842
|
+
#storeExtras = {};
|
|
843
|
+
|
|
844
|
+
/**
|
|
845
|
+
* Say where a piece can be had when the store has neither a resident nor a
|
|
846
|
+
* spilled copy.
|
|
847
|
+
*
|
|
848
|
+
* @param {object} extras
|
|
849
|
+
* @returns {void}
|
|
850
|
+
*/
|
|
851
|
+
buildStoresWith(extras) {
|
|
852
|
+
this.#storeExtras = extras && typeof extras === "object" ? extras : {};
|
|
853
|
+
}
|
|
854
|
+
|
|
786
855
|
constructor({ maxDiskBytes, memoryBytes, dhtBootstrap } = {}) {
|
|
787
856
|
this.#memoryBytes = Number.isFinite(memoryBytes) && memoryBytes > 0 ? memoryBytes : undefined;
|
|
788
857
|
|
|
@@ -851,8 +920,8 @@ export class TorrentPool {
|
|
|
851
920
|
? maxDiskBytes
|
|
852
921
|
: computeDefaultDiskCap(os.tmpdir());
|
|
853
922
|
if (this.#maxDiskBytes > 0) {
|
|
854
|
-
const
|
|
855
|
-
logger.info(`torrent-pool: disk cap ${
|
|
923
|
+
const gigabytes = (this.#maxDiskBytes / (1024 * 1024 * 1024)).toFixed(1);
|
|
924
|
+
logger.info(`torrent-pool: disk cap ${gigabytes} GB (LRU eviction of idle torrents above it)`);
|
|
856
925
|
this.#diskSweepTimer = setInterval(() => this.#enforceDiskCap(), DISK_CAP_SWEEP_INTERVAL_MS);
|
|
857
926
|
this.#diskSweepTimer.unref?.();
|
|
858
927
|
}
|
|
@@ -869,6 +938,67 @@ export class TorrentPool {
|
|
|
869
938
|
this.#uploadAdjustTimer.unref?.();
|
|
870
939
|
}
|
|
871
940
|
|
|
941
|
+
/**
|
|
942
|
+
* Leave the swarm of a torrent nobody is reading, and rejoin it when
|
|
943
|
+
* somebody is.
|
|
944
|
+
*
|
|
945
|
+
* A PROXY THAT NEEDS NOTHING FROM A SWARM SHOULD NOT BE IN THAT SWARM. While
|
|
946
|
+
* a file is being watched every connection is worth keeping — the one that
|
|
947
|
+
* has delivered nothing yet may deliver next — so this is not a limit on
|
|
948
|
+
* connections and never fires during playback. It fires when NOTHING is
|
|
949
|
+
* stated about this torrent at all: no window from any reader, no file held.
|
|
950
|
+
*
|
|
951
|
+
* What that state cost until now: WebTorrent enforces `maxConns` only on
|
|
952
|
+
* peers it dials (`_drain`), while `_addIncomingPeer` checks that the torrent
|
|
953
|
+
* is neither destroyed nor paused and registers the peer. A proxy with its
|
|
954
|
+
* port mapped is reachable, so connections arrive and are never turned away —
|
|
955
|
+
* field 2026-09-11, 249 connected at the start of one viewing and 596 at the
|
|
956
|
+
* end, 15 862 more queued, on a file complete for three quarters of an hour,
|
|
957
|
+
* each of them served by reading a 4 MB piece off the disk for every 16 KB
|
|
958
|
+
* sent.
|
|
959
|
+
*
|
|
960
|
+
* `paused` is the library's own word for this and the path it already checks,
|
|
961
|
+
* so nothing here fights it. The pause alone does not close what is already
|
|
962
|
+
* open, so the peers are let go by hand; the data stays exactly where it is,
|
|
963
|
+
* and the next reader resumes the torrent rather than fetching it again.
|
|
964
|
+
*
|
|
965
|
+
* @param {import("webtorrent").Torrent} torrent
|
|
966
|
+
* @returns {void}
|
|
967
|
+
*/
|
|
968
|
+
followTheReaders(torrent) {
|
|
969
|
+
if (!torrent || torrent.destroyed) {
|
|
970
|
+
return;
|
|
971
|
+
}
|
|
972
|
+
const usage = this.fileUsageByTorrent.get(torrent);
|
|
973
|
+
const isRead = Boolean(usage && usage.size > 0);
|
|
974
|
+
const isWanted = isRead || demandFor(torrent).register.windows().length > 0;
|
|
975
|
+
if (isWanted && torrent.paused === true) {
|
|
976
|
+
torrent.resume();
|
|
977
|
+
logger.info(
|
|
978
|
+
`torrent-pool: [${String(torrent.infoHash ?? "?").slice(0, 8)}] rejoined the swarm — somebody is reading it again`
|
|
979
|
+
);
|
|
980
|
+
return;
|
|
981
|
+
}
|
|
982
|
+
if (isWanted || torrent.paused === true) {
|
|
983
|
+
return;
|
|
984
|
+
}
|
|
985
|
+
torrent.pause();
|
|
986
|
+
const open = Array.isArray(torrent.wires) ? torrent.wires.length : 0;
|
|
987
|
+
let closed = 0;
|
|
988
|
+
for (const peer of [...(torrent._peers?.values?.() ?? [])]) {
|
|
989
|
+
try {
|
|
990
|
+
peer.destroy();
|
|
991
|
+
closed += 1;
|
|
992
|
+
} catch {
|
|
993
|
+
// A peer already going: nothing to do, and nothing worth failing for.
|
|
994
|
+
}
|
|
995
|
+
}
|
|
996
|
+
logger.info(
|
|
997
|
+
`torrent-pool: [${String(torrent.infoHash ?? "?").slice(0, 8)}] left the swarm — nobody is reading it, ` +
|
|
998
|
+
`${open} connection(s) open, ${closed} let go; the data stays and the next reader rejoins`
|
|
999
|
+
);
|
|
1000
|
+
}
|
|
1001
|
+
|
|
872
1002
|
/**
|
|
873
1003
|
* Re-evaluate and apply the client-wide upload limit from current swarm state
|
|
874
1004
|
* (see {@link decideUploadLimit}). Runs on a timer; only calls into WebTorrent
|
|
@@ -1201,6 +1331,9 @@ export class TorrentPool {
|
|
|
1201
1331
|
this.#stateBackgroundFill(torrent);
|
|
1202
1332
|
}
|
|
1203
1333
|
this.#reportStalledDownloads();
|
|
1334
|
+
for (const torrent of this.torrents.values()) {
|
|
1335
|
+
this.followTheReaders(torrent);
|
|
1336
|
+
}
|
|
1204
1337
|
const { bytesPerSec, reason } = decideUploadLimit(active);
|
|
1205
1338
|
if (bytesPerSec === this.#uploadLimit) {
|
|
1206
1339
|
return;
|
|
@@ -1241,11 +1374,14 @@ export class TorrentPool {
|
|
|
1241
1374
|
}
|
|
1242
1375
|
// Candidates: pooled torrents with zero active readers, LRU first.
|
|
1243
1376
|
const candidates = [...this.torrents.values()]
|
|
1244
|
-
.filter((
|
|
1245
|
-
const usage = this.fileUsageByTorrent.get(
|
|
1377
|
+
.filter((torrent) => {
|
|
1378
|
+
const usage = this.fileUsageByTorrent.get(torrent);
|
|
1246
1379
|
return !usage || usage.size === 0;
|
|
1247
1380
|
})
|
|
1248
|
-
.sort(
|
|
1381
|
+
.sort(
|
|
1382
|
+
(earlier, later) =>
|
|
1383
|
+
(this.#lastAccess.get(earlier) ?? 0) - (this.#lastAccess.get(later) ?? 0)
|
|
1384
|
+
);
|
|
1249
1385
|
|
|
1250
1386
|
for (const torrent of candidates) {
|
|
1251
1387
|
if (used <= this.#maxDiskBytes) {
|
|
@@ -1253,9 +1389,9 @@ export class TorrentPool {
|
|
|
1253
1389
|
}
|
|
1254
1390
|
const freed = Math.max(0, torrentDownloadedBytes(torrent));
|
|
1255
1391
|
const name = typeof torrent?.name === "string" ? torrent.name : "(unknown)";
|
|
1256
|
-
const
|
|
1392
|
+
const gigabytes = (this.#maxDiskBytes / (1024 * 1024 * 1024)).toFixed(1);
|
|
1257
1393
|
logger.info(
|
|
1258
|
-
`torrent-pool: disk cap ${
|
|
1394
|
+
`torrent-pool: disk cap ${gigabytes} GB exceeded — evicting idle torrent "${name}" ` +
|
|
1259
1395
|
`(~${(freed / (1024 * 1024)).toFixed(0)} MB)`
|
|
1260
1396
|
);
|
|
1261
1397
|
this.#cancelIdleRemoval(torrent);
|
|
@@ -1330,6 +1466,21 @@ export class TorrentPool {
|
|
|
1330
1466
|
logger.warn(`torrent-pool: [${label()}] warning: ${formatWarning(warning)}`);
|
|
1331
1467
|
});
|
|
1332
1468
|
|
|
1469
|
+
// A TORRENT THAT DIED WITHOUT US ASKING LEAVES ITS RECORD BEHIND, and the
|
|
1470
|
+
// record goes on answering. Field 2026-09-11: the store refused a block,
|
|
1471
|
+
// the client destroyed the torrent, and for the rest of the process every
|
|
1472
|
+
// read answered `File 1 not found in torrent:d4022ff4…` while `/stats`
|
|
1473
|
+
// reported `peers=0 connected of 1186 known` — so the browser's own remedy,
|
|
1474
|
+
// adding the source again, returned the corpse and the viewer could not
|
|
1475
|
+
// open anything until the addon was restarted.
|
|
1476
|
+
//
|
|
1477
|
+
// Forgetting the record is the whole fix: the next request builds the
|
|
1478
|
+
// torrent again, from the same magnet, against the same data on disk.
|
|
1479
|
+
torrent.on("error", (error) => {
|
|
1480
|
+
logger.error(`torrent-pool: [${label()}] died: ${formatWarning(error)}`);
|
|
1481
|
+
this.#forgetDeadTorrent(torrent);
|
|
1482
|
+
});
|
|
1483
|
+
|
|
1333
1484
|
// Everything below needs the torrent to have been PARSED, and `add` returns
|
|
1334
1485
|
// before that: `announce`, `files` and `private` are all still empty, and
|
|
1335
1486
|
// `discovery` — which owns the tracker client — is not created until
|
|
@@ -1437,7 +1588,7 @@ export class TorrentPool {
|
|
|
1437
1588
|
const message = error instanceof Error ? error.message : String(error);
|
|
1438
1589
|
const dupMatch = /duplicate torrent ([0-9a-f]{40})/i.exec(message);
|
|
1439
1590
|
if (dupMatch) {
|
|
1440
|
-
const existing = this.client.torrents.find((
|
|
1591
|
+
const existing = this.client.torrents.find((candidate) => candidate?.infoHash === dupMatch[1]);
|
|
1441
1592
|
// A torrent already here but WITHOUT metadata is not an answer to a
|
|
1442
1593
|
// request that carries metadata. A magnet whose swarm never answered
|
|
1443
1594
|
// leaves exactly that: an entry with the right infohash, no file
|
|
@@ -1501,7 +1652,7 @@ export class TorrentPool {
|
|
|
1501
1652
|
const addedReplacement = this.client.add(torrentId, {
|
|
1502
1653
|
store: SharedPieceStore,
|
|
1503
1654
|
storeCacheSlots: 0,
|
|
1504
|
-
storeOpts: { memoryBytes: this.#memoryBytes },
|
|
1655
|
+
storeOpts: { memoryBytes: this.#memoryBytes, ...this.#storeExtras },
|
|
1505
1656
|
deselect: true
|
|
1506
1657
|
}, (replacement) => {
|
|
1507
1658
|
this.torrents.set(key, replacement);
|
|
@@ -1534,14 +1685,19 @@ export class TorrentPool {
|
|
|
1534
1685
|
const added = this.client.add(torrentId, {
|
|
1535
1686
|
store: SharedPieceStore,
|
|
1536
1687
|
storeCacheSlots: 0,
|
|
1537
|
-
storeOpts: { memoryBytes: this.#memoryBytes },
|
|
1688
|
+
storeOpts: { memoryBytes: this.#memoryBytes, ...this.#storeExtras },
|
|
1538
1689
|
// Nothing is fetched until somebody says they want it. WebTorrent's own
|
|
1539
1690
|
// default is `this.select(0, this.pieces.length - 1)` — the whole
|
|
1540
1691
|
// torrent — and this proxy used to undo that afterwards by deselecting
|
|
1541
1692
|
// the files nobody had opened. On a season pack that meant every
|
|
1542
1693
|
// episode was being fetched for as long as the viewer took to choose
|
|
1543
1694
|
// one. The download set is built up from stated needs instead.
|
|
1544
|
-
deselect: true
|
|
1695
|
+
deselect: true,
|
|
1696
|
+
// Nothing to check when every file of this source is already here
|
|
1697
|
+
// whole: what would be verified was written out of pieces this client
|
|
1698
|
+
// had hashed, and the file's size was checked against what the torrent
|
|
1699
|
+
// says. See `#wholeSources`.
|
|
1700
|
+
skipVerify: this.#wholeSources.has(key)
|
|
1545
1701
|
}, (readyTorrent) => {
|
|
1546
1702
|
this.client.off("error", onError);
|
|
1547
1703
|
this.torrents.set(key, readyTorrent);
|
|
@@ -1592,6 +1748,15 @@ export class TorrentPool {
|
|
|
1592
1748
|
// it recently accessed so LRU eviction keeps it.
|
|
1593
1749
|
this.#cancelIdleRemoval(torrent);
|
|
1594
1750
|
this.#lastAccess.set(torrent, Date.now());
|
|
1751
|
+
// And rejoin its swarm HERE rather than at the next five-second pass: a
|
|
1752
|
+
// reader that has just arrived is about to ask for bytes, and a torrent
|
|
1753
|
+
// still paused answers by fetching nothing at all.
|
|
1754
|
+
if (torrent.paused === true) {
|
|
1755
|
+
torrent.resume();
|
|
1756
|
+
logger.info(
|
|
1757
|
+
`torrent-pool: [${String(torrent.infoHash ?? "?").slice(0, 8)}] rejoined the swarm — a reader arrived`
|
|
1758
|
+
);
|
|
1759
|
+
}
|
|
1595
1760
|
usage.set(fileIndex, (usage.get(fileIndex) ?? 0) + 1);
|
|
1596
1761
|
|
|
1597
1762
|
let released = false;
|
|
@@ -1628,18 +1793,18 @@ export class TorrentPool {
|
|
|
1628
1793
|
}
|
|
1629
1794
|
this.#cancelIdleRemoval(torrent);
|
|
1630
1795
|
const name = typeof torrent.name === "string" ? torrent.name : "(unknown)";
|
|
1631
|
-
const
|
|
1632
|
-
logger.info(`torrent-pool: scheduling idle removal for "${name}" [${
|
|
1796
|
+
const infoHashShort = String(torrent.infoHash ?? "?").slice(0, 8);
|
|
1797
|
+
logger.info(`torrent-pool: scheduling idle removal for "${name}" [${infoHashShort}] in ${TORRENT_IDLE_TTL_MS / 1000}s`);
|
|
1633
1798
|
const timer = setTimeout(() => {
|
|
1634
1799
|
this.#idleTimers.delete(torrent);
|
|
1635
1800
|
// Re-check: a new acquire since scheduling would have cancelled this
|
|
1636
1801
|
// timer, but guard anyway against a race.
|
|
1637
1802
|
const usage = this.fileUsageByTorrent.get(torrent);
|
|
1638
1803
|
if (usage && usage.size > 0) {
|
|
1639
|
-
logger.info(`torrent-pool: idle timer fired for "${name}" [${
|
|
1804
|
+
logger.info(`torrent-pool: idle timer fired for "${name}" [${infoHashShort}] but refcount ${usage.size} >0 — keep`);
|
|
1640
1805
|
return;
|
|
1641
1806
|
}
|
|
1642
|
-
logger.warn(`torrent-pool: idle TTL fired for "${name}" [${
|
|
1807
|
+
logger.warn(`torrent-pool: idle TTL fired for "${name}" [${infoHashShort}] — removing torrent (reason=idle-ttl caller=scheduleIdleRemoval)`);
|
|
1643
1808
|
this.#removeTorrent(torrent, "idle-ttl");
|
|
1644
1809
|
}, TORRENT_IDLE_TTL_MS);
|
|
1645
1810
|
timer.unref?.();
|
|
@@ -1668,6 +1833,59 @@ export class TorrentPool {
|
|
|
1668
1833
|
* @param {string} [reason="unknown"] - Why removal was requested (idle-ttl, disk-cap, evict, api).
|
|
1669
1834
|
* @returns {void}
|
|
1670
1835
|
*/
|
|
1836
|
+
/**
|
|
1837
|
+
* Forget a torrent that died on its own, keeping its bytes.
|
|
1838
|
+
*
|
|
1839
|
+
* NOT `#removeTorrent`: that one destroys the store as well, and the data on
|
|
1840
|
+
* disk is the expensive thing here — re-downloading a film costs the swarm
|
|
1841
|
+
* and the viewer, while re-adding a torrent costs a tracker announce. What
|
|
1842
|
+
* has to go is only the RECORD, which otherwise answers every later request
|
|
1843
|
+
* with a corpse.
|
|
1844
|
+
*
|
|
1845
|
+
* @param {import("webtorrent").Torrent} torrent
|
|
1846
|
+
* @returns {void}
|
|
1847
|
+
*/
|
|
1848
|
+
#forgetDeadTorrent(torrent) {
|
|
1849
|
+
if (!torrent) {
|
|
1850
|
+
return;
|
|
1851
|
+
}
|
|
1852
|
+
let forgotten = false;
|
|
1853
|
+
for (const [key, value] of this.torrents) {
|
|
1854
|
+
if (value === torrent) {
|
|
1855
|
+
this.torrents.delete(key);
|
|
1856
|
+
forgotten = true;
|
|
1857
|
+
break;
|
|
1858
|
+
}
|
|
1859
|
+
}
|
|
1860
|
+
if (!forgotten) {
|
|
1861
|
+
return;
|
|
1862
|
+
}
|
|
1863
|
+
forgetTorrent(torrent);
|
|
1864
|
+
this.fileUsageByTorrent.delete(torrent);
|
|
1865
|
+
this.#lastAccess.delete(torrent);
|
|
1866
|
+
this.#readPositionByTorrent.delete(torrent);
|
|
1867
|
+
const name = typeof torrent.name === "string" ? torrent.name : "(unknown)";
|
|
1868
|
+
logger.warn(
|
|
1869
|
+
`torrent-pool: forgot "${name}" [${String(torrent.infoHash ?? "?").slice(0, 8)}] — ` +
|
|
1870
|
+
"it died without being asked to; its data stays on disk and the next request adds it again"
|
|
1871
|
+
);
|
|
1872
|
+
}
|
|
1873
|
+
|
|
1874
|
+
/**
|
|
1875
|
+
* Remove a torrent and its store, by name of the reason.
|
|
1876
|
+
*
|
|
1877
|
+
* The public form of what the idle timer and the disk cap already do. Taking
|
|
1878
|
+
* the store with it is the point where a torrent downloaded whole is
|
|
1879
|
+
* concerned: its pieces are a duplicate of the files kept elsewhere.
|
|
1880
|
+
*
|
|
1881
|
+
* @param {import("webtorrent").Torrent} torrent
|
|
1882
|
+
* @param {string} reason
|
|
1883
|
+
* @returns {void}
|
|
1884
|
+
*/
|
|
1885
|
+
remove(torrent, reason) {
|
|
1886
|
+
this.#removeTorrent(torrent, reason);
|
|
1887
|
+
}
|
|
1888
|
+
|
|
1671
1889
|
#removeTorrent(torrent, reason = "unknown") {
|
|
1672
1890
|
// Everything anybody stated for this torrent goes with it.
|
|
1673
1891
|
forgetTorrent(torrent);
|
|
@@ -1755,6 +1973,7 @@ export class TorrentPool {
|
|
|
1755
1973
|
downloadSpeed,
|
|
1756
1974
|
uploadSpeed,
|
|
1757
1975
|
connectedPeers: reach.connectedPeers,
|
|
1976
|
+
deliveringPeers: reach.deliveringPeers,
|
|
1758
1977
|
knownPeers: reach.knownPeers,
|
|
1759
1978
|
queuedPeers: reach.queuedPeers,
|
|
1760
1979
|
trackerSeeders: announce.seeders,
|
|
@@ -94,6 +94,18 @@ export class TorrentWorkerClient {
|
|
|
94
94
|
/** Reads consuming fragments in place, keyed by request id. */
|
|
95
95
|
#fragmentReaders = new Map();
|
|
96
96
|
|
|
97
|
+
/**
|
|
98
|
+
* Files downloaded whole, by `${infoHash}/${fileIndex}`.
|
|
99
|
+
*
|
|
100
|
+
* Held on THIS thread because it is this thread that serves them: a file that
|
|
101
|
+
* is whole is read with an ordinary file read, and asking the torrent thread
|
|
102
|
+
* for anything about it would put back the very hop this removes. The torrent
|
|
103
|
+
* thread writes the files and says so; this is what it says.
|
|
104
|
+
*
|
|
105
|
+
* @type {Map<string, { path: string, length: number, name: string }>}
|
|
106
|
+
*/
|
|
107
|
+
wholeFiles = new Map();
|
|
108
|
+
|
|
97
109
|
/** @type {(event: { sourceKey: string, fileIndex: number, trackIndex: number, cues: object[], language: string }) => void} */
|
|
98
110
|
#onSubtitleCues;
|
|
99
111
|
|
|
@@ -208,6 +220,15 @@ export class TorrentWorkerClient {
|
|
|
208
220
|
case Event.LOG:
|
|
209
221
|
logger.info(`torrent-worker: ${message.message}`);
|
|
210
222
|
break;
|
|
223
|
+
case Event.FILE_COMPLETE:
|
|
224
|
+
// Recorded here so the stream route can answer from the file without
|
|
225
|
+
// going near the torrent thread at all.
|
|
226
|
+
this.wholeFiles.set(`${message.infoHash}/${message.fileIndex}`, {
|
|
227
|
+
path: message.path,
|
|
228
|
+
length: message.length,
|
|
229
|
+
name: message.name
|
|
230
|
+
});
|
|
231
|
+
break;
|
|
211
232
|
case Event.SUBTITLE_CUES_READY:
|
|
212
233
|
this.#onSubtitleCues({
|
|
213
234
|
sourceKey: message.sourceKey,
|
|
@@ -164,7 +164,15 @@ export const Event = {
|
|
|
164
164
|
* {@link Command.SUBTITLE_CUES} call. Lets the main thread PUSH them to
|
|
165
165
|
* whichever browser is watching instead of waiting to be asked.
|
|
166
166
|
*/
|
|
167
|
-
SUBTITLE_CUES_READY: "subtitle-cues-ready"
|
|
167
|
+
SUBTITLE_CUES_READY: "subtitle-cues-ready",
|
|
168
|
+
/**
|
|
169
|
+
* A file has been downloaded whole and written out as a file.
|
|
170
|
+
*
|
|
171
|
+
* The main thread serves it from disk after this, without asking this thread
|
|
172
|
+
* for anything: an ordinary read of an ordinary file, with no piece store
|
|
173
|
+
* between them and nothing that can refuse it for want of memory.
|
|
174
|
+
*/
|
|
175
|
+
FILE_COMPLETE: "file-complete"
|
|
168
176
|
};
|
|
169
177
|
|
|
170
178
|
/**
|
|
@@ -36,6 +36,8 @@ import {
|
|
|
36
36
|
warmResumePosition
|
|
37
37
|
} from "./container-tracks.js";
|
|
38
38
|
import { fillFileInBackground } from "./background-fill.js";
|
|
39
|
+
import { CompletedFiles, completedFilesRoot } from "../files/CompletedFiles.js";
|
|
40
|
+
import { pieceFromWholeFiles, pieceIsInWholeFiles } from "../files/piece-from-whole-file.js";
|
|
39
41
|
import { Command, Event } from "./protocol.js";
|
|
40
42
|
import { startMemoryReport, WORKER_MEMORY_SAMPLE_MS } from "../memory-report.js";
|
|
41
43
|
import { forwardLogsTo, logger } from "../../utils/logger.js";
|
|
@@ -65,7 +67,7 @@ forwardLogsTo((_level, message) => {
|
|
|
65
67
|
// the hook above had a chance to register. Verified the hard way: with a static
|
|
66
68
|
// import the process still aborted, and the stack named the genuine polyfill.
|
|
67
69
|
const { TorrentPool, resolveDhtBootstrap } = await import("../torrent-pool.js");
|
|
68
|
-
const { collectStoreStats, machineReserveBytes, pieceBufferCollection, reviseSpillBudgets, reviseStoreBudgets } =
|
|
70
|
+
const { collectStoreStats, findSharedStore, machineReserveBytes, pieceBufferCollection, reviseSpillBudgets, reviseStoreBudgets } =
|
|
69
71
|
await import("../piece-store/shared-piece-store.js");
|
|
70
72
|
|
|
71
73
|
// Resolved before the client exists, because the client builds its DHT in its
|
|
@@ -733,7 +735,9 @@ setInterval(() => {
|
|
|
733
735
|
`on-disk=${Math.round((stats.spilledBytes || 0) / 1048576)}MB ` +
|
|
734
736
|
`pinned=${stats.pinned} spilled=${stats.spilled} reads=${reads} (${fromMemoryShare}% from memory) ` +
|
|
735
737
|
`spills=${stats.spills} revivals=${stats.revivals}` +
|
|
736
|
-
(stats.blockedByPins > 0 ? `
|
|
738
|
+
(stats.blockedByPins > 0 ? ` no-block-for=${stats.blockedByPins}` : "") +
|
|
739
|
+
(stats.admittedWithoutSlot > 0 ? ` to-disk-for-want-of-memory=${stats.admittedWithoutSlot}` : "") +
|
|
740
|
+
(stats.stillMs > 1000 ? ` nothing-moved-for=${Math.round(stats.stillMs / 1000)}s` : "") +
|
|
737
741
|
(stats.evictedOnRevise > 0 ? ` evictedOnRevise=${stats.evictedOnRevise}` : "") +
|
|
738
742
|
(stats.spillFailures > 0 ? ` spill-failures=${stats.spillFailures}` : "") +
|
|
739
743
|
(stats.outstanding > 0 ? ` outstanding=${stats.outstanding}` : "")
|
|
@@ -922,6 +926,183 @@ setInterval(() => {
|
|
|
922
926
|
}
|
|
923
927
|
}, SUBTITLE_WARMUP_INTERVAL_MS).unref();
|
|
924
928
|
|
|
929
|
+
/**
|
|
930
|
+
* Files this proxy has downloaded whole. One directory, two readers of it: this
|
|
931
|
+
* thread writes them, the main thread serves them without asking anybody.
|
|
932
|
+
*/
|
|
933
|
+
const completedFiles = new CompletedFiles({ root: completedFilesRoot() });
|
|
934
|
+
|
|
935
|
+
/**
|
|
936
|
+
* Which torrent a set of files belongs to.
|
|
937
|
+
*
|
|
938
|
+
* The piece store hands its own files back without knowing what they are; this
|
|
939
|
+
* thread does know, and a file carries its torrent.
|
|
940
|
+
*
|
|
941
|
+
* @param {object[]} files
|
|
942
|
+
* @returns {string}
|
|
943
|
+
*/
|
|
944
|
+
const infoHashOf = (files) => String(files?.[0]?._torrent?.infoHash ?? "").toLowerCase();
|
|
945
|
+
|
|
946
|
+
// WHERE A PIECE COMES FROM WHEN NEITHER TIER HAS IT. Handed to every store this
|
|
947
|
+
// pool builds, so a film already assembled into a file is read from that file:
|
|
948
|
+
// which is what lets its spilled copy be dropped as the duplicate it has become,
|
|
949
|
+
// and what lets a torrent be destroyed and added again without fetching a byte.
|
|
950
|
+
pool.buildStoresWith({
|
|
951
|
+
readPieceElsewhere: ({ index, pieceLength, length, files }) =>
|
|
952
|
+
pieceFromWholeFiles({
|
|
953
|
+
index,
|
|
954
|
+
pieceLength,
|
|
955
|
+
length,
|
|
956
|
+
files,
|
|
957
|
+
wholeFileAt: (fileIndex) => completedFiles.find(infoHashOf(files), fileIndex)
|
|
958
|
+
}),
|
|
959
|
+
isPieceElsewhere: ({ index, pieceLength, length, files }) =>
|
|
960
|
+
pieceIsInWholeFiles({
|
|
961
|
+
index,
|
|
962
|
+
pieceLength,
|
|
963
|
+
length,
|
|
964
|
+
files,
|
|
965
|
+
wholeFileAt: (fileIndex) => completedFiles.find(infoHashOf(files), fileIndex)
|
|
966
|
+
})
|
|
967
|
+
});
|
|
968
|
+
void completedFiles.adopt(() => null).then((adopted) => {
|
|
969
|
+
if (adopted > 0) {
|
|
970
|
+
logger.info(
|
|
971
|
+
`whole files: took up ${adopted} file(s) a previous life left in ${completedFiles.root}`
|
|
972
|
+
);
|
|
973
|
+
}
|
|
974
|
+
});
|
|
975
|
+
|
|
976
|
+
/**
|
|
977
|
+
* How often whole files are looked for.
|
|
978
|
+
*
|
|
979
|
+
* The work itself is one pass over the files of each torrent asking a boolean
|
|
980
|
+
* the library already keeps; writing one out happens at most once per file,
|
|
981
|
+
* ever.
|
|
982
|
+
*/
|
|
983
|
+
const WHOLE_FILE_SWEEP_MS = 10_000;
|
|
984
|
+
|
|
985
|
+
/** Files being written out right now, so a sweep does not start a second one. */
|
|
986
|
+
const beingKept = new Set();
|
|
987
|
+
|
|
988
|
+
/**
|
|
989
|
+
* Keep every file that is now whole, and let go of a torrent that has nothing
|
|
990
|
+
* left to fetch.
|
|
991
|
+
*
|
|
992
|
+
* The instruction this serves, 2026-09-11: as soon as a torrent is fully
|
|
993
|
+
* downloaded, downloading stops, the torrent is deleted, and what was
|
|
994
|
+
* downloaded stays for as long as it is wanted.
|
|
995
|
+
*
|
|
996
|
+
* `file.done` is the library's own answer to "is every piece of this file
|
|
997
|
+
* here", and `torrent.done` to "is that true of every file". The second is a
|
|
998
|
+
* strong condition and will not fire for a season pack of which one episode is
|
|
999
|
+
* watched — nothing fetches the other four — and that is right: the instruction
|
|
1000
|
+
* is about a torrent downloaded WHOLE.
|
|
1001
|
+
*
|
|
1002
|
+
* @returns {Promise<void>}
|
|
1003
|
+
*/
|
|
1004
|
+
async function keepWholeFiles() {
|
|
1005
|
+
for (const [sourceKey, torrent] of [...pool.torrents]) {
|
|
1006
|
+
// Lower case, because that is what the source key carries and the main
|
|
1007
|
+
// thread looks these up by the key alone.
|
|
1008
|
+
const infoHash = String(torrent?.infoHash ?? "").toLowerCase();
|
|
1009
|
+
if (!infoHash || !Array.isArray(torrent.files)) {
|
|
1010
|
+
continue;
|
|
1011
|
+
}
|
|
1012
|
+
const usage = pool.fileUsageByTorrent?.get?.(torrent);
|
|
1013
|
+
for (const [fileIndex, file] of torrent.files.entries()) {
|
|
1014
|
+
const key = `${infoHash}/${fileIndex}`;
|
|
1015
|
+
if (file?.done !== true || completedFiles.find(infoHash, fileIndex) || beingKept.has(key)) {
|
|
1016
|
+
continue;
|
|
1017
|
+
}
|
|
1018
|
+
// NOT WHILE SOMEBODY IS READING IT. Writing a film out is a read of the
|
|
1019
|
+
// whole of it and a write of the whole of it — a gigabyte and a half on
|
|
1020
|
+
// the file this was measured against — and doing that beside a viewer
|
|
1021
|
+
// takes the disk and the piece store from them for nothing they asked
|
|
1022
|
+
// for. The file is complete; it will still be complete when they leave.
|
|
1023
|
+
if (usage?.has?.(fileIndex)) {
|
|
1024
|
+
continue;
|
|
1025
|
+
}
|
|
1026
|
+
beingKept.add(key);
|
|
1027
|
+
try {
|
|
1028
|
+
const kept = await completedFiles.keep({
|
|
1029
|
+
infoHash,
|
|
1030
|
+
fileIndex,
|
|
1031
|
+
length: file.length,
|
|
1032
|
+
name: file.name,
|
|
1033
|
+
open: () => file.createReadStream()
|
|
1034
|
+
});
|
|
1035
|
+
if (kept) {
|
|
1036
|
+
logger.info(
|
|
1037
|
+
`whole files: kept "${file.name}" (${Math.round(kept.length / 1048576)}MB) — ` +
|
|
1038
|
+
"it is a file now, and reading it needs no torrent"
|
|
1039
|
+
);
|
|
1040
|
+
// The pieces it was built from are a second copy of the same bytes.
|
|
1041
|
+
// Nothing is lost by dropping them: a read that wants one of them is
|
|
1042
|
+
// answered from the file.
|
|
1043
|
+
const store = findSharedStore(torrent);
|
|
1044
|
+
const dropped = store?.dropDuplicatesHeldElsewhere?.() ?? 0;
|
|
1045
|
+
if (dropped > 0) {
|
|
1046
|
+
logger.info(
|
|
1047
|
+
`whole files: dropped ${dropped} spilled piece(s) of "${file.name}" — ` +
|
|
1048
|
+
"the film was on this disk twice and is not any more"
|
|
1049
|
+
);
|
|
1050
|
+
}
|
|
1051
|
+
parentPort.postMessage({
|
|
1052
|
+
type: Event.FILE_COMPLETE,
|
|
1053
|
+
infoHash,
|
|
1054
|
+
fileIndex,
|
|
1055
|
+
path: kept.path,
|
|
1056
|
+
length: kept.length,
|
|
1057
|
+
name: kept.name
|
|
1058
|
+
});
|
|
1059
|
+
}
|
|
1060
|
+
} catch (error) {
|
|
1061
|
+
logger.warn(`whole files: could not keep "${file?.name}": ${error?.message ?? error}`);
|
|
1062
|
+
} finally {
|
|
1063
|
+
beingKept.delete(key);
|
|
1064
|
+
}
|
|
1065
|
+
}
|
|
1066
|
+
// EVERYTHING, WITHOUT EXCEPTION, AND EVERY BYTE OF IT A FILE ON DISK. The
|
|
1067
|
+
// torrent has no job left: there is nothing to fetch, and this proxy does
|
|
1068
|
+
// not seed what nobody is watching.
|
|
1069
|
+
//
|
|
1070
|
+
// Safe to destroy only because a piece can now be read out of those files:
|
|
1071
|
+
// a path that asks for this source again adds the torrent back, and what it
|
|
1072
|
+
// verifies it reads from the files rather than from the swarm. Not while
|
|
1073
|
+
// anybody is reading it, for the same reason the writing above waits.
|
|
1074
|
+
const isWhole =
|
|
1075
|
+
torrent.done === true &&
|
|
1076
|
+
torrent.files.every((unused, fileIndex) => completedFiles.find(infoHash, fileIndex) !== null);
|
|
1077
|
+
if (isWhole && !(usage?.size > 0)) {
|
|
1078
|
+
logger.info(
|
|
1079
|
+
`whole files: "${torrent.name}" is downloaded whole and saved — removing the torrent, keeping the files`
|
|
1080
|
+
);
|
|
1081
|
+
// THE RECIPE STAYS, and so does the entry that leads to it. Everything
|
|
1082
|
+
// that asks this thread about a source — the track table, the media
|
|
1083
|
+
// info, the keyframe table, the stats the browser polls — comes through
|
|
1084
|
+
// `requireTorrent`, which adds a torrent back when the one it holds is
|
|
1085
|
+
// no longer usable. Deleting the entry here would turn a viewer
|
|
1086
|
+
// returning to this film into `Unknown source`, which is a worse failure
|
|
1087
|
+
// than the one this removal is for.
|
|
1088
|
+
//
|
|
1089
|
+
// What the torrent finds when it comes back is the whole files: its
|
|
1090
|
+
// store reads pieces from them, so it fetches nothing. It is added with
|
|
1091
|
+
// verification skipped, and that is not a shortcut — the file was
|
|
1092
|
+
// written out of pieces this client had already hashed, and its size was
|
|
1093
|
+
// checked against what the torrent says. Re-hashing a gigabyte and a
|
|
1094
|
+
// half to learn what we wrote down is a minute of a viewer's time for
|
|
1095
|
+
// nothing.
|
|
1096
|
+
pool.remove(torrent, "downloaded-whole");
|
|
1097
|
+
pool.addWholeSource(sourceKey);
|
|
1098
|
+
}
|
|
1099
|
+
}
|
|
1100
|
+
}
|
|
1101
|
+
|
|
1102
|
+
setInterval(() => {
|
|
1103
|
+
void keepWholeFiles();
|
|
1104
|
+
}, WHOLE_FILE_SWEEP_MS).unref();
|
|
1105
|
+
|
|
925
1106
|
/**
|
|
926
1107
|
* Keeps this thread alive ON PURPOSE — the one interval left accounted for
|
|
927
1108
|
* (no `.unref()`).
|