@torrent-tv/proxy 2.89.19 → 2.90.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/services/encode/output/cut-grid.js +35 -0
- package/services/encode/output/index.js +1 -0
- package/services/encode/run-command.js +26 -14
- package/services/server/OutputLifecycle.js +7 -2
- package/services/server/SegmentServing.js +7 -28
- package/services/storage/memory-report.js +37 -1
- package/services/storage/segment-store/SegmentStore.js +87 -9
- package/services/torrent/destroyed-torrents.js +48 -0
- package/services/torrent/torrent-pool.js +28 -6
- package/services/torrent/worker/held-objects.js +55 -0
- package/services/torrent/worker/whole-file-stats.js +46 -0
- package/services/torrent/worker/whole-sources.js +77 -0
- package/services/torrent/worker/worker.js +87 -16
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@torrent-tv/proxy",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.90.0",
|
|
4
4
|
"description": "Torrent proxy client that exposes webseed-like HTTP stream endpoint.",
|
|
5
5
|
"license": "GPL-3.0-or-later",
|
|
6
6
|
"repository": {
|
|
@@ -54,6 +54,6 @@
|
|
|
54
54
|
"@biomejs/biome": "^2.5.7"
|
|
55
55
|
},
|
|
56
56
|
"overrides": {
|
|
57
|
-
"utp-native": "npm:@torrent-tv/utp-native@2.5.3-ttv.
|
|
57
|
+
"utp-native": "npm:@torrent-tv/utp-native@2.5.3-ttv.9"
|
|
58
58
|
}
|
|
59
59
|
}
|
|
@@ -105,3 +105,38 @@ export function computeCutGrid({ useKeyframeGrid, durationSeconds, segDur, keyfr
|
|
|
105
105
|
// all, and an even one serves better than a table with a single entry.
|
|
106
106
|
return boundaries.length >= 2 ? { boundaries, sourceTimes } : uniform();
|
|
107
107
|
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Whether an output is cut at times we hand the muxer, rather than at a
|
|
111
|
+
* duration the muxer chooses for itself.
|
|
112
|
+
*
|
|
113
|
+
* One answer for the command a run is given and for the serving of what it
|
|
114
|
+
* made, because both follow from it: given times mean the `segment` muxer,
|
|
115
|
+
* which writes each piece self-contained under a working name and says when it
|
|
116
|
+
* has closed it; without them the `hls` muxer names its pieces itself and
|
|
117
|
+
* writes one init file beside them.
|
|
118
|
+
*
|
|
119
|
+
* Every grid is handed over, the even one included. The `hls` muxer finishes
|
|
120
|
+
* the piece it has open whenever it ends — on our SIGTERM, and when its input
|
|
121
|
+
* stops — and gives that piece its served name, so a piece shorter than its
|
|
122
|
+
* span became servable with nothing to say so. Field 2026-09-27, on the
|
|
123
|
+
* `segment` muxer before its own guard existed: a run stopped while it wrote
|
|
124
|
+
* #111 left 0.37 s of a 4.2 s piece, the browser appended it, counted the
|
|
125
|
+
* fragment as loaded and went on to the next one, and the picture stood at
|
|
126
|
+
* 463.3 s while the player fetched 515.7 s and beyond.
|
|
127
|
+
*
|
|
128
|
+
* What is left without times is an output with no grid at all, which has no
|
|
129
|
+
* duration to cut.
|
|
130
|
+
*
|
|
131
|
+
* @param {{ segmentFormat?: { explicitTimesMuxerArgs?: () => string[] | null } | null,
|
|
132
|
+
* timeline?: { published?: number[], boundaries?: number[] } | null }} output
|
|
133
|
+
* @returns {boolean}
|
|
134
|
+
*/
|
|
135
|
+
export function cutsAtGivenTimes(output) {
|
|
136
|
+
if (!output?.segmentFormat?.explicitTimesMuxerArgs?.()) {
|
|
137
|
+
return false;
|
|
138
|
+
}
|
|
139
|
+
const published = output.timeline?.published;
|
|
140
|
+
const grid = Array.isArray(published) && published.length > 0 ? published : output.timeline?.boundaries;
|
|
141
|
+
return Array.isArray(grid) && grid.length >= 2;
|
|
142
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
export { OutputSpec, VideoOutput, AudioOutput, CutGrid, isOutputName } from "./OutputSpec.js";
|
|
2
2
|
export { PLAYLIST_FILE_NAME } from "./playlists.js";
|
|
3
|
+
export { cutsAtGivenTimes } from "./cut-grid.js";
|
|
3
4
|
|
|
4
5
|
// FFmpeg may close a segment this far before its requested cut. Both the
|
|
5
6
|
// command and coverage validation use this existing muxer configuration.
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
|
|
18
18
|
import { PLAYLIST_FILE_NAME } from "./output/playlists.js";
|
|
19
19
|
import { AUDIO_TRANSCODE_KBPS } from "./args.js";
|
|
20
|
-
import { SEGMENT_CUT_TIME_DELTA_SECONDS } from "./output/index.js";
|
|
20
|
+
import { cutsAtGivenTimes, SEGMENT_CUT_TIME_DELTA_SECONDS } from "./output/index.js";
|
|
21
21
|
|
|
22
22
|
/**
|
|
23
23
|
* What ffmpeg's own CLI subtracts from an input seek, and therefore what has to
|
|
@@ -364,15 +364,26 @@ export function buildRunCommand({
|
|
|
364
364
|
// the first requested cut. It is never published over another run's output.
|
|
365
365
|
const inputIndex = !transcodeVideo && !audioOnly && explicitTimes && safeIndex > 0
|
|
366
366
|
? safeIndex - 1 : safeIndex;
|
|
367
|
-
// A
|
|
368
|
-
//
|
|
369
|
-
//
|
|
370
|
-
//
|
|
371
|
-
//
|
|
372
|
-
//
|
|
373
|
-
//
|
|
374
|
-
//
|
|
375
|
-
//
|
|
367
|
+
// EVERY OUTPUT WITH A GRID is cut by this list, whatever the grid is and
|
|
368
|
+
// whether the picture is copied or re-encoded (`cutsAtGivenTimes`).
|
|
369
|
+
//
|
|
370
|
+
// A copy needs it even when no keyframe index could be read and the
|
|
371
|
+
// boundaries are a plain grid: saying them outright is what keeps the
|
|
372
|
+
// playlist and the muxer agreeing — ffmpeg moves each cut forward to the
|
|
373
|
+
// first real keyframe, and serving reads back where the piece truly begins.
|
|
374
|
+
// Requiring a keyframe grid here dropped a copy with no index onto the `hls`
|
|
375
|
+
// muxer, which takes no cut list and writes no self-contained pieces, so
|
|
376
|
+
// nothing could read a true start and segments were stamped with times the
|
|
377
|
+
// file does not have — the 4.17 s speech-against-subtitles drift, back again.
|
|
378
|
+
//
|
|
379
|
+
// A re-encode on the even grid needs it for the proof of a closed piece. The
|
|
380
|
+
// `hls` muxer finishes the piece it has open whenever it ends, on our SIGTERM
|
|
381
|
+
// and when its input stops, and renames it to the served name itself, so a
|
|
382
|
+
// stopped run left a piece shorter than its span where any request could
|
|
383
|
+
// take it. Measured 2026-10-05 with ffmpeg 8.1.2 on the addon host, a
|
|
384
|
+
// 4 s piece stopped 3 s in stayed as `segment-00001.mp4` with 3.04 s in it.
|
|
385
|
+
// Through the `segment` muxer the piece is published only on the encoder's
|
|
386
|
+
// word and only after its media is shown to reach its cut.
|
|
376
387
|
//
|
|
377
388
|
// Cut on the grid the PLAYER WAS GIVEN, not on the corrected one. A player
|
|
378
389
|
// places a fragment by the playlist it holds, and that text was written
|
|
@@ -387,7 +398,7 @@ export function buildRunCommand({
|
|
|
387
398
|
// created later inherits the corrected table and PUBLISHES it, so its own
|
|
388
399
|
// playlist and its own cuts agree from the start. What they may not do is
|
|
389
400
|
// move the cuts of a session whose playlist is already being read.
|
|
390
|
-
const gridCutTimes =
|
|
401
|
+
const gridCutTimes = cutsAtGivenTimes({ segmentFormat, timeline })
|
|
391
402
|
? segmentCutTimesFrom(publishedGridFor(timeline), inputIndex)
|
|
392
403
|
: null;
|
|
393
404
|
// Cut times are stated on the grid, for both branches.
|
|
@@ -721,9 +732,10 @@ export function buildRunCommand({
|
|
|
721
732
|
// into the same directory — and the name arrives on the channel above the
|
|
722
733
|
// instant ffmpeg closes it, which is what turns it into the served one.
|
|
723
734
|
//
|
|
724
|
-
// The other branch
|
|
725
|
-
// temporary name of its own (`+temp_file`)
|
|
726
|
-
//
|
|
735
|
+
// The other branch cannot do this: the HLS muxer writes through a
|
|
736
|
+
// temporary name of its own (`+temp_file`) and renames each file itself,
|
|
737
|
+
// including the one it was writing when it ended. It is left for the runs
|
|
738
|
+
// that have no cut list at all.
|
|
727
739
|
//
|
|
728
740
|
// TAGGED WITH THE STRETCH IT WAS GIVEN, which names the run without any
|
|
729
741
|
// counter to keep: intervals never overlap, so two live runs of one output
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
10
|
import { logger } from "../../utils/logger.js";
|
|
11
|
-
import { isOutputName } from "../encode/output/index.js";
|
|
11
|
+
import { isOutputName, PLAYLIST_FILE_NAME } from "../encode/output/index.js";
|
|
12
12
|
import { IDLE_KEEP_MS } from "../storage/keep.js";
|
|
13
13
|
/**
|
|
14
14
|
* How long produced segments are kept after the last request for them.
|
|
@@ -273,7 +273,12 @@ export class OutputLifecycle {
|
|
|
273
273
|
adoptSegmentsLeftBehind() {
|
|
274
274
|
// Which container the segments are in is stated by the key itself; a key
|
|
275
275
|
// that cannot say is a directory that goes (`encode/output-key-format.js`).
|
|
276
|
-
|
|
276
|
+
// The `hls` muxer writes its own list, under the playlist's name, beside the
|
|
277
|
+
// pieces it named itself; none of those pieces is proven whole.
|
|
278
|
+
return this.#host.segmentStore.adoptWhatSurvived(
|
|
279
|
+
(key) => this.#host.segmentFormatOfKey(key),
|
|
280
|
+
{ selfNamedListFileName: PLAYLIST_FILE_NAME }
|
|
281
|
+
);
|
|
277
282
|
}
|
|
278
283
|
|
|
279
284
|
/**
|
|
@@ -13,7 +13,7 @@ import { access, readFile, stat, unlink } from "node:fs/promises";
|
|
|
13
13
|
import { Readable } from "node:stream";
|
|
14
14
|
import path from "node:path";
|
|
15
15
|
import { logger } from "../../utils/logger.js";
|
|
16
|
-
import { isOutputName, PLAYLIST_FILE_NAME } from "../encode/output/index.js";
|
|
16
|
+
import { cutsAtGivenTimes, isOutputName, PLAYLIST_FILE_NAME } from "../encode/output/index.js";
|
|
17
17
|
// The index of variants. Served from the same route as the media playlist, so
|
|
18
18
|
// it needs no path of its own.
|
|
19
19
|
export const MASTER_PLAYLIST_FILE_NAME = "master.m3u8";
|
|
@@ -50,25 +50,6 @@ function isSafeFileName(fileName, segmentFormat) {
|
|
|
50
50
|
segmentFormat.isSegmentFileName(fileName)
|
|
51
51
|
);
|
|
52
52
|
}
|
|
53
|
-
/**
|
|
54
|
-
* Whether this output is cut at times we hand the muxer, rather than at a
|
|
55
|
-
* duration it chooses for itself.
|
|
56
|
-
*
|
|
57
|
-
* A property of the output and not of a run: the cut grid and the branch decide
|
|
58
|
-
* it, so every run of one output answers alike. It decides how a segment is
|
|
59
|
-
* judged finished — see getFileStream.
|
|
60
|
-
*
|
|
61
|
-
* @param {object} session
|
|
62
|
-
* @returns {boolean}
|
|
63
|
-
*/
|
|
64
|
-
function cutsAtGivenTimes(session) {
|
|
65
|
-
const explicit = session?.segmentFormat?.explicitTimesMuxerArgs?.() ?? null;
|
|
66
|
-
if (!explicit) {
|
|
67
|
-
return false;
|
|
68
|
-
}
|
|
69
|
-
return !session.spec.transcodesVideo || session.timeline?.cutGrid === "keyframe";
|
|
70
|
-
}
|
|
71
|
-
|
|
72
53
|
export class SegmentServing {
|
|
73
54
|
/** Serving state keyed by the output object it belongs to. */
|
|
74
55
|
#states = new WeakMap();
|
|
@@ -275,14 +256,12 @@ export class SegmentServing {
|
|
|
275
256
|
return this.#holdForProduction(session, fileName, isPlaylist, options);
|
|
276
257
|
}
|
|
277
258
|
try {
|
|
278
|
-
// Existing is not the same as finished. The `hls` muxer
|
|
279
|
-
// segment to a temporary name and
|
|
280
|
-
//
|
|
281
|
-
//
|
|
282
|
-
//
|
|
283
|
-
//
|
|
284
|
-
// running happily ahead. A segment is finished once the NEXT one has been
|
|
285
|
-
// started, or once the run producing it has ended.
|
|
259
|
+
// Existing is not the same as finished. The `hls` muxer writes each
|
|
260
|
+
// segment to a temporary name and renames it when it ends the segment —
|
|
261
|
+
// at its cut, but also on SIGTERM and when its input stops, so a file
|
|
262
|
+
// appearing there proves only that the muxer let go of it. Every output
|
|
263
|
+
// with a grid is therefore cut by the `segment` muxer, whose file appears
|
|
264
|
+
// when writing begins and is served only once it is published below.
|
|
286
265
|
if (!isPlaylist && cutsAtGivenTimes(session)) {
|
|
287
266
|
// WHAT PROVES A PIECE IS WHOLE is the encoder's own word for it: it
|
|
288
267
|
// names each file on a channel of its own the moment it closes it, and
|
|
@@ -382,6 +382,25 @@ export function readingIsWorthWriting({
|
|
|
382
382
|
return Math.abs(watchedBytes - lastWrittenBytes) >= changeBytes;
|
|
383
383
|
}
|
|
384
384
|
|
|
385
|
+
/**
|
|
386
|
+
* Array-buffer memory that no torrent store accounts for.
|
|
387
|
+
*
|
|
388
|
+
* The heap is not the only thing a snapshot explains: every `ArrayBuffer` is in
|
|
389
|
+
* it, with the size of its backing store and who holds it. On 2026-10-01 the
|
|
390
|
+
* torrent worker reached 4.3 GB of array buffers while its stores had committed
|
|
391
|
+
* 40 MB, the kernel killed the process twice, and no snapshot existed, because
|
|
392
|
+
* the only trigger watched a heap that stood at 300 MB. Pieces the stores hold
|
|
393
|
+
* are expected; what is above them is the question.
|
|
394
|
+
*
|
|
395
|
+
* @param {number} arrayBuffers - The isolate's own `arrayBuffers` figure.
|
|
396
|
+
* @param {{ committedBytes?: number }[]} stores
|
|
397
|
+
* @returns {number}
|
|
398
|
+
*/
|
|
399
|
+
export function unaccountedBufferBytes(arrayBuffers, stores = []) {
|
|
400
|
+
const committed = stores.reduce((sum, store) => sum + (Number(store?.committedBytes) || 0), 0);
|
|
401
|
+
return Math.max(0, (Number(arrayBuffers) || 0) - committed);
|
|
402
|
+
}
|
|
403
|
+
|
|
385
404
|
/**
|
|
386
405
|
* Report memory on a timer until stopped.
|
|
387
406
|
*
|
|
@@ -408,6 +427,9 @@ export function readingIsWorthWriting({
|
|
|
408
427
|
* @param {number} [options.snapshotFloorBytes]
|
|
409
428
|
* @param {number} [options.snapshotGrowthBytes]
|
|
410
429
|
* @param {number} [options.keepSnapshots] - Newest to keep; zero keeps all.
|
|
430
|
+
* @param {number} [options.bufferSnapshotFloorBytes] - Array buffers above what
|
|
431
|
+
* the stores hold that earn a snapshot of their own; zero never takes one.
|
|
432
|
+
* @param {number} [options.bufferSnapshotGrowthBytes]
|
|
411
433
|
* @param {() => string} [options.readExtra] - Figures to append to the line,
|
|
412
434
|
* read at the same instant as the memory itself.
|
|
413
435
|
* @returns {{ stop: () => void }}
|
|
@@ -426,9 +448,12 @@ export function startMemoryReport({
|
|
|
426
448
|
mayKeep = null,
|
|
427
449
|
snapshotFloorBytes = 500 * 1024 * 1024,
|
|
428
450
|
snapshotGrowthBytes = 100 * 1024 * 1024,
|
|
429
|
-
keepSnapshots = 0
|
|
451
|
+
keepSnapshots = 0,
|
|
452
|
+
bufferSnapshotFloorBytes = 0,
|
|
453
|
+
bufferSnapshotGrowthBytes = 500 * 1024 * 1024
|
|
430
454
|
}) {
|
|
431
455
|
let highWater = 0;
|
|
456
|
+
let bufferHighWater = 0;
|
|
432
457
|
/** @type {Record<string, number>} */
|
|
433
458
|
let lastWritten = {};
|
|
434
459
|
let lastWrittenAt = 0;
|
|
@@ -556,6 +581,17 @@ export function startMemoryReport({
|
|
|
556
581
|
highWater = heapWatched;
|
|
557
582
|
await takeSnapshot(heapWatched, "a new high-water of the heap");
|
|
558
583
|
}
|
|
584
|
+
// The same for array buffers, which a snapshot names as well as it names
|
|
585
|
+
// the heap: each one is in it with its size and its holder.
|
|
586
|
+
const unaccounted = unaccountedBufferBytes(processMemory.arrayBuffers, stores);
|
|
587
|
+
if (
|
|
588
|
+
bufferSnapshotFloorBytes > 0 &&
|
|
589
|
+
unaccounted > bufferSnapshotFloorBytes &&
|
|
590
|
+
unaccounted > bufferHighWater + bufferSnapshotGrowthBytes
|
|
591
|
+
) {
|
|
592
|
+
bufferHighWater = unaccounted;
|
|
593
|
+
await takeSnapshot(unaccounted, "a new high-water of array buffers no torrent store accounts for");
|
|
594
|
+
}
|
|
559
595
|
// Under `write` by construction: `anonymousBytes` is only read when the
|
|
560
596
|
// line is, and at one reading a second an unconditional warning would be
|
|
561
597
|
// a line a second for as long as the process stayed large.
|
|
@@ -403,6 +403,74 @@ export class SegmentStore {
|
|
|
403
403
|
return removed;
|
|
404
404
|
}
|
|
405
405
|
|
|
406
|
+
/**
|
|
407
|
+
* Remove the pieces a muxer that names its own pieces left in a directory,
|
|
408
|
+
* and the list it named them in.
|
|
409
|
+
*
|
|
410
|
+
* Asked only of what a previous life of the process left. The `hls` muxer
|
|
411
|
+
* renames the piece it has open when it ends, so a stopped run leaves a
|
|
412
|
+
* piece shorter than its span under the served name — the shape of field
|
|
413
|
+
* 2026-09-27, where #111 held 0.37 s of 4.2 s, the browser appended it and
|
|
414
|
+
* never asked for that stretch again. The list cannot say which piece is the
|
|
415
|
+
* short one, so every piece it names goes and is made again.
|
|
416
|
+
*
|
|
417
|
+
* @param {string} key
|
|
418
|
+
* @param {string} dir
|
|
419
|
+
* @param {{ isSegmentFileName?: (name: string) => boolean, initFileName?: string | null }} format
|
|
420
|
+
* @param {string | null} listFileName
|
|
421
|
+
* @returns {number} How many pieces were removed.
|
|
422
|
+
*/
|
|
423
|
+
#sweepSelfNamed(key, dir, format, listFileName) {
|
|
424
|
+
if (!listFileName) {
|
|
425
|
+
return 0;
|
|
426
|
+
}
|
|
427
|
+
const listPath = path.join(dir, listFileName);
|
|
428
|
+
let text;
|
|
429
|
+
try {
|
|
430
|
+
text = readFileSync(listPath, "utf8");
|
|
431
|
+
} catch {
|
|
432
|
+
return 0;
|
|
433
|
+
}
|
|
434
|
+
let removed = 0;
|
|
435
|
+
for (const line of text.split(/\r?\n/)) {
|
|
436
|
+
const name = line.trim();
|
|
437
|
+
if (name.length === 0 || name.startsWith("#") || !format?.isSegmentFileName?.(name)) {
|
|
438
|
+
continue;
|
|
439
|
+
}
|
|
440
|
+
const piecePath = path.join(dir, name);
|
|
441
|
+
if (!existsSync(piecePath)) {
|
|
442
|
+
continue;
|
|
443
|
+
}
|
|
444
|
+
try {
|
|
445
|
+
rmSync(piecePath, { force: true });
|
|
446
|
+
removed += 1;
|
|
447
|
+
} catch {
|
|
448
|
+
// Then it stays, and is served as it was before this sweep existed.
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
// The list goes with the pieces it named, and so does the init file the
|
|
452
|
+
// same muxer wrote beside them: nothing else writes one into this
|
|
453
|
+
// directory, and a piece cut by the list carries its own header.
|
|
454
|
+
for (const leftover of [listFileName, format?.initFileName]) {
|
|
455
|
+
if (!leftover) {
|
|
456
|
+
continue;
|
|
457
|
+
}
|
|
458
|
+
try {
|
|
459
|
+
rmSync(path.join(dir, leftover), { force: true });
|
|
460
|
+
} catch {
|
|
461
|
+
// It names or describes pieces that are gone; reading it again removes nothing.
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
if (removed > 0) {
|
|
465
|
+
this.#held.delete(key);
|
|
466
|
+
this.#logger.info(
|
|
467
|
+
`segment-store removed ${removed} piece(s) of ${path.basename(dir)} that the hls muxer ` +
|
|
468
|
+
"named itself: nothing proves they reach their cut"
|
|
469
|
+
);
|
|
470
|
+
}
|
|
471
|
+
return removed;
|
|
472
|
+
}
|
|
473
|
+
|
|
406
474
|
/**
|
|
407
475
|
* The encoder has closed a piece: give it the name it is served under.
|
|
408
476
|
*
|
|
@@ -564,10 +632,11 @@ export class SegmentStore {
|
|
|
564
632
|
* muxer writes under a working name and reports the closure on a channel of
|
|
565
633
|
* its own (`publish`); the `hls` muxer writes through a temporary name of its
|
|
566
634
|
* own (`+temp_file`) and reports nothing, so the file simply appears under the
|
|
567
|
-
* name it is served as. That branch
|
|
568
|
-
* even grid — the ordinary quality step — and for those `waitFor` could
|
|
569
|
-
* end on its deadline: the segment was on disk and the request that
|
|
570
|
-
* went on waiting, up to a full minute, and was then answered 503.
|
|
635
|
+
* name it is served as. That branch was then taken by every re-encoded output
|
|
636
|
+
* on the even grid — the ordinary quality step — and for those `waitFor` could
|
|
637
|
+
* only end on its deadline: the segment was on disk and the request that
|
|
638
|
+
* wanted it went on waiting, up to a full minute, and was then answered 503.
|
|
639
|
+
* It is now taken only by a run given no cut list.
|
|
571
640
|
*
|
|
572
641
|
* So the store watches what it owns. The rename into place moves the
|
|
573
642
|
* directory's modification time, the kernel says so, and both branches reach
|
|
@@ -1243,9 +1312,12 @@ export class SegmentStore {
|
|
|
1243
1312
|
* @param {(key: string) => object | null} formatFor - How to read the file
|
|
1244
1313
|
* names of an output, given its key. Null when this proxy cannot serve that
|
|
1245
1314
|
* output at all, and then the directory goes.
|
|
1315
|
+
* @param {{ selfNamedListFileName?: string | null }} [writers] - The list a
|
|
1316
|
+
* muxer that names its own pieces writes beside them. Every piece in it was
|
|
1317
|
+
* given its served name by that muxer and not by a closure we were told of.
|
|
1246
1318
|
* @returns {{ adopted: number, dropped: number, unprovenRemoved: number }}
|
|
1247
1319
|
*/
|
|
1248
|
-
adoptWhatSurvived(formatFor) {
|
|
1320
|
+
adoptWhatSurvived(formatFor, { selfNamedListFileName = null } = {}) {
|
|
1249
1321
|
const swept = this.sweep();
|
|
1250
1322
|
let adopted = 0;
|
|
1251
1323
|
let dropped = 0;
|
|
@@ -1266,15 +1338,21 @@ export class SegmentStore {
|
|
|
1266
1338
|
continue;
|
|
1267
1339
|
}
|
|
1268
1340
|
this.#formats.set(entry.key, format);
|
|
1269
|
-
//
|
|
1270
|
-
//
|
|
1271
|
-
// prove here and nothing to un-prove: what
|
|
1272
|
-
//
|
|
1341
|
+
// A SEGMENT GIVEN ITS SERVED NAME BY US IS COMPLETE, because we give it
|
|
1342
|
+
// only once the encoder has said it is closed and its media reaches its
|
|
1343
|
+
// cut. So there is nothing to prove here and nothing to un-prove: what
|
|
1344
|
+
// the directory holds under those names is what a killed process
|
|
1345
|
+
// finished.
|
|
1273
1346
|
//
|
|
1274
1347
|
// What it may also hold is pieces it was in the middle of, under their
|
|
1275
1348
|
// working names, and those are swept — the file a run was writing when the
|
|
1276
1349
|
// kernel took the process is exactly this.
|
|
1277
1350
|
unprovenRemoved += this.#sweepUnfinished(entry.key, entry.dir, format);
|
|
1351
|
+
// And pieces the `hls` muxer renamed itself. It renames the piece it has
|
|
1352
|
+
// open whenever it ends, on SIGTERM and when its input stops, so nothing
|
|
1353
|
+
// says any of them reaches its cut; an earlier version cut every
|
|
1354
|
+
// re-encode on the even grid that way. The muxer's own list names them.
|
|
1355
|
+
unprovenRemoved += this.#sweepSelfNamed(entry.key, entry.dir, format, selfNamedListFileName);
|
|
1278
1356
|
adopted += 1;
|
|
1279
1357
|
this.#logger.info(
|
|
1280
1358
|
`segment-store adopted ${path.basename(entry.dir)}: ${this.provenNumbers(entry.key).length} ` +
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file Torrents this process has destroyed and the collector has not taken
|
|
3
|
+
* back yet.
|
|
4
|
+
*
|
|
5
|
+
* Destroying a torrent ends its swarm; it frees nothing while anything still
|
|
6
|
+
* refers to the object. On 2026-10-04 a torrent worker held 595 destroyed
|
|
7
|
+
* copies of one film, kept by the listeners of uTP connections whose close
|
|
8
|
+
* never came back and by idle timers left running for an hour, and the only
|
|
9
|
+
* sign of it was memory growing at 17 MB a minute. The heap snapshot that named
|
|
10
|
+
* it was taken by luck. This says it every time the worker reports memory.
|
|
11
|
+
*
|
|
12
|
+
* The count is the collector's own answer, so it lags: a figure that returns
|
|
13
|
+
* towards zero is ordinary, one that only grows is a reference that was not let
|
|
14
|
+
* go.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
let destroyedTotal = 0;
|
|
18
|
+
let notCollected = 0;
|
|
19
|
+
|
|
20
|
+
/** @type {WeakSet<object>} */
|
|
21
|
+
const counted = new WeakSet();
|
|
22
|
+
|
|
23
|
+
const collected = new FinalizationRegistry(() => {
|
|
24
|
+
notCollected -= 1;
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Say that a torrent has been destroyed. Counted once however often it is said.
|
|
29
|
+
*
|
|
30
|
+
* @param {object} torrent
|
|
31
|
+
* @returns {void}
|
|
32
|
+
*/
|
|
33
|
+
export function noteTorrentDestroyed(torrent) {
|
|
34
|
+
if (!torrent || typeof torrent !== "object" || counted.has(torrent)) {
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
counted.add(torrent);
|
|
38
|
+
destroyedTotal += 1;
|
|
39
|
+
notCollected += 1;
|
|
40
|
+
collected.register(torrent, null);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* @returns {{ total: number, notCollected: number }}
|
|
45
|
+
*/
|
|
46
|
+
export function destroyedTorrents() {
|
|
47
|
+
return { total: destroyedTotal, notCollected };
|
|
48
|
+
}
|
|
@@ -21,6 +21,7 @@ import { demandFor, forgetTorrent, reconcileAll, hasUnmetDemand } from "./downlo
|
|
|
21
21
|
import { withdrawClaim } from "./download/withdraw-claim.js";
|
|
22
22
|
import { isAtAWatchingViewer, isBehindEverybody, isNobodyComingNow } from "../viewer/PriorityMap.js";
|
|
23
23
|
import { deriveSourceKey } from "../../utils/torrent-source-key.js";
|
|
24
|
+
import { noteTorrentDestroyed } from "./destroyed-torrents.js";
|
|
24
25
|
|
|
25
26
|
/** How a window stated from the priority map names itself. */
|
|
26
27
|
const MAP_CLAIMANT = "priority-map";
|
|
@@ -1683,6 +1684,10 @@ export class TorrentPool {
|
|
|
1683
1684
|
logger.warn(`torrent-pool: [${label()}] warning: ${formatWarning(warning)}`);
|
|
1684
1685
|
});
|
|
1685
1686
|
|
|
1687
|
+
// Counted from here until the collector takes it, which is the only way a
|
|
1688
|
+
// destroyed torrent that something still holds can be seen at all.
|
|
1689
|
+
torrent.once("close", () => noteTorrentDestroyed(torrent));
|
|
1690
|
+
|
|
1686
1691
|
// A TORRENT THAT DIED WITHOUT US ASKING LEAVES ITS RECORD BEHIND, and the
|
|
1687
1692
|
// record goes on answering. Field 2026-09-11: the store refused a block,
|
|
1688
1693
|
// the client destroyed the torrent, and for the rest of the process every
|
|
@@ -1862,8 +1867,7 @@ export class TorrentPool {
|
|
|
1862
1867
|
this.torrents.delete(otherKey);
|
|
1863
1868
|
}
|
|
1864
1869
|
}
|
|
1865
|
-
this.#
|
|
1866
|
-
this.#readPositionByTorrent.delete(existing);
|
|
1870
|
+
this.#forgetBookkeeping(existing);
|
|
1867
1871
|
this.client.remove(existing, { destroyStore: true }, () => {
|
|
1868
1872
|
const addedReplacement = this.client.add(torrentId, {
|
|
1869
1873
|
store: this.#pieceStore,
|
|
@@ -2083,8 +2087,7 @@ export class TorrentPool {
|
|
|
2083
2087
|
return;
|
|
2084
2088
|
}
|
|
2085
2089
|
forgetTorrent(torrent);
|
|
2086
|
-
this.#
|
|
2087
|
-
this.#readPositionByTorrent.delete(torrent);
|
|
2090
|
+
this.#forgetBookkeeping(torrent);
|
|
2088
2091
|
const name = typeof torrent.name === "string" ? torrent.name : "(unknown)";
|
|
2089
2092
|
logger.warn(
|
|
2090
2093
|
`torrent-pool: forgot "${name}" [${String(torrent.infoHash ?? "?").slice(0, 8)}] — ` +
|
|
@@ -2107,6 +2110,26 @@ export class TorrentPool {
|
|
|
2107
2110
|
this.#removeTorrent(torrent, reason);
|
|
2108
2111
|
}
|
|
2109
2112
|
|
|
2113
|
+
/**
|
|
2114
|
+
* Drop everything this pool keeps per torrent, for one that is going.
|
|
2115
|
+
*
|
|
2116
|
+
* Every one of these holds the torrent strongly, and the idle timer above all:
|
|
2117
|
+
* left running, it kept each removed torrent — its pieces, its wires, the
|
|
2118
|
+
* whole object — for the hour it was set for, then fired on the corpse and
|
|
2119
|
+
* logged a second removal. Field 2026-10-04: 184 such removals of a torrent
|
|
2120
|
+
* already gone, and 595 destroyed copies of it alive in one heap.
|
|
2121
|
+
*
|
|
2122
|
+
* @param {import("webtorrent").Torrent} torrent
|
|
2123
|
+
* @returns {void}
|
|
2124
|
+
*/
|
|
2125
|
+
#forgetBookkeeping(torrent) {
|
|
2126
|
+
this.#cancelIdleRemoval(torrent);
|
|
2127
|
+
this.#lastAccess.delete(torrent);
|
|
2128
|
+
this.#readPositionByTorrent.delete(torrent);
|
|
2129
|
+
this.#stallSince.delete(torrent);
|
|
2130
|
+
this.#stallReportedAt.delete(torrent);
|
|
2131
|
+
}
|
|
2132
|
+
|
|
2110
2133
|
#removeTorrent(torrent, reason = "unknown") {
|
|
2111
2134
|
// Everything anybody stated for this torrent goes with it.
|
|
2112
2135
|
forgetTorrent(torrent);
|
|
@@ -2126,8 +2149,7 @@ export class TorrentPool {
|
|
|
2126
2149
|
break;
|
|
2127
2150
|
}
|
|
2128
2151
|
}
|
|
2129
|
-
this.#
|
|
2130
|
-
this.#readPositionByTorrent.delete(torrent);
|
|
2152
|
+
this.#forgetBookkeeping(torrent);
|
|
2131
2153
|
logger.warn(`torrent-pool: removing torrent "${name}" [${infoHash}] reason=${reason} stated=${stated} downloaded=${hasData}B caller=${caller}`);
|
|
2132
2154
|
try {
|
|
2133
2155
|
torrent.destroy({ destroyStore: true }, () => {
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file What the torrent thread holds beyond the bytes it accounts for: live
|
|
3
|
+
* and destroyed torrents, and the uTP sockets and connections behind them.
|
|
4
|
+
*
|
|
5
|
+
* These are the objects the heap snapshot of 2026-10-04 found holding the
|
|
6
|
+
* growth: 957 uTP sockets whose close never completed, 2101 of their 128 KB
|
|
7
|
+
* read buffers, and through their listeners 595 torrents destroyed long
|
|
8
|
+
* before. None of it was in any figure this proxy printed. Read on the same
|
|
9
|
+
* line as the memory, so the count and the mass are the same instant.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { createRequire } from "node:module";
|
|
13
|
+
|
|
14
|
+
const require = createRequire(import.meta.url);
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* The uTP module's own count of what it holds open, or null where the module is
|
|
18
|
+
* not built or is a version that does not count.
|
|
19
|
+
*
|
|
20
|
+
* Required by name, which resolves to the same copy WebTorrent loads: the image
|
|
21
|
+
* removes every nested copy and refuses to build if one survives.
|
|
22
|
+
*
|
|
23
|
+
* @returns {{ sockets: number, connections: number, closing: number } | null}
|
|
24
|
+
*/
|
|
25
|
+
export function readUtpOpenCount() {
|
|
26
|
+
try {
|
|
27
|
+
const utp = require("utp-native");
|
|
28
|
+
return typeof utp?.openCount === "function" ? utp.openCount() : null;
|
|
29
|
+
} catch {
|
|
30
|
+
// silent-ok: no uTP on this host means TCP peers only, and nothing to count.
|
|
31
|
+
return null;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* One clause for the memory line.
|
|
37
|
+
*
|
|
38
|
+
* @param {object} held
|
|
39
|
+
* @param {number} held.liveTorrents - Torrents the client has now.
|
|
40
|
+
* @param {{ total: number, notCollected: number }} held.destroyed
|
|
41
|
+
* @param {{ sockets: number, connections: number, closing: number } | null} held.utp
|
|
42
|
+
* @returns {string}
|
|
43
|
+
*/
|
|
44
|
+
export function describeHeldObjects({ liveTorrents, destroyed, utp }) {
|
|
45
|
+
const torrents =
|
|
46
|
+
`torrents ${liveTorrents} live, ${destroyed.notCollected} destroyed and not collected ` +
|
|
47
|
+
`(${destroyed.total} destroyed in all)`;
|
|
48
|
+
if (!utp) {
|
|
49
|
+
return torrents;
|
|
50
|
+
}
|
|
51
|
+
return (
|
|
52
|
+
`${torrents}; uTP ${utp.sockets} socket(s), ${utp.connections} connection(s), ` +
|
|
53
|
+
`${utp.closing} destroyed and still closing`
|
|
54
|
+
);
|
|
55
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file The download figures of a file that is held whole on disk.
|
|
3
|
+
*
|
|
4
|
+
* Once every file of a torrent is a file, the torrent is removed: it has
|
|
5
|
+
* nothing left to fetch. The figures a viewer's page polls for still have an
|
|
6
|
+
* answer, and it is known without the torrent — everything is here and nothing
|
|
7
|
+
* is arriving. Asking the torrent for them instead brought it back: on
|
|
8
|
+
* 2026-10-04 a film watched from its whole file was removed and added again
|
|
9
|
+
* 671 times in two hours, once for every poll after each removal, and every
|
|
10
|
+
* copy it left behind stayed in memory.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* The same shape `TorrentPool.getFileStats` answers with, for a whole file.
|
|
15
|
+
*
|
|
16
|
+
* @param {number} length - Bytes in the file.
|
|
17
|
+
* @returns {object}
|
|
18
|
+
*/
|
|
19
|
+
export function wholeFileStats(length) {
|
|
20
|
+
const fileLength = Number.isFinite(length) && length > 0 ? length : 0;
|
|
21
|
+
return {
|
|
22
|
+
numPeers: 0,
|
|
23
|
+
downloadSpeed: 0,
|
|
24
|
+
uploadSpeed: 0,
|
|
25
|
+
connectedPeers: 0,
|
|
26
|
+
deliveringPeers: 0,
|
|
27
|
+
knownPeers: null,
|
|
28
|
+
queuedPeers: null,
|
|
29
|
+
trackerSeeders: null,
|
|
30
|
+
trackerLeechers: null,
|
|
31
|
+
trackersAnswered: 0,
|
|
32
|
+
secondsToFirstPeer: null,
|
|
33
|
+
secondsWaitingForFirstPeer: null,
|
|
34
|
+
fileProgress: 1,
|
|
35
|
+
fileDownloaded: fileLength,
|
|
36
|
+
fileLength,
|
|
37
|
+
residence: fileLength > 0 ? [{ start: 0, end: fileLength, location: "whole-file" }] : [],
|
|
38
|
+
resumeNeededBytes: null,
|
|
39
|
+
resumeDownloadedBytes: null,
|
|
40
|
+
resumeAnchorByteStart: null,
|
|
41
|
+
headerBytes: null,
|
|
42
|
+
headerDownloadedBytes: null,
|
|
43
|
+
// Nothing is delivered by a swarm, so there is no supply to describe.
|
|
44
|
+
supply: null
|
|
45
|
+
};
|
|
46
|
+
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file Which files and sources are held whole on disk, asked by the torrent
|
|
3
|
+
* thread before it brings a torrent back.
|
|
4
|
+
*
|
|
5
|
+
* A torrent whose every file is a file has nothing to fetch, so it is removed;
|
|
6
|
+
* anything that names the source again rebuilt it from its recipe. That is
|
|
7
|
+
* right for a read the files cannot answer, and it was wrong for everything
|
|
8
|
+
* else. The page polls the download figures and the viewers state a priority
|
|
9
|
+
* map for as long as the film plays, and each of those rebuilt the torrent
|
|
10
|
+
* only to be told nothing was missing; ten seconds later the sweep removed it
|
|
11
|
+
* again. Field 2026-10-04: 671 removals in two hours of one viewer watching one
|
|
12
|
+
* whole file, and every copy left behind in memory.
|
|
13
|
+
*
|
|
14
|
+
* So the rule is stated once, here: a file held whole does not bring its
|
|
15
|
+
* torrent back to be steered or measured, and a source held whole is described
|
|
16
|
+
* by what was written down when its torrent went. The torrent is used if it
|
|
17
|
+
* exists.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* What the torrent thread says about a source when it is opened.
|
|
22
|
+
*
|
|
23
|
+
* @typedef {object} SourceDescription
|
|
24
|
+
* @property {string} infoHash
|
|
25
|
+
* @property {string} name
|
|
26
|
+
* @property {number} pieceLength
|
|
27
|
+
* @property {{ index: number, name: string, path: string, length: number }[]} files
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* @param {object} params
|
|
32
|
+
* @param {(infoHash: string, fileIndex: number) => ({ length: number } | null)} params.find
|
|
33
|
+
* The whole file kept for this infohash and index, or null.
|
|
34
|
+
* @returns {{
|
|
35
|
+
* fileOf: (sourceKey: string, fileIndex: number) => ({ length: number } | null),
|
|
36
|
+
* remember: (sourceKey: string, description: SourceDescription) => void,
|
|
37
|
+
* isWhole: (sourceKey: string) => boolean,
|
|
38
|
+
* describe: (sourceKey: string) => (SourceDescription | null)
|
|
39
|
+
* }}
|
|
40
|
+
*/
|
|
41
|
+
export function createWholeSources({ find }) {
|
|
42
|
+
/** @type {Map<string, SourceDescription>} */
|
|
43
|
+
const descriptions = new Map();
|
|
44
|
+
|
|
45
|
+
const infoHashOf = (sourceKey) => {
|
|
46
|
+
const key = String(sourceKey ?? "");
|
|
47
|
+
return key.startsWith("torrent:") ? key.slice("torrent:".length).toLowerCase() : null;
|
|
48
|
+
};
|
|
49
|
+
|
|
50
|
+
const fileOf = (sourceKey, fileIndex) => {
|
|
51
|
+
const infoHash = infoHashOf(sourceKey);
|
|
52
|
+
if (infoHash === null || !Number.isInteger(fileIndex) || fileIndex < 0) {
|
|
53
|
+
return null;
|
|
54
|
+
}
|
|
55
|
+
return find(infoHash, fileIndex) ?? null;
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
// Asked again each time rather than remembered as a verdict: a whole file can
|
|
59
|
+
// be removed for space, and then the torrent has work again.
|
|
60
|
+
const isWhole = (sourceKey) => {
|
|
61
|
+
const files = descriptions.get(String(sourceKey))?.files ?? [];
|
|
62
|
+
return files.length > 0 && files.every((file) => fileOf(sourceKey, file.index) !== null);
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
return {
|
|
66
|
+
fileOf,
|
|
67
|
+
remember(sourceKey, description) {
|
|
68
|
+
if (infoHashOf(sourceKey) !== null && Array.isArray(description?.files) && description.files.length > 0) {
|
|
69
|
+
descriptions.set(String(sourceKey), description);
|
|
70
|
+
}
|
|
71
|
+
},
|
|
72
|
+
isWhole,
|
|
73
|
+
describe(sourceKey) {
|
|
74
|
+
return isWhole(sourceKey) ? descriptions.get(String(sourceKey)) : null;
|
|
75
|
+
}
|
|
76
|
+
};
|
|
77
|
+
}
|
|
@@ -35,6 +35,10 @@ import { CompletedFiles, completedFilesRoot } from "../../storage/files/Complete
|
|
|
35
35
|
import { pieceFromWholeFiles, pieceIsInWholeFiles } from "../../storage/files/piece-from-whole-file.js";
|
|
36
36
|
import { Command, Event } from "./protocol.js";
|
|
37
37
|
import { filesInUse } from "./files-in-use.js";
|
|
38
|
+
import { createWholeSources } from "./whole-sources.js";
|
|
39
|
+
import { wholeFileStats } from "./whole-file-stats.js";
|
|
40
|
+
import { describeHeldObjects, readUtpOpenCount } from "./held-objects.js";
|
|
41
|
+
import { destroyedTorrents } from "../destroyed-torrents.js";
|
|
38
42
|
import { startMemoryReport, WORKER_MEMORY_SAMPLE_MS } from "../../storage/memory-report.js";
|
|
39
43
|
import { forwardLogsTo, logger } from "../../../utils/logger.js";
|
|
40
44
|
|
|
@@ -153,6 +157,22 @@ async function knownTorrent(sourceKey) {
|
|
|
153
157
|
return isUsableTorrentHandle(torrent) ? torrent : null;
|
|
154
158
|
}
|
|
155
159
|
|
|
160
|
+
/**
|
|
161
|
+
* The torrent for one file, but never one rebuilt for a file held whole.
|
|
162
|
+
*
|
|
163
|
+
* For the questions that steer or measure a download: a file held whole has no
|
|
164
|
+
* download, and rebuilding its torrent to be told so is what made the torrent
|
|
165
|
+
* come back after every removal (see `whole-sources.js`). The torrent is still
|
|
166
|
+
* used while it exists.
|
|
167
|
+
*
|
|
168
|
+
* @param {string} sourceKey
|
|
169
|
+
* @param {number} fileIndex
|
|
170
|
+
* @returns {Promise<import("webtorrent").Torrent | null>}
|
|
171
|
+
*/
|
|
172
|
+
async function torrentUnlessWhole(sourceKey, fileIndex) {
|
|
173
|
+
return wholeSources.fileOf(sourceKey, fileIndex) ? knownTorrent(sourceKey) : requireTorrent(sourceKey);
|
|
174
|
+
}
|
|
175
|
+
|
|
156
176
|
async function requireTorrent(sourceKey) {
|
|
157
177
|
const pending = torrentsByKey.get(sourceKey);
|
|
158
178
|
if (!pending) {
|
|
@@ -350,7 +370,18 @@ async function runCommand(command, params, id) {
|
|
|
350
370
|
}
|
|
351
371
|
});
|
|
352
372
|
}
|
|
353
|
-
|
|
373
|
+
// A torrent removed because it was downloaded whole is answered for by
|
|
374
|
+
// what was written down when it went: a destroyed torrent keeps its object
|
|
375
|
+
// and empties its `files`, so it would hand a viewer opening the film an
|
|
376
|
+
// empty list, and rebuilding it here would bring it back on every stream
|
|
377
|
+
// and stats request, which all come through this command. Any other
|
|
378
|
+
// destroyed torrent is rebuilt, because its files still need it.
|
|
379
|
+
const added = await pending;
|
|
380
|
+
const described = isUsableTorrentHandle(added) ? null : wholeSources.describe(params.sourceKey);
|
|
381
|
+
if (described) {
|
|
382
|
+
return described;
|
|
383
|
+
}
|
|
384
|
+
const torrent = isUsableTorrentHandle(added) ? added : await requireTorrent(params.sourceKey);
|
|
354
385
|
return {
|
|
355
386
|
infoHash: torrent.infoHash,
|
|
356
387
|
name: torrent.name,
|
|
@@ -369,6 +400,10 @@ async function runCommand(command, params, id) {
|
|
|
369
400
|
}
|
|
370
401
|
|
|
371
402
|
case Command.LIST_FILES: {
|
|
403
|
+
const described = (await knownTorrent(params.sourceKey)) ? null : wholeSources.describe(params.sourceKey);
|
|
404
|
+
if (described) {
|
|
405
|
+
return described.files;
|
|
406
|
+
}
|
|
372
407
|
const torrent = await requireTorrent(params.sourceKey);
|
|
373
408
|
return (torrent.files ?? []).map((file, index) => ({
|
|
374
409
|
index,
|
|
@@ -419,19 +454,24 @@ async function runCommand(command, params, id) {
|
|
|
419
454
|
|
|
420
455
|
|
|
421
456
|
case Command.FILL_FILE: {
|
|
422
|
-
const torrent = await
|
|
457
|
+
const torrent = await torrentUnlessWhole(params.sourceKey, params.fileIndex);
|
|
423
458
|
return {
|
|
424
|
-
started: fillFileInBackground(torrent, params.fileIndex, params.sourceKey)
|
|
459
|
+
started: torrent ? fillFileInBackground(torrent, params.fileIndex, params.sourceKey) : false
|
|
425
460
|
};
|
|
426
461
|
}
|
|
427
462
|
|
|
428
463
|
case Command.FILL_TORRENT: {
|
|
429
|
-
const torrent =
|
|
430
|
-
|
|
464
|
+
const torrent = wholeSources.isWhole(params.sourceKey)
|
|
465
|
+
? await knownTorrent(params.sourceKey)
|
|
466
|
+
: await requireTorrent(params.sourceKey);
|
|
467
|
+
return { started: torrent ? pool.fillTorrentAsCapacityAllows(torrent) : false };
|
|
431
468
|
}
|
|
432
469
|
|
|
433
470
|
case Command.WARM_POSITION: {
|
|
434
|
-
const torrent = await
|
|
471
|
+
const torrent = await torrentUnlessWhole(params.sourceKey, params.fileIndex);
|
|
472
|
+
if (!torrent) {
|
|
473
|
+
return { started: false };
|
|
474
|
+
}
|
|
435
475
|
return {
|
|
436
476
|
started: await warmResumePosition(
|
|
437
477
|
torrent,
|
|
@@ -451,7 +491,10 @@ async function runCommand(command, params, id) {
|
|
|
451
491
|
|
|
452
492
|
|
|
453
493
|
case Command.FILE_STATS: {
|
|
454
|
-
const torrent = await
|
|
494
|
+
const torrent = await torrentUnlessWhole(params.sourceKey, params.fileIndex);
|
|
495
|
+
if (!torrent) {
|
|
496
|
+
return wholeFileStats(wholeSources.fileOf(params.sourceKey, params.fileIndex)?.length ?? 0);
|
|
497
|
+
}
|
|
455
498
|
const stats = pool.getFileStats(torrent, params.fileIndex, {
|
|
456
499
|
resumeAnchorByteStart: params.resumeAnchorByteStart ?? null
|
|
457
500
|
});
|
|
@@ -475,9 +518,10 @@ async function runCommand(command, params, id) {
|
|
|
475
518
|
// handle from its recipe, so asking that way would re-add a torrent in
|
|
476
519
|
// order to be told that nothing is wanted of it.
|
|
477
520
|
const zones = Array.isArray(params.zones) ? params.zones : [];
|
|
521
|
+
// Nor may one for a file held whole, which has nothing left to fetch.
|
|
478
522
|
const torrent = zones.length === 0
|
|
479
523
|
? await knownTorrent(params.sourceKey)
|
|
480
|
-
: await
|
|
524
|
+
: await torrentUnlessWhole(params.sourceKey, params.fileIndex);
|
|
481
525
|
if (torrent) {
|
|
482
526
|
pool.applyPriorityMap(torrent, params.fileIndex, zones, params.durationSeconds);
|
|
483
527
|
}
|
|
@@ -485,16 +529,18 @@ async function runCommand(command, params, id) {
|
|
|
485
529
|
}
|
|
486
530
|
|
|
487
531
|
case Command.PRIORITIZE: {
|
|
488
|
-
const torrent = await
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
532
|
+
const torrent = await torrentUnlessWhole(params.sourceKey, params.fileIndex);
|
|
533
|
+
if (torrent) {
|
|
534
|
+
pool.prioritizeByteRange(torrent, params.fileIndex, params.byteStart, params.windowBytes, {
|
|
535
|
+
wholeFileRead: params.wholeFileRead === true
|
|
536
|
+
});
|
|
537
|
+
}
|
|
492
538
|
return true;
|
|
493
539
|
}
|
|
494
540
|
|
|
495
541
|
case Command.PREFETCH_EDGES: {
|
|
496
|
-
const torrent = await
|
|
497
|
-
return pool.prefetchFileEdges(torrent, params.fileIndex, params.options ?? {});
|
|
542
|
+
const torrent = await torrentUnlessWhole(params.sourceKey, params.fileIndex);
|
|
543
|
+
return torrent ? pool.prefetchFileEdges(torrent, params.fileIndex, params.options ?? {}) : undefined;
|
|
498
544
|
}
|
|
499
545
|
|
|
500
546
|
case Command.READ_RANGE: {
|
|
@@ -520,7 +566,11 @@ async function runCommand(command, params, id) {
|
|
|
520
566
|
}
|
|
521
567
|
|
|
522
568
|
case Command.HELD_RANGES: {
|
|
523
|
-
const torrent = await
|
|
569
|
+
const torrent = await torrentUnlessWhole(params.sourceKey, params.fileIndex);
|
|
570
|
+
if (!torrent) {
|
|
571
|
+
const length = wholeSources.fileOf(params.sourceKey, params.fileIndex)?.length ?? 0;
|
|
572
|
+
return { ranges: length > 0 ? [[0, length - 1]] : [] };
|
|
573
|
+
}
|
|
524
574
|
return { ranges: heldRangesOf(torrent, params.fileIndex) };
|
|
525
575
|
}
|
|
526
576
|
|
|
@@ -619,7 +669,14 @@ startMemoryReport({
|
|
|
619
669
|
// thread still refers to or buffers the collector has not reached yet. On
|
|
620
670
|
// separate timers the two were up to a minute apart and could not be
|
|
621
671
|
// compared at all (roadmap item 2).
|
|
622
|
-
readExtra:
|
|
672
|
+
readExtra: () => [
|
|
673
|
+
describePieceBuffers(),
|
|
674
|
+
describeHeldObjects({
|
|
675
|
+
liveTorrents: pool.client?.torrents?.length ?? 0,
|
|
676
|
+
destroyed: destroyedTorrents(),
|
|
677
|
+
utp: readUtpOpenCount()
|
|
678
|
+
})
|
|
679
|
+
].filter(Boolean).join("; "),
|
|
623
680
|
scope: "thread",
|
|
624
681
|
label: "torrent worker",
|
|
625
682
|
intervalMs: WORKER_MEMORY_SAMPLE_MS,
|
|
@@ -628,6 +685,13 @@ startMemoryReport({
|
|
|
628
685
|
snapshotDir: workerData?.stateDir || undefined,
|
|
629
686
|
snapshotFloorBytes: 400 * 1024 * 1024,
|
|
630
687
|
snapshotGrowthBytes: 400 * 1024 * 1024,
|
|
688
|
+
// CHOSEN, not derived: no healthy session has recorded this figure yet, so
|
|
689
|
+
// there is no measured normal to stand above. It is set well below the
|
|
690
|
+
// 2.1-4.3 GB the worker held in the minutes before the kills of 2026-10-01,
|
|
691
|
+
// while the machine still has room to write a snapshot. The line prints what the
|
|
692
|
+
// stores hold beside it, which is what a derived floor will come from.
|
|
693
|
+
bufferSnapshotFloorBytes: 500 * 1024 * 1024,
|
|
694
|
+
bufferSnapshotGrowthBytes: 500 * 1024 * 1024,
|
|
631
695
|
keepSnapshots: 3
|
|
632
696
|
});
|
|
633
697
|
|
|
@@ -880,6 +944,7 @@ setInterval(() => {
|
|
|
880
944
|
* thread writes them, the main thread serves them without asking anybody.
|
|
881
945
|
*/
|
|
882
946
|
const completedFiles = new CompletedFiles({ root: completedFilesRoot() });
|
|
947
|
+
const wholeSources = createWholeSources({ find: (infoHash, fileIndex) => completedFiles.find(infoHash, fileIndex) });
|
|
883
948
|
|
|
884
949
|
/**
|
|
885
950
|
* Which torrent a set of files belongs to.
|
|
@@ -1047,6 +1112,12 @@ async function keepWholeFiles() {
|
|
|
1047
1112
|
// checked against what the torrent says. Re-hashing a gigabyte and a
|
|
1048
1113
|
// half to learn what we wrote down is a minute of a viewer's time for
|
|
1049
1114
|
// nothing.
|
|
1115
|
+
wholeSources.remember(sourceKey, {
|
|
1116
|
+
infoHash: torrent.infoHash,
|
|
1117
|
+
name: torrent.name,
|
|
1118
|
+
pieceLength: Number(torrent.pieceLength) || 0,
|
|
1119
|
+
files: torrent.files.map((file, index) => ({ index, name: file.name, path: file.path, length: file.length }))
|
|
1120
|
+
});
|
|
1050
1121
|
pool.remove(torrent, "downloaded-whole");
|
|
1051
1122
|
pool.addWholeSource(sourceKey);
|
|
1052
1123
|
}
|