@torrent-tv/proxy 2.9.71 → 2.9.72
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/package.json +1 -1
- package/services/torrent-worker/pool-adapter.js +179 -171
- package/services/torrent-worker/worker.js +265 -255
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
## 2.9.72
|
|
2
|
+
|
|
3
|
+
- **Fix**: Playback broke entirely after the torrent moved to its own thread (2.9.71): the torrent was deleted **with its downloaded data** while still being read, after which every read hung and ffmpeg saw an empty input (`Stream ends prematurely at 0, should be 3303133078`), and the container-index read took 73 s to return nothing. Cause: `acquireFile` was dispatched without awaiting while its release was sent normally, so a release could overtake the acquire it belonged to; the reader count then hit zero mid-read and the idle sweep fired (`removed idle torrent ... and its store`). Two fixes, each sufficient alone: the release is now chained onto the acquire so it can never arrive first, and the worker additionally holds the file for the whole duration of the read — a claim that lives inside the read and so cannot be reordered against it. Not reproducible locally, where the test torrent was fully downloaded and never went idle.
|
|
4
|
+
|
|
1
5
|
## 2.9.71
|
|
2
6
|
|
|
3
7
|
- **New**: The torrent client now runs on its own thread (`services/torrent-worker/`). Profiling a live seek (2026-08-02) found the main thread ~85% occupied by WebTorrent — buffer concatenation in `uint8-util` ~15%, `_updateWire` and its wrapper ~9%, garbage collection ~5%, and **no piece hashing at all**, which had been the standing assumption — while three of four cores idled. Serving a segment shared that thread, so reading an already-finished 10 MB file off SSD took **12-23 s** where handing it to the channel took 125 ms. Measured after the split, through the real `/stream` route: **3 MB in 0.05-0.12 s** (~500 Mbps), roughly a hundredfold improvement, with main-thread event-loop delay down from 300-390 ms to **28-38 ms**.
|
package/package.json
CHANGED
|
@@ -1,171 +1,179 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @file `TorrentPool`'s interface, served from the worker thread.
|
|
3
|
-
*
|
|
4
|
-
* The routes, the planner, the health report and the session manager all reach
|
|
5
|
-
* for a torrent pool and use it the same handful of ways. Rather than rewrite
|
|
6
|
-
* every one of them to thread a `sourceKey` through and await what used to be
|
|
7
|
-
* immediate, this presents the shape they already expect and does the thread
|
|
8
|
-
* hop behind it. Swapping the implementation is then a one-line change at
|
|
9
|
-
* construction, and the call sites are untouched — which is what keeps a change
|
|
10
|
-
* of this size reviewable.
|
|
11
|
-
*
|
|
12
|
-
* Two accommodations are needed, and both are deliberate:
|
|
13
|
-
*
|
|
14
|
-
* - **`acquireFile` and `prioritizeByteRange` stay synchronous.** They return
|
|
15
|
-
* nothing the caller inspects, so the command is dispatched and not awaited.
|
|
16
|
-
* `acquireFile` hands back a release function exactly as before, which sends
|
|
17
|
-
* its own command when called. Awaiting them would mean touching every call
|
|
18
|
-
* site for no observable gain.
|
|
19
|
-
* - **`getTorrent` needs a `sourceKey`.** Torrent objects cannot cross a
|
|
20
|
-
* thread, so the worker keys them. Callers that have one pass it; the rest
|
|
21
|
-
* get one derived from the source itself, so the identity stays stable
|
|
22
|
-
* across calls for the same torrent.
|
|
23
|
-
*/
|
|
24
|
-
|
|
25
|
-
import crypto from "node:crypto";
|
|
26
|
-
import { TorrentWorkerClient } from "./client.js";
|
|
27
|
-
|
|
28
|
-
/**
|
|
29
|
-
* Stable key for a source, matching how the worker keys its torrents.
|
|
30
|
-
*
|
|
31
|
-
* Derived from the source itself rather than handed out per request, so two
|
|
32
|
-
* routes asking for the same torrent name the same thing on the worker side.
|
|
33
|
-
*
|
|
34
|
-
* @param {"magnet" | "torrent"} sourceType
|
|
35
|
-
* @param {string} source
|
|
36
|
-
* @returns {string}
|
|
37
|
-
*/
|
|
38
|
-
function deriveSourceKey(sourceType, source) {
|
|
39
|
-
return `${sourceType}:${crypto.createHash("sha1").update(source).digest("hex")}`;
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
/**
|
|
43
|
-
* A torrent pool whose work happens on another thread.
|
|
44
|
-
*
|
|
45
|
-
* See `protocol.js` for why: the torrent was taking ~85% of the main thread and
|
|
46
|
-
* everything owed to a viewer queued behind it.
|
|
47
|
-
*/
|
|
48
|
-
export class WorkerTorrentPool {
|
|
49
|
-
#client;
|
|
50
|
-
/** Stand-ins by source key, so repeat calls return the same object. */
|
|
51
|
-
#torrents = new Map();
|
|
52
|
-
|
|
53
|
-
/**
|
|
54
|
-
* @param {{ maxDiskBytes?: number }} [options]
|
|
55
|
-
*/
|
|
56
|
-
constructor(options = {}) {
|
|
57
|
-
this.#client = new TorrentWorkerClient(options);
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
/**
|
|
61
|
-
* Load (or join) a torrent and return a stand-in for it.
|
|
62
|
-
*
|
|
63
|
-
* @param {"magnet" | "torrent"} sourceType
|
|
64
|
-
* @param {string} source
|
|
65
|
-
* @returns {Promise<object>}
|
|
66
|
-
*/
|
|
67
|
-
async getTorrent(sourceType, source) {
|
|
68
|
-
const sourceKey = deriveSourceKey(sourceType, source);
|
|
69
|
-
const existing = this.#torrents.get(sourceKey);
|
|
70
|
-
if (existing) {
|
|
71
|
-
return existing;
|
|
72
|
-
}
|
|
73
|
-
const torrent = await this.#client.getTorrent({ sourceKey, sourceType, source });
|
|
74
|
-
this.#torrents.set(sourceKey, torrent);
|
|
75
|
-
return torrent;
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
/**
|
|
79
|
-
* Claim a file for reading; the returned function releases it.
|
|
80
|
-
*
|
|
81
|
-
* Synchronous by design — see the file header.
|
|
82
|
-
*
|
|
83
|
-
* @param {object} torrent - A stand-in from {@link getTorrent}.
|
|
84
|
-
* @param {number} fileIndex
|
|
85
|
-
* @returns {() => void}
|
|
86
|
-
*/
|
|
87
|
-
acquireFile(torrent, fileIndex) {
|
|
88
|
-
const sourceKey = torrent?.sourceKey;
|
|
89
|
-
if (!sourceKey) {
|
|
90
|
-
return () => undefined;
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
*
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
1
|
+
/**
|
|
2
|
+
* @file `TorrentPool`'s interface, served from the worker thread.
|
|
3
|
+
*
|
|
4
|
+
* The routes, the planner, the health report and the session manager all reach
|
|
5
|
+
* for a torrent pool and use it the same handful of ways. Rather than rewrite
|
|
6
|
+
* every one of them to thread a `sourceKey` through and await what used to be
|
|
7
|
+
* immediate, this presents the shape they already expect and does the thread
|
|
8
|
+
* hop behind it. Swapping the implementation is then a one-line change at
|
|
9
|
+
* construction, and the call sites are untouched — which is what keeps a change
|
|
10
|
+
* of this size reviewable.
|
|
11
|
+
*
|
|
12
|
+
* Two accommodations are needed, and both are deliberate:
|
|
13
|
+
*
|
|
14
|
+
* - **`acquireFile` and `prioritizeByteRange` stay synchronous.** They return
|
|
15
|
+
* nothing the caller inspects, so the command is dispatched and not awaited.
|
|
16
|
+
* `acquireFile` hands back a release function exactly as before, which sends
|
|
17
|
+
* its own command when called. Awaiting them would mean touching every call
|
|
18
|
+
* site for no observable gain.
|
|
19
|
+
* - **`getTorrent` needs a `sourceKey`.** Torrent objects cannot cross a
|
|
20
|
+
* thread, so the worker keys them. Callers that have one pass it; the rest
|
|
21
|
+
* get one derived from the source itself, so the identity stays stable
|
|
22
|
+
* across calls for the same torrent.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import crypto from "node:crypto";
|
|
26
|
+
import { TorrentWorkerClient } from "./client.js";
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Stable key for a source, matching how the worker keys its torrents.
|
|
30
|
+
*
|
|
31
|
+
* Derived from the source itself rather than handed out per request, so two
|
|
32
|
+
* routes asking for the same torrent name the same thing on the worker side.
|
|
33
|
+
*
|
|
34
|
+
* @param {"magnet" | "torrent"} sourceType
|
|
35
|
+
* @param {string} source
|
|
36
|
+
* @returns {string}
|
|
37
|
+
*/
|
|
38
|
+
function deriveSourceKey(sourceType, source) {
|
|
39
|
+
return `${sourceType}:${crypto.createHash("sha1").update(source).digest("hex")}`;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* A torrent pool whose work happens on another thread.
|
|
44
|
+
*
|
|
45
|
+
* See `protocol.js` for why: the torrent was taking ~85% of the main thread and
|
|
46
|
+
* everything owed to a viewer queued behind it.
|
|
47
|
+
*/
|
|
48
|
+
export class WorkerTorrentPool {
|
|
49
|
+
#client;
|
|
50
|
+
/** Stand-ins by source key, so repeat calls return the same object. */
|
|
51
|
+
#torrents = new Map();
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* @param {{ maxDiskBytes?: number }} [options]
|
|
55
|
+
*/
|
|
56
|
+
constructor(options = {}) {
|
|
57
|
+
this.#client = new TorrentWorkerClient(options);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Load (or join) a torrent and return a stand-in for it.
|
|
62
|
+
*
|
|
63
|
+
* @param {"magnet" | "torrent"} sourceType
|
|
64
|
+
* @param {string} source
|
|
65
|
+
* @returns {Promise<object>}
|
|
66
|
+
*/
|
|
67
|
+
async getTorrent(sourceType, source) {
|
|
68
|
+
const sourceKey = deriveSourceKey(sourceType, source);
|
|
69
|
+
const existing = this.#torrents.get(sourceKey);
|
|
70
|
+
if (existing) {
|
|
71
|
+
return existing;
|
|
72
|
+
}
|
|
73
|
+
const torrent = await this.#client.getTorrent({ sourceKey, sourceType, source });
|
|
74
|
+
this.#torrents.set(sourceKey, torrent);
|
|
75
|
+
return torrent;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Claim a file for reading; the returned function releases it.
|
|
80
|
+
*
|
|
81
|
+
* Synchronous by design — see the file header.
|
|
82
|
+
*
|
|
83
|
+
* @param {object} torrent - A stand-in from {@link getTorrent}.
|
|
84
|
+
* @param {number} fileIndex
|
|
85
|
+
* @returns {() => void}
|
|
86
|
+
*/
|
|
87
|
+
acquireFile(torrent, fileIndex) {
|
|
88
|
+
const sourceKey = torrent?.sourceKey;
|
|
89
|
+
if (!sourceKey) {
|
|
90
|
+
return () => undefined;
|
|
91
|
+
}
|
|
92
|
+
// Dispatched, not awaited — callers use the result immediately and inspect
|
|
93
|
+
// nothing. But the release MUST NOT overtake it: both are ordinary messages
|
|
94
|
+
// to the worker, and if release arrives first the reader count drops to zero
|
|
95
|
+
// while a read is still running. The idle sweep then removes the torrent AND
|
|
96
|
+
// its downloaded data out from under the encoder — field 2026-08-02:
|
|
97
|
+
// "removed idle torrent ... and its store" mid-playback, after which every
|
|
98
|
+
// read hung and ffmpeg saw an empty input ("Stream ends prematurely at 0").
|
|
99
|
+
// Chaining the release onto the acquire keeps them in order.
|
|
100
|
+
const acquired = this.#client.acquireFile(sourceKey, fileIndex).catch(() => undefined);
|
|
101
|
+
let released = false;
|
|
102
|
+
return () => {
|
|
103
|
+
if (released) {
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
released = true;
|
|
107
|
+
void acquired.then(() => this.#client.releaseFile(sourceKey, fileIndex)).catch(() => undefined);
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Live download figures for the progress display.
|
|
113
|
+
*
|
|
114
|
+
* @param {object} torrent
|
|
115
|
+
* @param {number | null} [fileIndex]
|
|
116
|
+
* @param {{ resumeAnchorByteStart?: number | null }} [options]
|
|
117
|
+
* @returns {Promise<object | null>}
|
|
118
|
+
*/
|
|
119
|
+
async getFileStats(torrent, fileIndex = null, options = {}) {
|
|
120
|
+
const sourceKey = torrent?.sourceKey;
|
|
121
|
+
if (!sourceKey) {
|
|
122
|
+
return null;
|
|
123
|
+
}
|
|
124
|
+
return this.#client.getFileStats({
|
|
125
|
+
sourceKey,
|
|
126
|
+
fileIndex,
|
|
127
|
+
resumeAnchorByteStart: options?.resumeAnchorByteStart ?? null
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Reorder piece selection around a read position.
|
|
133
|
+
*
|
|
134
|
+
* Synchronous by design — see the file header.
|
|
135
|
+
*
|
|
136
|
+
* @param {object} torrent
|
|
137
|
+
* @param {number} fileIndex
|
|
138
|
+
* @param {number} byteStart
|
|
139
|
+
* @param {number} [windowBytes]
|
|
140
|
+
* @returns {void}
|
|
141
|
+
*/
|
|
142
|
+
prioritizeByteRange(torrent, fileIndex, byteStart, windowBytes) {
|
|
143
|
+
const sourceKey = torrent?.sourceKey;
|
|
144
|
+
if (!sourceKey) {
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
void this.#client
|
|
148
|
+
.prioritizeByteRange({ sourceKey, fileIndex, byteStart, windowBytes })
|
|
149
|
+
.catch(() => undefined);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* Pre-fetch the head and tail the codec probe needs.
|
|
154
|
+
*
|
|
155
|
+
* @param {object} torrent
|
|
156
|
+
* @param {number} fileIndex
|
|
157
|
+
* @param {number} [headBytes]
|
|
158
|
+
* @param {number} [tailBytes]
|
|
159
|
+
* @param {number} [timeoutMs]
|
|
160
|
+
* @returns {Promise<unknown>}
|
|
161
|
+
*/
|
|
162
|
+
async prefetchFileEdges(torrent, fileIndex, headBytes, tailBytes, timeoutMs) {
|
|
163
|
+
const sourceKey = torrent?.sourceKey;
|
|
164
|
+
if (!sourceKey) {
|
|
165
|
+
return null;
|
|
166
|
+
}
|
|
167
|
+
return this.#client.prefetchFileEdges({ sourceKey, fileIndex, headBytes, tailBytes, timeoutMs });
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Shut the torrent client down and stop the thread.
|
|
172
|
+
*
|
|
173
|
+
* @returns {Promise<void>}
|
|
174
|
+
*/
|
|
175
|
+
async destroyAll() {
|
|
176
|
+
this.#torrents.clear();
|
|
177
|
+
await this.#client.destroyAll();
|
|
178
|
+
}
|
|
179
|
+
}
|
|
@@ -1,255 +1,265 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @file The torrent thread: WebTorrent and nothing else.
|
|
3
|
-
*
|
|
4
|
-
* Everything that made the main thread unresponsive lives here now — peer
|
|
5
|
-
* connections, buffer concatenation, piece bookkeeping, garbage collection from
|
|
6
|
-
* all of it. The main thread keeps only what owes a viewer a prompt answer.
|
|
7
|
-
*
|
|
8
|
-
* This file deliberately holds no HTTP, no session logic and no knowledge of
|
|
9
|
-
* HLS: it answers the commands in `protocol.js` and streams bytes back. That
|
|
10
|
-
* boundary is what keeps the split honest — anything added here will compete
|
|
11
|
-
* with the torrent for this thread, which is exactly the problem being solved.
|
|
12
|
-
*
|
|
13
|
-
* The existing `TorrentPool` is reused wholesale rather than reimplemented. It
|
|
14
|
-
* already carries the parts that took field failures to get right — refcounted
|
|
15
|
-
* file claims, idle removal, the global disk cap with LRU eviction, seek-aware
|
|
16
|
-
* piece prioritisation, adaptive upload — and none of that changes by moving
|
|
17
|
-
* threads.
|
|
18
|
-
*/
|
|
19
|
-
|
|
20
|
-
import { parentPort, workerData } from "node:worker_threads";
|
|
21
|
-
import { TorrentPool } from "../torrent-pool.js";
|
|
22
|
-
import { createSendStream } from "./channel.js";
|
|
23
|
-
import { Command, Event, STREAM_CHUNK_BYTES } from "./protocol.js";
|
|
24
|
-
|
|
25
|
-
const pool = new TorrentPool({ maxDiskBytes: workerData?.maxDiskBytes });
|
|
26
|
-
|
|
27
|
-
/** Torrents by sourceKey — the main thread names them, this thread owns them. */
|
|
28
|
-
const torrentsByKey = new Map();
|
|
29
|
-
/** File-claim release callbacks, keyed `${sourceKey}:${fileIndex}`. */
|
|
30
|
-
const releaseByClaim = new Map();
|
|
31
|
-
/** In-flight reads, so a cancel can stop one mid-body. */
|
|
32
|
-
const readsById = new Map();
|
|
33
|
-
|
|
34
|
-
/**
|
|
35
|
-
* Forward a log line to the main thread, so worker output is not lost or
|
|
36
|
-
* interleaved separately from everything else.
|
|
37
|
-
*
|
|
38
|
-
* @param {string} message
|
|
39
|
-
* @returns {void}
|
|
40
|
-
*/
|
|
41
|
-
function log(message) {
|
|
42
|
-
parentPort.postMessage({ type: Event.LOG, message });
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
/**
|
|
46
|
-
* The torrent for a sourceKey, or throw a message the caller can surface.
|
|
47
|
-
*
|
|
48
|
-
* @param {string} sourceKey
|
|
49
|
-
* @returns {import("webtorrent").Torrent}
|
|
50
|
-
*/
|
|
51
|
-
function requireTorrent(sourceKey) {
|
|
52
|
-
const torrent = torrentsByKey.get(sourceKey);
|
|
53
|
-
if (!torrent) {
|
|
54
|
-
throw new Error(`Unknown source ${sourceKey}.`);
|
|
55
|
-
}
|
|
56
|
-
return torrent;
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
/**
|
|
60
|
-
* Stream a byte range back as CHUNK messages.
|
|
61
|
-
*
|
|
62
|
-
* Reads through WebTorrent's own read stream — which serves already-downloaded
|
|
63
|
-
* pieces from disk and waits for the rest — and forwards it in
|
|
64
|
-
* {@link STREAM_CHUNK_BYTES} pieces, transferring ownership of each so nothing
|
|
65
|
-
* is copied across the boundary. `createSendStream` applies the backpressure,
|
|
66
|
-
* so a fast disk cannot outrun the main thread and rebuild the queue in memory.
|
|
67
|
-
*
|
|
68
|
-
* @param {object} params
|
|
69
|
-
* @param {number} params.id - Request id; CHUNK/READ_END carry it.
|
|
70
|
-
* @param {string} params.sourceKey
|
|
71
|
-
* @param {number} params.fileIndex
|
|
72
|
-
* @param {number | null} params.start - Inclusive, or null for the whole file.
|
|
73
|
-
* @param {number | null} params.end - Inclusive.
|
|
74
|
-
* @returns {Promise<void>}
|
|
75
|
-
*/
|
|
76
|
-
async function streamRange({ id, sourceKey, fileIndex, start, end }) {
|
|
77
|
-
const torrent = requireTorrent(sourceKey);
|
|
78
|
-
const file = torrent.files?.[fileIndex];
|
|
79
|
-
if (!file) {
|
|
80
|
-
throw new Error(`File ${fileIndex} not found in ${sourceKey}.`);
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
const sender = createSendStream({ port: parentPort, requestId: id });
|
|
84
|
-
readsById.set(id, sender);
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
//
|
|
90
|
-
//
|
|
91
|
-
//
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
const
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
}
|
|
164
|
-
}
|
|
165
|
-
|
|
166
|
-
case Command.
|
|
167
|
-
const torrent = requireTorrent(params.sourceKey);
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
const claimKey = `${params.sourceKey}:${params.fileIndex}`;
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
}
|
|
184
|
-
return true;
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
case Command.
|
|
188
|
-
const
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
return true;
|
|
231
|
-
}
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
}
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
1
|
+
/**
|
|
2
|
+
* @file The torrent thread: WebTorrent and nothing else.
|
|
3
|
+
*
|
|
4
|
+
* Everything that made the main thread unresponsive lives here now — peer
|
|
5
|
+
* connections, buffer concatenation, piece bookkeeping, garbage collection from
|
|
6
|
+
* all of it. The main thread keeps only what owes a viewer a prompt answer.
|
|
7
|
+
*
|
|
8
|
+
* This file deliberately holds no HTTP, no session logic and no knowledge of
|
|
9
|
+
* HLS: it answers the commands in `protocol.js` and streams bytes back. That
|
|
10
|
+
* boundary is what keeps the split honest — anything added here will compete
|
|
11
|
+
* with the torrent for this thread, which is exactly the problem being solved.
|
|
12
|
+
*
|
|
13
|
+
* The existing `TorrentPool` is reused wholesale rather than reimplemented. It
|
|
14
|
+
* already carries the parts that took field failures to get right — refcounted
|
|
15
|
+
* file claims, idle removal, the global disk cap with LRU eviction, seek-aware
|
|
16
|
+
* piece prioritisation, adaptive upload — and none of that changes by moving
|
|
17
|
+
* threads.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { parentPort, workerData } from "node:worker_threads";
|
|
21
|
+
import { TorrentPool } from "../torrent-pool.js";
|
|
22
|
+
import { createSendStream } from "./channel.js";
|
|
23
|
+
import { Command, Event, STREAM_CHUNK_BYTES } from "./protocol.js";
|
|
24
|
+
|
|
25
|
+
const pool = new TorrentPool({ maxDiskBytes: workerData?.maxDiskBytes });
|
|
26
|
+
|
|
27
|
+
/** Torrents by sourceKey — the main thread names them, this thread owns them. */
|
|
28
|
+
const torrentsByKey = new Map();
|
|
29
|
+
/** File-claim release callbacks, keyed `${sourceKey}:${fileIndex}`. */
|
|
30
|
+
const releaseByClaim = new Map();
|
|
31
|
+
/** In-flight reads, so a cancel can stop one mid-body. */
|
|
32
|
+
const readsById = new Map();
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Forward a log line to the main thread, so worker output is not lost or
|
|
36
|
+
* interleaved separately from everything else.
|
|
37
|
+
*
|
|
38
|
+
* @param {string} message
|
|
39
|
+
* @returns {void}
|
|
40
|
+
*/
|
|
41
|
+
function log(message) {
|
|
42
|
+
parentPort.postMessage({ type: Event.LOG, message });
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* The torrent for a sourceKey, or throw a message the caller can surface.
|
|
47
|
+
*
|
|
48
|
+
* @param {string} sourceKey
|
|
49
|
+
* @returns {import("webtorrent").Torrent}
|
|
50
|
+
*/
|
|
51
|
+
function requireTorrent(sourceKey) {
|
|
52
|
+
const torrent = torrentsByKey.get(sourceKey);
|
|
53
|
+
if (!torrent) {
|
|
54
|
+
throw new Error(`Unknown source ${sourceKey}.`);
|
|
55
|
+
}
|
|
56
|
+
return torrent;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Stream a byte range back as CHUNK messages.
|
|
61
|
+
*
|
|
62
|
+
* Reads through WebTorrent's own read stream — which serves already-downloaded
|
|
63
|
+
* pieces from disk and waits for the rest — and forwards it in
|
|
64
|
+
* {@link STREAM_CHUNK_BYTES} pieces, transferring ownership of each so nothing
|
|
65
|
+
* is copied across the boundary. `createSendStream` applies the backpressure,
|
|
66
|
+
* so a fast disk cannot outrun the main thread and rebuild the queue in memory.
|
|
67
|
+
*
|
|
68
|
+
* @param {object} params
|
|
69
|
+
* @param {number} params.id - Request id; CHUNK/READ_END carry it.
|
|
70
|
+
* @param {string} params.sourceKey
|
|
71
|
+
* @param {number} params.fileIndex
|
|
72
|
+
* @param {number | null} params.start - Inclusive, or null for the whole file.
|
|
73
|
+
* @param {number | null} params.end - Inclusive.
|
|
74
|
+
* @returns {Promise<void>}
|
|
75
|
+
*/
|
|
76
|
+
async function streamRange({ id, sourceKey, fileIndex, start, end }) {
|
|
77
|
+
const torrent = requireTorrent(sourceKey);
|
|
78
|
+
const file = torrent.files?.[fileIndex];
|
|
79
|
+
if (!file) {
|
|
80
|
+
throw new Error(`File ${fileIndex} not found in ${sourceKey}.`);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const sender = createSendStream({ port: parentPort, requestId: id });
|
|
84
|
+
readsById.set(id, sender);
|
|
85
|
+
|
|
86
|
+
// Hold the file for as long as this read runs. The caller also acquires it,
|
|
87
|
+
// but that acquire and its release are separate messages from another thread
|
|
88
|
+
// and can be reordered; this one cannot, because it lives entirely inside the
|
|
89
|
+
// read. Without it the idle sweep saw a zero reader count and removed the
|
|
90
|
+
// torrent AND its store mid-read — field 2026-08-02: "removed idle torrent
|
|
91
|
+
// ... and its store", after which every subsequent read hung and ffmpeg got
|
|
92
|
+
// an empty input.
|
|
93
|
+
const releaseRead = pool.acquireFile(torrent, fileIndex);
|
|
94
|
+
|
|
95
|
+
const options = start === null || start === undefined ? {} : { start, end };
|
|
96
|
+
const source = file.createReadStream(options);
|
|
97
|
+
|
|
98
|
+
// Coalesce WebTorrent's own chunking (piece-sized, often much smaller) up to
|
|
99
|
+
// our chunk size: a round trip costs ~100 µs, so sending its native pieces
|
|
100
|
+
// straight through would multiply the crossings for no benefit.
|
|
101
|
+
let pendingParts = [];
|
|
102
|
+
let pendingBytes = 0;
|
|
103
|
+
|
|
104
|
+
const flush = async () => {
|
|
105
|
+
if (pendingBytes === 0) {
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
const merged = pendingParts.length === 1 ? pendingParts[0] : Buffer.concat(pendingParts, pendingBytes);
|
|
109
|
+
pendingParts = [];
|
|
110
|
+
pendingBytes = 0;
|
|
111
|
+
await sender.send(merged);
|
|
112
|
+
};
|
|
113
|
+
|
|
114
|
+
try {
|
|
115
|
+
for await (const part of source) {
|
|
116
|
+
if (sender.isCancelled()) {
|
|
117
|
+
break;
|
|
118
|
+
}
|
|
119
|
+
pendingParts.push(part);
|
|
120
|
+
pendingBytes += part.length;
|
|
121
|
+
if (pendingBytes >= STREAM_CHUNK_BYTES) {
|
|
122
|
+
await flush();
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
if (!sender.isCancelled()) {
|
|
126
|
+
await flush();
|
|
127
|
+
}
|
|
128
|
+
} finally {
|
|
129
|
+
readsById.delete(id);
|
|
130
|
+
releaseRead();
|
|
131
|
+
sender.end();
|
|
132
|
+
// A cancelled read must stop the underlying torrent stream too, or the
|
|
133
|
+
// pieces keep being fetched for a viewer who has gone.
|
|
134
|
+
if (typeof source.destroy === "function") {
|
|
135
|
+
source.destroy();
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Run one command and return its result.
|
|
142
|
+
*
|
|
143
|
+
* @param {string} command
|
|
144
|
+
* @param {object} params
|
|
145
|
+
* @param {number} id
|
|
146
|
+
* @returns {Promise<unknown>}
|
|
147
|
+
*/
|
|
148
|
+
async function runCommand(command, params, id) {
|
|
149
|
+
switch (command) {
|
|
150
|
+
case Command.ADD_SOURCE: {
|
|
151
|
+
const torrent = await pool.getTorrent(params.sourceType, params.source);
|
|
152
|
+
torrentsByKey.set(params.sourceKey, torrent);
|
|
153
|
+
return {
|
|
154
|
+
infoHash: torrent.infoHash,
|
|
155
|
+
name: torrent.name,
|
|
156
|
+
// Files cross as plain data; the objects stay here.
|
|
157
|
+
files: (torrent.files ?? []).map((file, index) => ({
|
|
158
|
+
index,
|
|
159
|
+
name: file.name,
|
|
160
|
+
path: file.path,
|
|
161
|
+
length: file.length
|
|
162
|
+
}))
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
case Command.LIST_FILES: {
|
|
167
|
+
const torrent = requireTorrent(params.sourceKey);
|
|
168
|
+
return (torrent.files ?? []).map((file, index) => ({
|
|
169
|
+
index,
|
|
170
|
+
name: file.name,
|
|
171
|
+
path: file.path,
|
|
172
|
+
length: file.length
|
|
173
|
+
}));
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
case Command.ACQUIRE_FILE: {
|
|
177
|
+
const torrent = requireTorrent(params.sourceKey);
|
|
178
|
+
const claimKey = `${params.sourceKey}:${params.fileIndex}`;
|
|
179
|
+
// One claim per key; a second acquire without release would leak the
|
|
180
|
+
// first release callback and pin the file forever.
|
|
181
|
+
if (!releaseByClaim.has(claimKey)) {
|
|
182
|
+
releaseByClaim.set(claimKey, pool.acquireFile(torrent, params.fileIndex));
|
|
183
|
+
}
|
|
184
|
+
return true;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
case Command.RELEASE_FILE: {
|
|
188
|
+
const claimKey = `${params.sourceKey}:${params.fileIndex}`;
|
|
189
|
+
const release = releaseByClaim.get(claimKey);
|
|
190
|
+
if (release) {
|
|
191
|
+
releaseByClaim.delete(claimKey);
|
|
192
|
+
release();
|
|
193
|
+
}
|
|
194
|
+
return true;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
case Command.FILE_STATS: {
|
|
198
|
+
const torrent = requireTorrent(params.sourceKey);
|
|
199
|
+
return pool.getFileStats(torrent, params.fileIndex, {
|
|
200
|
+
resumeAnchorByteStart: params.resumeAnchorByteStart ?? null
|
|
201
|
+
});
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
case Command.PRIORITIZE: {
|
|
205
|
+
const torrent = requireTorrent(params.sourceKey);
|
|
206
|
+
pool.prioritizeByteRange(torrent, params.fileIndex, params.byteStart, params.windowBytes);
|
|
207
|
+
return true;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
case Command.PREFETCH_EDGES: {
|
|
211
|
+
const torrent = requireTorrent(params.sourceKey);
|
|
212
|
+
return pool.prefetchFileEdges(torrent, params.fileIndex, params.headBytes, params.tailBytes, params.timeoutMs);
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
case Command.READ_RANGE: {
|
|
216
|
+
// Streams its own reply; the caller's promise resolves once the body has
|
|
217
|
+
// been fully sent, which is what lets the client await completion.
|
|
218
|
+
await streamRange({
|
|
219
|
+
id,
|
|
220
|
+
sourceKey: params.sourceKey,
|
|
221
|
+
fileIndex: params.fileIndex,
|
|
222
|
+
start: params.start ?? null,
|
|
223
|
+
end: params.end ?? null
|
|
224
|
+
});
|
|
225
|
+
return true;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
case Command.CANCEL_READ: {
|
|
229
|
+
readsById.get(params.readId)?.cancel();
|
|
230
|
+
return true;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
case Command.DESTROY_ALL: {
|
|
234
|
+
for (const [, release] of releaseByClaim) {
|
|
235
|
+
release();
|
|
236
|
+
}
|
|
237
|
+
releaseByClaim.clear();
|
|
238
|
+
torrentsByKey.clear();
|
|
239
|
+
await pool.destroyAll();
|
|
240
|
+
return true;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
default:
|
|
244
|
+
throw new Error(`Unknown torrent-worker command: ${command}`);
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
parentPort.on("message", async (message) => {
|
|
249
|
+
// Chunk acknowledgements are not commands — they release backpressure on an
|
|
250
|
+
// in-flight read.
|
|
251
|
+
if (message?.type === Event.CHUNK_ACK) {
|
|
252
|
+
readsById.get(message.id)?.ack();
|
|
253
|
+
return;
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
const { command, id, params } = message ?? {};
|
|
257
|
+
try {
|
|
258
|
+
const result = await runCommand(command, params ?? {}, id);
|
|
259
|
+
parentPort.postMessage({ type: Event.RESULT, id, result });
|
|
260
|
+
} catch (error) {
|
|
261
|
+
parentPort.postMessage({ type: Event.ERROR, id, error: error?.message ?? String(error) });
|
|
262
|
+
}
|
|
263
|
+
});
|
|
264
|
+
|
|
265
|
+
log("torrent worker started");
|