@torrent-tv/proxy 2.66.0 → 2.67.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1322 -1310
- package/package.json +1 -1
- package/routes/api/sources/warm/post.js +13 -0
- package/server.js +7 -9
- package/services/hls-session-manager.js +40 -0
- package/services/supply-margin.js +74 -10
- package/services/torrent-worker/background-fill.js +191 -0
- package/services/torrent-worker/client.js +10 -0
- package/services/torrent-worker/piece-reader.js +79 -3
- package/services/torrent-worker/pool-adapter.js +20 -0
- package/services/torrent-worker/protocol.js +6 -0
- package/services/torrent-worker/worker.js +792 -784
- package/test/background-fill.test.js +156 -0
- package/test/supply-margin.test.js +59 -11
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fetching a file nobody is playing yet, without taking anything from the
|
|
3
|
+
* viewer.
|
|
4
|
+
*
|
|
5
|
+
* The ordering these checks pin is the whole point of the thing: what plays now
|
|
6
|
+
* comes first, the other soundtracks and subtitle files next, and reading the
|
|
7
|
+
* film far ahead last. The middle tier stays below the first by standing aside
|
|
8
|
+
* whenever a reader on the torrent is inside a wait.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import assert from "node:assert/strict";
|
|
12
|
+
import test from "node:test";
|
|
13
|
+
import { EventEmitter } from "node:events";
|
|
14
|
+
import { fillFileInBackground, fillIsRunning } from "../services/torrent-worker/background-fill.js";
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* A file whose reads resolve when the test says so, recording what was asked
|
|
18
|
+
* for.
|
|
19
|
+
*
|
|
20
|
+
* @param {{ length: number, name?: string }} params
|
|
21
|
+
*/
|
|
22
|
+
function fakeFile({ length, name = "dub.mka" }) {
|
|
23
|
+
const reads = [];
|
|
24
|
+
return {
|
|
25
|
+
name,
|
|
26
|
+
length,
|
|
27
|
+
reads,
|
|
28
|
+
createReadStream({ start, end }) {
|
|
29
|
+
const stream = new EventEmitter();
|
|
30
|
+
stream.destroy = () => {};
|
|
31
|
+
reads.push({ start, end });
|
|
32
|
+
// Deliver on the next turn, so a test can observe the read in flight.
|
|
33
|
+
queueMicrotask(() => {
|
|
34
|
+
stream.emit("data", Buffer.alloc(end - start + 1));
|
|
35
|
+
stream.emit("end");
|
|
36
|
+
});
|
|
37
|
+
return stream;
|
|
38
|
+
}
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* @param {() => boolean} until
|
|
44
|
+
* @param {number} [limit]
|
|
45
|
+
*/
|
|
46
|
+
async function waitFor(until, limit = 2000) {
|
|
47
|
+
const deadline = Date.now() + limit;
|
|
48
|
+
while (!until()) {
|
|
49
|
+
if (Date.now() > deadline) {
|
|
50
|
+
throw new Error("condition was never reached");
|
|
51
|
+
}
|
|
52
|
+
await new Promise((resolve) => setTimeout(resolve, 5));
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
test("a file is walked whole, a piece at a time", async () => {
|
|
57
|
+
const file = fakeFile({ length: 10 });
|
|
58
|
+
const torrent = { infoHash: "aaa", pieceLength: 4, files: [file] };
|
|
59
|
+
|
|
60
|
+
const started = fillFileInBackground(torrent, 0, "source-a", { chunkBytes: 4 });
|
|
61
|
+
|
|
62
|
+
assert.equal(started, true);
|
|
63
|
+
await waitFor(() => !fillIsRunning("source-a", 0));
|
|
64
|
+
// 0-3, 4-7, 8-9: the last chunk is short because the file ends, and its end
|
|
65
|
+
// is the last byte rather than one past it.
|
|
66
|
+
assert.deepEqual(file.reads, [
|
|
67
|
+
{ start: 0, end: 3 },
|
|
68
|
+
{ start: 4, end: 7 },
|
|
69
|
+
{ start: 8, end: 9 }
|
|
70
|
+
]);
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
test("one fill per file, however many times it is asked for", async () => {
|
|
74
|
+
const file = fakeFile({ length: 8 });
|
|
75
|
+
const torrent = { infoHash: "bbb", pieceLength: 4, files: [file] };
|
|
76
|
+
|
|
77
|
+
const first = fillFileInBackground(torrent, 0, "source-b", { chunkBytes: 4 });
|
|
78
|
+
const second = fillFileInBackground(torrent, 0, "source-b", { chunkBytes: 4 });
|
|
79
|
+
|
|
80
|
+
assert.equal(first, true);
|
|
81
|
+
assert.equal(second, false, "the second call does not start a second walk");
|
|
82
|
+
await waitFor(() => !fillIsRunning("source-b", 0));
|
|
83
|
+
assert.equal(file.reads.length, 2, "the file was walked once");
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
test("a file that is gone is not read, and does not throw", async () => {
|
|
87
|
+
const torrent = { infoHash: "ccc", pieceLength: 4, files: [] };
|
|
88
|
+
|
|
89
|
+
const started = fillFileInBackground(torrent, 0, "source-c", { chunkBytes: 4 });
|
|
90
|
+
|
|
91
|
+
assert.equal(started, false);
|
|
92
|
+
assert.equal(fillIsRunning("source-c", 0), false);
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
test("a read that returns nothing stops the walk instead of spinning", async () => {
|
|
96
|
+
const file = {
|
|
97
|
+
name: "dub.mka",
|
|
98
|
+
length: 100,
|
|
99
|
+
reads: [],
|
|
100
|
+
createReadStream({ start, end }) {
|
|
101
|
+
const stream = new EventEmitter();
|
|
102
|
+
stream.destroy = () => {};
|
|
103
|
+
this.reads.push({ start, end });
|
|
104
|
+
queueMicrotask(() => stream.emit("error", new Error("gone")));
|
|
105
|
+
return stream;
|
|
106
|
+
}
|
|
107
|
+
};
|
|
108
|
+
const torrent = { infoHash: "ddd", pieceLength: 10, files: [file] };
|
|
109
|
+
|
|
110
|
+
fillFileInBackground(torrent, 0, "source-d", { chunkBytes: 10 });
|
|
111
|
+
|
|
112
|
+
await waitFor(() => !fillIsRunning("source-d", 0));
|
|
113
|
+
assert.equal(file.reads.length, 1, "it gave up after the first failed read");
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
test("it stands aside while the viewer's own reading is blocked", async () => {
|
|
117
|
+
const file = fakeFile({ length: 8 });
|
|
118
|
+
const torrent = { infoHash: "eee", pieceLength: 4, files: [file] };
|
|
119
|
+
let blocked = true;
|
|
120
|
+
|
|
121
|
+
fillFileInBackground(torrent, 0, "source-e", { chunkBytes: 4, isBlocked: () => blocked });
|
|
122
|
+
|
|
123
|
+
// The picture is starving, so nothing is asked of the swarm on the
|
|
124
|
+
// soundtrack's behalf — however long that lasts.
|
|
125
|
+
await new Promise((resolve) => setTimeout(resolve, 120));
|
|
126
|
+
assert.deepEqual(file.reads, [], "not one read while a reader is inside a wait");
|
|
127
|
+
|
|
128
|
+
blocked = false;
|
|
129
|
+
await waitFor(() => !fillIsRunning("source-e", 0));
|
|
130
|
+
assert.equal(file.reads.length, 2, "and it resumes once the room is there");
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
test("the gate is re-asked before every chunk, not once at the start", async () => {
|
|
134
|
+
const file = fakeFile({ length: 12 });
|
|
135
|
+
const torrent = { infoHash: "fff", pieceLength: 4, files: [file] };
|
|
136
|
+
// Starve the viewer again the moment the first chunk has been read, and stay
|
|
137
|
+
// that way until the test lifts it.
|
|
138
|
+
let starvedAfterFirstChunk = false;
|
|
139
|
+
let lifted = false;
|
|
140
|
+
const isBlocked = () => {
|
|
141
|
+
if (file.reads.length >= 1 && !lifted) {
|
|
142
|
+
starvedAfterFirstChunk = true;
|
|
143
|
+
}
|
|
144
|
+
return starvedAfterFirstChunk && !lifted;
|
|
145
|
+
};
|
|
146
|
+
|
|
147
|
+
fillFileInBackground(torrent, 0, "source-f", { chunkBytes: 4, isBlocked });
|
|
148
|
+
|
|
149
|
+
await waitFor(() => file.reads.length === 1);
|
|
150
|
+
await new Promise((resolve) => setTimeout(resolve, 120));
|
|
151
|
+
assert.equal(file.reads.length, 1, "a torrent healthy a moment ago is not evidence about now");
|
|
152
|
+
|
|
153
|
+
lifted = true;
|
|
154
|
+
await waitFor(() => !fillIsRunning("source-f", 0));
|
|
155
|
+
assert.equal(file.reads.length, 3);
|
|
156
|
+
});
|
|
@@ -29,18 +29,65 @@ function evenlySpaced(count, intervalSec, waitSec) {
|
|
|
29
29
|
}
|
|
30
30
|
|
|
31
31
|
test("the margin is what the supply's own interruptions demand", () => {
|
|
32
|
-
// The field torrent:
|
|
33
|
-
|
|
34
|
-
|
|
32
|
+
// The field torrent: waits of 1.49 s with 0.73 s of running between them, and
|
|
33
|
+
// the worst of them 3.16 s. Written out rather than generated, because the
|
|
34
|
+
// point of the test is that these exact spans give that exact answer.
|
|
35
|
+
const waits = [
|
|
36
|
+
{ waitedMs: 1490, at: 1490 }, // 0.00..1.49
|
|
37
|
+
{ waitedMs: 1490, at: 3710 }, // 2.22..3.71
|
|
38
|
+
{ waitedMs: 1490, at: 5930 }, // 4.44..5.93
|
|
39
|
+
{ waitedMs: 3160, at: 9820 }, // 6.66..9.82 ← the worst
|
|
40
|
+
{ waitedMs: 1490, at: 12_040 }, // 10.55..12.04
|
|
41
|
+
{ waitedMs: 1490, at: 14_260 } // 12.77..14.26
|
|
42
|
+
];
|
|
35
43
|
|
|
36
44
|
const answer = requiredSpeedFrom(waits);
|
|
37
45
|
|
|
38
46
|
assert.ok(answer);
|
|
39
47
|
assert.equal(answer.worstWaitSec, 3.16);
|
|
40
|
-
|
|
41
|
-
//
|
|
42
|
-
//
|
|
43
|
-
|
|
48
|
+
// 2.22 s from one wait's end to the next one's END, of which 1.49 s was spent
|
|
49
|
+
// waiting — so the encoder ran for 0.73 s. That running stretch is what the
|
|
50
|
+
// derivation calls T: `(v - 1) x T > W` prices what is GAINED between
|
|
51
|
+
// interruptions, and nothing is gained during one.
|
|
52
|
+
assert.ok(Math.abs(answer.medianIntervalSec - 0.73) < 0.001, `got ${answer.medianIntervalSec}`);
|
|
53
|
+
// 1 + 3.16 / 0.73 = 5.33. Measuring end-to-end instead gave 2.42, and the
|
|
54
|
+
// symptom that this whole file was written against is that a step admitted at
|
|
55
|
+
// 1.5 ran at 1.05x and stalled — so the bar was too low, not too high.
|
|
56
|
+
assert.ok(Math.abs(answer.requiredSpeed - 5.3288) < 0.001, `got ${answer.requiredSpeed}`);
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
test("one stall seen by three readers is one interruption, not three", () => {
|
|
60
|
+
// The picture and two audio renditions walk the same file, so a piece that
|
|
61
|
+
// has not arrived blocks all three, and their waits end within milliseconds
|
|
62
|
+
// of each other. Field 2026-08-31: `worst wait 13.26s, one every 0.00s,
|
|
63
|
+
// 2 measured` produced a required speed of 4422.00x, and every quality step
|
|
64
|
+
// was refused against it.
|
|
65
|
+
const answer = requiredSpeedFrom([
|
|
66
|
+
{ waitedMs: 13_260, at: 1_000_000 },
|
|
67
|
+
{ waitedMs: 13_257, at: 1_000_003 }
|
|
68
|
+
]);
|
|
69
|
+
|
|
70
|
+
// Two overlapping waits are one interruption, and one interruption shows no
|
|
71
|
+
// interval — so the honest answer is that it is not known yet.
|
|
72
|
+
assert.equal(answer, null);
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
test("overlapping waits merge, and the gap between stalls is what is left", () => {
|
|
76
|
+
const answer = requiredSpeedFrom([
|
|
77
|
+
// First stall: 10s..20s, noticed by two readers a moment apart.
|
|
78
|
+
{ waitedMs: 10_000, at: 20_000 },
|
|
79
|
+
{ waitedMs: 9_000, at: 19_500 },
|
|
80
|
+
// Second stall: 30s..34s, again seen twice.
|
|
81
|
+
{ waitedMs: 4_000, at: 34_000 },
|
|
82
|
+
{ waitedMs: 3_500, at: 33_800 }
|
|
83
|
+
]);
|
|
84
|
+
|
|
85
|
+
assert.ok(answer);
|
|
86
|
+
assert.equal(answer.samples, 2, "two interruptions");
|
|
87
|
+
assert.equal(answer.waits, 4, "from four waits");
|
|
88
|
+
assert.equal(answer.worstWaitSec, 10, "the merged stall, not one reader's view of it");
|
|
89
|
+
assert.equal(answer.medianIntervalSec, 10, "20s to 30s is when the encoder ran");
|
|
90
|
+
assert.equal(answer.requiredSpeed, 2, "1 + 10/10");
|
|
44
91
|
});
|
|
45
92
|
|
|
46
93
|
test("a copy at 8x clears its own supply with room to spare", () => {
|
|
@@ -48,9 +95,10 @@ test("a copy at 8x clears its own supply with room to spare", () => {
|
|
|
48
95
|
const waits = evenlySpaced(6, 15.5, 4.82);
|
|
49
96
|
const answer = requiredSpeedFrom(waits);
|
|
50
97
|
assert.ok(answer);
|
|
51
|
-
//
|
|
52
|
-
//
|
|
53
|
-
|
|
98
|
+
// Waits end 15.5 s apart and last 4.82 s, so the encoder runs 10.68 s between
|
|
99
|
+
// them: 1 + 4.82/10.68 = 1.45, against 8x measured. Which is why a copy is the
|
|
100
|
+
// step a stranded viewer is always able to return to.
|
|
101
|
+
assert.ok(answer.requiredSpeed < 1.5, `got ${answer.requiredSpeed}`);
|
|
54
102
|
});
|
|
55
103
|
|
|
56
104
|
test("with too little evidence it says so instead of inventing a number", () => {
|
|
@@ -77,7 +125,7 @@ test("readings that are not measurements are ignored, not averaged in", () => {
|
|
|
77
125
|
{ waitedMs: 1500, at: 7000 }
|
|
78
126
|
]);
|
|
79
127
|
assert.ok(answer);
|
|
80
|
-
assert.equal(answer.
|
|
128
|
+
assert.equal(answer.waits, 3, "only the three real waits count");
|
|
81
129
|
});
|
|
82
130
|
|
|
83
131
|
test("the buffer is one segment plus the worst interruption, and names which", () => {
|