@torrent-tv/proxy 2.71.1 → 2.72.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/bin/cli.js +8 -0
- package/package.json +1 -1
- package/server.js +4 -0
- package/services/container/Container.js +35 -0
- package/services/container/MatroskaContainer.js +42 -0
- package/services/container/Mp4Container.js +22 -1
- package/services/container/SubtitleFileContainer.js +261 -0
- package/services/container/index.js +1 -0
- package/services/container-index/matroska-subtitles.js +6 -1
- package/services/controllers/SubtitleController.js +6 -25
- package/services/health-collector.js +37 -3
- package/services/hls-session-manager.js +10584 -10243
- package/services/piece-store/piece-lru.js +43 -0
- package/services/piece-store/shared-piece-store.js +214 -11
- package/services/playback-planner.js +7 -0
- package/services/subtitle-convert.js +74 -80
- package/services/torrent-worker/subtitle-cues.js +18 -53
- package/services/tracks/TextSubtitleTrack.js +18 -0
- package/services/tracks/index.js +1 -0
- package/services/tracks/subtitle-markup.js +104 -0
- package/services/tunnel-client.js +27 -0
- package/test/health-metrics.test.js +37 -0
- package/test/piece-store-eviction.test.js +98 -10
- package/test/piece-store-slow-disk.test.js +114 -0
- package/test/produced-copy-choice.test.js +361 -0
- package/test/subtitle-cue-framing.test.js +202 -0
- package/test/subtitle-language.test.js +19 -8
|
@@ -18,8 +18,11 @@
|
|
|
18
18
|
*/
|
|
19
19
|
|
|
20
20
|
import { readSubtitlePlan, harvestCluster } from "../container-index/matroska-subtitles.js";
|
|
21
|
-
import {
|
|
21
|
+
import { readMp4SubtitlePlan } from "../container-index/mp4-subtitles.js";
|
|
22
22
|
import { iterateElements } from "../container-index/ebml-reader.js";
|
|
23
|
+
import { MatroskaContainer } from "../container/MatroskaContainer.js";
|
|
24
|
+
import { Mp4Container } from "../container/Mp4Container.js";
|
|
25
|
+
import { finalizeCues } from "../subtitle-convert.js";
|
|
23
26
|
import { detectLanguage } from "../language-detect.js";
|
|
24
27
|
import { logger } from "../../utils/logger.js";
|
|
25
28
|
|
|
@@ -358,7 +361,8 @@ async function walkFor(torrent, fileIndex, state, plan, track, trackNumber) {
|
|
|
358
361
|
continue;
|
|
359
362
|
}
|
|
360
363
|
harvested.add(sample.offset);
|
|
361
|
-
|
|
364
|
+
// The MP4 has framed this cue and is the one that unframes it.
|
|
365
|
+
const text = Mp4Container.cueTextOf(bytes, track.codecId);
|
|
362
366
|
if (text) {
|
|
363
367
|
cues.push({
|
|
364
368
|
startSeconds: sample.startSeconds,
|
|
@@ -424,9 +428,18 @@ async function walkFor(torrent, fileIndex, state, plan, track, trackNumber) {
|
|
|
424
428
|
state.cues.set(candidate.trackNumber, into);
|
|
425
429
|
}
|
|
426
430
|
let found = false;
|
|
427
|
-
for (const
|
|
428
|
-
|
|
429
|
-
|
|
431
|
+
for (const block of harvestCluster(bytes, candidate.trackNumber, plan.secondsPerTick)) {
|
|
432
|
+
// The block's bytes become this track's text HERE, where the container
|
|
433
|
+
// that framed them is known. A cue kept in its framed form and unframed
|
|
434
|
+
// later cannot be unframed at all: nothing downstream knows which
|
|
435
|
+
// container it came out of, and guessing from the field count is what
|
|
436
|
+
// showed the dialogue row's own fields to the viewer.
|
|
437
|
+
into.push({
|
|
438
|
+
startSeconds: block.startSeconds,
|
|
439
|
+
endSeconds: block.endSeconds,
|
|
440
|
+
text: MatroskaContainer.cueTextOf(block.payload, candidate.codecId),
|
|
441
|
+
seq: nextSeq(state, candidate.trackNumber)
|
|
442
|
+
});
|
|
430
443
|
found = true;
|
|
431
444
|
}
|
|
432
445
|
if (found) {
|
|
@@ -515,54 +528,6 @@ export async function warmSubtitleCues(torrent, fileIndex, sourceKey) {
|
|
|
515
528
|
return fresh;
|
|
516
529
|
}
|
|
517
530
|
|
|
518
|
-
/**
|
|
519
|
-
* Resolve a cue's end time and strip codec-specific formatting, turning a raw
|
|
520
|
-
* block-derived cue into text a player can show directly. The same step
|
|
521
|
-
* `routes/api/subtitles/get.js` applies when building WebVTT for a pull —
|
|
522
|
-
* factored out here so a pushed cue and a pulled one read identically.
|
|
523
|
-
*
|
|
524
|
-
* A cue with no duration — a SimpleBlock, which subtitles rarely use — is
|
|
525
|
-
* given the time until the next one IN THIS LIST, and the last such cue a few
|
|
526
|
-
* seconds. Not an invention about the film: it is what a player does with an
|
|
527
|
-
* open-ended cue, made explicit so every consumer agrees on it.
|
|
528
|
-
*
|
|
529
|
-
* @param {{ startSeconds: number, endSeconds: number | null, text: string }[]} cues
|
|
530
|
-
* @param {string} codecId
|
|
531
|
-
* @returns {{ startSeconds: number, endSeconds: number, text: string }[]}
|
|
532
|
-
*/
|
|
533
|
-
export function finalizeCues(cues, codecId) {
|
|
534
|
-
const isAss = codecId === "S_TEXT/ASS" || codecId === "S_TEXT/SSA";
|
|
535
|
-
const result = [];
|
|
536
|
-
cues.forEach((cue, index) => {
|
|
537
|
-
const next = cues[index + 1];
|
|
538
|
-
const endSeconds = cue.endSeconds ?? (next ? next.startSeconds : cue.startSeconds + 4);
|
|
539
|
-
const text = isAss ? assDialogueToText(cue.text) : cue.text.trim();
|
|
540
|
-
if (!text) {
|
|
541
|
-
return;
|
|
542
|
-
}
|
|
543
|
-
result.push({ startSeconds: cue.startSeconds, endSeconds, text });
|
|
544
|
-
});
|
|
545
|
-
return result;
|
|
546
|
-
}
|
|
547
|
-
|
|
548
|
-
/**
|
|
549
|
-
* The visible text of an ASS dialogue row.
|
|
550
|
-
*
|
|
551
|
-
* A block carries the fields after `Dialogue:` without their header — nine of
|
|
552
|
-
* them, then the text, which itself holds override groups in braces.
|
|
553
|
-
*
|
|
554
|
-
* @param {string} raw
|
|
555
|
-
* @returns {string}
|
|
556
|
-
*/
|
|
557
|
-
function assDialogueToText(raw) {
|
|
558
|
-
const fields = raw.split(",");
|
|
559
|
-
const text = fields.length > 9 ? fields.slice(9).join(",") : raw;
|
|
560
|
-
return text
|
|
561
|
-
.replace(/\{[^}]*\}/g, "")
|
|
562
|
-
.replace(/\\N/gi, "\n")
|
|
563
|
-
.trim();
|
|
564
|
-
}
|
|
565
|
-
|
|
566
531
|
/**
|
|
567
532
|
* The text subtitle tracks of a file, for the menu the viewer sees.
|
|
568
533
|
*
|
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
import { SubtitleTrack } from "./SubtitleTrack.js";
|
|
10
|
+
import { markupKindOf, plainCueText } from "./subtitle-markup.js";
|
|
10
11
|
|
|
11
12
|
const TEXT_CODECS_MATROSKA = new Set(["S_TEXT/UTF8", "S_TEXT/ASS", "S_TEXT/SSA", "S_TEXT/WEBVTT"]);
|
|
12
13
|
const TEXT_FORMATS_MP4 = new Set(["tx3g", "text", "wvtt"]);
|
|
@@ -21,6 +22,23 @@ export class TextSubtitleTrack extends SubtitleTrack {
|
|
|
21
22
|
return true;
|
|
22
23
|
}
|
|
23
24
|
|
|
25
|
+
/** Which markup this track's cue text carries — see `subtitle-markup.js`. */
|
|
26
|
+
get markupKind() {
|
|
27
|
+
return markupKindOf(this.textCodec);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* The visible text of one of this track's cues.
|
|
32
|
+
*
|
|
33
|
+
* @param {string} textField - The cue's text field, already out of its
|
|
34
|
+
* container's framing. This method knows the codec and not the container,
|
|
35
|
+
* which is why it cannot be handed a whole dialogue row.
|
|
36
|
+
* @returns {string}
|
|
37
|
+
*/
|
|
38
|
+
plainText(textField) {
|
|
39
|
+
return plainCueText(textField, this.textCodec);
|
|
40
|
+
}
|
|
41
|
+
|
|
24
42
|
static isTextCodec(codecId) {
|
|
25
43
|
return TEXT_CODECS_MATROSKA.has(codecId) || TEXT_FORMATS_MP4.has(codecId);
|
|
26
44
|
}
|
package/services/tracks/index.js
CHANGED
|
@@ -4,6 +4,7 @@ export { AudioTrack } from "./AudioTrack.js";
|
|
|
4
4
|
export { SubtitleTrack } from "./SubtitleTrack.js";
|
|
5
5
|
export { TextSubtitleTrack, TEXT_CODECS_MATROSKA, TEXT_FORMATS_MP4 } from "./TextSubtitleTrack.js";
|
|
6
6
|
export { ImageSubtitleTrack } from "./ImageSubtitleTrack.js";
|
|
7
|
+
export { MarkupKind, markupKindOf, plainCueText } from "./subtitle-markup.js";
|
|
7
8
|
// There is deliberately no class for a track that lives in a file of its own.
|
|
8
9
|
// `<name>.mka` is a Matroska container holding an `AudioTrack`, and
|
|
9
10
|
// `MatroskaContainer` reads it exactly as it reads the picture's — so "external"
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file The markup a subtitle codec puts INSIDE the text of one cue, and how to
|
|
3
|
+
* take it off. One axis of variation, and only one.
|
|
4
|
+
*
|
|
5
|
+
* Nothing here knows which container the text came out of. That is the other
|
|
6
|
+
* axis and it belongs to `container/`: how a cue's bytes are framed is stated by
|
|
7
|
+
* the CONTAINER's specification, not by the subtitle format's. Matroska
|
|
8
|
+
* reorders an ASS dialogue row and drops its two timing fields
|
|
9
|
+
* (`matroska.org/technical/subtitles.html`); a `.ass` file states its own field
|
|
10
|
+
* order in the `Format:` line of `[Events]`; an MP4 carries each cue as a sample
|
|
11
|
+
* with a length prefix. Every one of those is a fact about the container.
|
|
12
|
+
*
|
|
13
|
+
* What IS a fact about ASS, wherever it is stored: override groups in braces,
|
|
14
|
+
* `\N` and `\n` for a line break, `\h` for a hard space. That is this file.
|
|
15
|
+
*
|
|
16
|
+
* The two were mixed in one function until 2.72.1, which is what broke English
|
|
17
|
+
* subtitles on an embedded ASS track: the function counted commas to guess
|
|
18
|
+
* which framing it had been handed, expected ten fields — the shape of a row in
|
|
19
|
+
* a FILE — and a Matroska block carries nine. Every field of the row was then
|
|
20
|
+
* shown to the viewer as if it were dialogue. A function that has to guess the
|
|
21
|
+
* shape of its input is being called by someone who knew and did not say.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
/** How a cue's text is marked up, once the container's framing is off. */
|
|
25
|
+
export const MarkupKind = {
|
|
26
|
+
/** Sub Station Alpha and its advanced form: `{\pos(…)}`, `\N`, `\h`. */
|
|
27
|
+
ASS: "ass",
|
|
28
|
+
/** Nothing to strip: the text is what is shown. */
|
|
29
|
+
NONE: "none"
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Which markup a codec's cue text carries.
|
|
34
|
+
*
|
|
35
|
+
* The keys are every name this proxy has for a text subtitle codec: Matroska
|
|
36
|
+
* CodecIDs (RFC 9559 §5.1.4.1.28 and the codec mappings beside it), MP4 sample
|
|
37
|
+
* entry types (ISO/IEC 14496-12 §12.6, plus Apple's `tx3g`), and the file
|
|
38
|
+
* extensions a subtitle shipped beside the film uses. They are listed together
|
|
39
|
+
* because the ANSWER is the same for all of them — ASS is ASS whether it sits
|
|
40
|
+
* in a Matroska block, in a file, or nowhere yet — and keeping three tables
|
|
41
|
+
* would mean three places to forget.
|
|
42
|
+
*
|
|
43
|
+
* @type {Map<string, string>}
|
|
44
|
+
*/
|
|
45
|
+
const MARKUP_BY_CODEC = new Map([
|
|
46
|
+
["S_TEXT/ASS", MarkupKind.ASS],
|
|
47
|
+
["S_TEXT/SSA", MarkupKind.ASS],
|
|
48
|
+
[".ass", MarkupKind.ASS],
|
|
49
|
+
[".ssa", MarkupKind.ASS],
|
|
50
|
+
["S_TEXT/UTF8", MarkupKind.NONE],
|
|
51
|
+
["S_TEXT/WEBVTT", MarkupKind.NONE],
|
|
52
|
+
[".srt", MarkupKind.NONE],
|
|
53
|
+
[".vtt", MarkupKind.NONE],
|
|
54
|
+
[".webvtt", MarkupKind.NONE],
|
|
55
|
+
["tx3g", MarkupKind.NONE],
|
|
56
|
+
["text", MarkupKind.NONE],
|
|
57
|
+
["wvtt", MarkupKind.NONE]
|
|
58
|
+
]);
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* The markup kind of a codec, by any of its names.
|
|
62
|
+
*
|
|
63
|
+
* An unknown codec is answered `NONE` rather than refused: the text is then
|
|
64
|
+
* shown as it is, which is wrong only in so far as some markup stays visible,
|
|
65
|
+
* where a refusal would show nothing at all.
|
|
66
|
+
*
|
|
67
|
+
* @param {string} codecId - Matroska CodecID, MP4 sample entry type, or a file
|
|
68
|
+
* extension including the dot. Case is ignored for extensions, which arrive
|
|
69
|
+
* from file names, and kept for the others, which are spelled by a spec.
|
|
70
|
+
* @returns {string} One of {@link MarkupKind}.
|
|
71
|
+
*/
|
|
72
|
+
export function markupKindOf(codecId) {
|
|
73
|
+
const name = String(codecId ?? "");
|
|
74
|
+
return MARKUP_BY_CODEC.get(name) ?? MARKUP_BY_CODEC.get(name.toLowerCase()) ?? MarkupKind.NONE;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* The visible text of one cue: its markup taken off, nothing else touched.
|
|
79
|
+
*
|
|
80
|
+
* @param {string} text - The cue's text FIELD, already out of the container's
|
|
81
|
+
* framing. Handing a whole dialogue row to this is the mistake described at
|
|
82
|
+
* the top of this file.
|
|
83
|
+
* @param {string} codecId
|
|
84
|
+
* @returns {string} Possibly empty — a cue whose text is only a drawing command
|
|
85
|
+
* or a positioning group has nothing to show, and the caller drops it.
|
|
86
|
+
*/
|
|
87
|
+
export function plainCueText(text, codecId) {
|
|
88
|
+
const raw = String(text ?? "");
|
|
89
|
+
if (markupKindOf(codecId) !== MarkupKind.ASS) {
|
|
90
|
+
return raw.trim();
|
|
91
|
+
}
|
|
92
|
+
return raw
|
|
93
|
+
// An override group. Any brace content is a directive, never dialogue:
|
|
94
|
+
// drawing commands, karaoke timing, positioning, font changes.
|
|
95
|
+
.replace(/\{[^}]*\}/g, "")
|
|
96
|
+
// Both breaks reach a player as a break. ASS distinguishes them — `\N` is
|
|
97
|
+
// always a break, `\n` only where the style does not wrap — and a WebVTT
|
|
98
|
+
// cue has no way to express the difference, so it takes the break.
|
|
99
|
+
.replace(/\\N/g, "\n")
|
|
100
|
+
.replace(/\\n/g, "\n")
|
|
101
|
+
// A space the renderer may not collapse.
|
|
102
|
+
.replace(/\\h/g, " ")
|
|
103
|
+
.trim();
|
|
104
|
+
}
|
|
@@ -34,6 +34,12 @@ import { WebSocket } from "ws";
|
|
|
34
34
|
* Called each time the WebSocket connection becomes open (including reconnects).
|
|
35
35
|
* Use to re-register the proxy so the server's in-memory store stays consistent
|
|
36
36
|
* after server restarts.
|
|
37
|
+
* @property {(mediaInfo: object) => { copy: number[], transcode: number[] } | null} [onCanServeRequest]
|
|
38
|
+
* Called when the server asks whether this host could sustain a file it has
|
|
39
|
+
* been DESCRIBED — height, rate, bitrate, codec — rather than one it holds.
|
|
40
|
+
* Answered from this host's startup benchmarks alone: no torrent is added, no
|
|
41
|
+
* bytes are fetched and ffmpeg is not run, so it costs milliseconds and can
|
|
42
|
+
* be asked of every proxy in the pool at once.
|
|
37
43
|
* @property {() => HealthMetrics} [onHealthRequest]
|
|
38
44
|
* Called when the server sends a `health-request` message. The return value is
|
|
39
45
|
* sent back as `health-response` and used by the server to score this proxy.
|
|
@@ -117,6 +123,7 @@ export function createTunnelClient({
|
|
|
117
123
|
onSignal,
|
|
118
124
|
onConnect,
|
|
119
125
|
onHealthRequest,
|
|
126
|
+
onCanServeRequest,
|
|
120
127
|
onLog,
|
|
121
128
|
connectionLifetimeMs = CONNECTION_LIFETIME_MS
|
|
122
129
|
}) {
|
|
@@ -221,6 +228,26 @@ export function createTunnelClient({
|
|
|
221
228
|
return;
|
|
222
229
|
}
|
|
223
230
|
|
|
231
|
+
// Could this host serve a file it is only told ABOUT? Asked when the
|
|
232
|
+
// proxy a viewer landed on has refused the file, so the browser can be
|
|
233
|
+
// sent somewhere that will work instead of being shown an error. The
|
|
234
|
+
// description travels because the refusing proxy has already probed the
|
|
235
|
+
// file: the expensive half is done once, and every other proxy answers
|
|
236
|
+
// by arithmetic.
|
|
237
|
+
if (message.type === "can-serve-request") {
|
|
238
|
+
let offer = null;
|
|
239
|
+
try {
|
|
240
|
+
offer = typeof onCanServeRequest === "function"
|
|
241
|
+
? onCanServeRequest(message.mediaInfo ?? {})
|
|
242
|
+
: null;
|
|
243
|
+
} catch {
|
|
244
|
+
// silent-ok: an unanswerable question is answered "no", which is what
|
|
245
|
+
// a null offer means to the caller.
|
|
246
|
+
}
|
|
247
|
+
send({ type: "can-serve-response", requestId: message.requestId, offer });
|
|
248
|
+
return;
|
|
249
|
+
}
|
|
250
|
+
|
|
224
251
|
// Health check: server requests current metrics for proxy scoring.
|
|
225
252
|
if (message.type === "health-request") {
|
|
226
253
|
const metrics = typeof onHealthRequest === "function" ? onHealthRequest() : {};
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file What a proxy says about itself, and what the pool scores it on.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import test from "node:test";
|
|
6
|
+
import assert from "node:assert/strict";
|
|
7
|
+
import os from "node:os";
|
|
8
|
+
|
|
9
|
+
import { availableMemoryBytes, collectHealthMetrics } from "../services/health-collector.js";
|
|
10
|
+
|
|
11
|
+
test("free memory is what could be given out, not what is idle this instant", () => {
|
|
12
|
+
const available = availableMemoryBytes();
|
|
13
|
+
assert.ok(Number.isFinite(available) && available > 0);
|
|
14
|
+
|
|
15
|
+
// On Linux this reads the kernel's own `MemAvailable`, and it is at least
|
|
16
|
+
// `os.freemem()` by construction: the kernel keeps free memory low on purpose
|
|
17
|
+
// and fills the rest with cache, which it hands back the moment anything
|
|
18
|
+
// asks. The old reading was `os.freemem()`, so a host with 4 GB of cache and
|
|
19
|
+
// 200 MB genuinely free reported itself nearly full while it had 4.2 GB to
|
|
20
|
+
// give — and that figure weighs 0.4 of every proxy's score.
|
|
21
|
+
if (os.platform() === "linux") {
|
|
22
|
+
assert.ok(
|
|
23
|
+
available >= os.freemem(),
|
|
24
|
+
`MemAvailable ${available} is below freemem ${os.freemem()}, which cannot be`
|
|
25
|
+
);
|
|
26
|
+
}
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
test("the health report is three bounded numbers", () => {
|
|
30
|
+
const metrics = collectHealthMetrics();
|
|
31
|
+
assert.ok(metrics.cpuLoad >= 0);
|
|
32
|
+
assert.ok(metrics.memFree > 0 && metrics.memFree <= 1);
|
|
33
|
+
assert.ok(Number.isInteger(metrics.uptime) && metrics.uptime >= 0);
|
|
34
|
+
// Three decimals, so a value that has not really moved does not produce a
|
|
35
|
+
// different message on every poll.
|
|
36
|
+
assert.equal(metrics.memFree, Math.round(metrics.memFree * 1000) / 1000);
|
|
37
|
+
});
|
|
@@ -154,10 +154,15 @@ test("the store says why it spills: what is asked of it, what it had to take, ho
|
|
|
154
154
|
asked.demand.unionPieces > asked.demand.capacity,
|
|
155
155
|
"a reader asking for more than the store holds is arithmetic, not a policy fault"
|
|
156
156
|
);
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
157
|
+
// And it no longer has to take a declared piece to make room. The reader
|
|
158
|
+
// asked for ten pieces of a store that holds four, and the arrivals that
|
|
159
|
+
// will be wanted LATER than what is already resident are written straight
|
|
160
|
+
// to disk instead of displacing what will be wanted sooner. Before
|
|
161
|
+
// 2026-09-02 every one of them displaced something and was read back
|
|
162
|
+
// moments later: 233 evictions against 119 completed writes in a minute.
|
|
163
|
+
assert.equal(asked.evictedProtected, 0, "a nearer piece was pushed out for a further one");
|
|
164
|
+
assert.equal(asked.spills, 0, "nothing had to be written out to make room on admission");
|
|
165
|
+
assert.ok(asked.admittedToDisk > 0, "the further arrivals were supposed to go to disk");
|
|
161
166
|
|
|
162
167
|
// Read back the pieces that were spilled: each one comes home, and the
|
|
163
168
|
// store says how long it had been away.
|
|
@@ -167,12 +172,17 @@ test("the store says why it spills: what is asked of it, what it had to take, ho
|
|
|
167
172
|
}
|
|
168
173
|
|
|
169
174
|
const after = store.stats();
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
);
|
|
175
|
+
// Reading them back does evict, and that is ordinary: a piece brought into
|
|
176
|
+
// memory has to displace one. What the change of 2026-09-02 removed is the
|
|
177
|
+
// eviction on ADMISSION — `asked.spills` above is zero, where before it all
|
|
178
|
+
// six arrivals displaced a nearer piece and were read back moments later.
|
|
179
|
+
assert.ok(after.fromDisk > 0, "the pieces that went to disk were read back from it");
|
|
180
|
+
assert.ok(after.revivals > 0, "reading one back brings it into memory");
|
|
181
|
+
// No age to report, and that is right rather than missing: an age measures
|
|
182
|
+
// how long an EVICTED piece stayed away, and these were never resident —
|
|
183
|
+
// they were written on arrival and read back once.
|
|
184
|
+
assert.equal(after.revivalAgeSamples, 0);
|
|
185
|
+
assert.equal(after.revivalAgeMedianMs, null);
|
|
176
186
|
|
|
177
187
|
store.releaseProtection("video");
|
|
178
188
|
assert.equal(store.stats().demand.readers, 0, "a reader that ends stops being counted");
|
|
@@ -401,3 +411,81 @@ test("the allowance is never cut below one reader's whole window", async () => {
|
|
|
401
411
|
await fs.rm(directory, { recursive: true, force: true });
|
|
402
412
|
}
|
|
403
413
|
});
|
|
414
|
+
|
|
415
|
+
test("the pool stays the size it is allowed to be, however many pieces pass through it", async () => {
|
|
416
|
+
// The field failure of 2026-09-02, and the case none of the earlier checks
|
|
417
|
+
// covered: an allowance far smaller than the number of pieces in flight.
|
|
418
|
+
// Both paths that register a piece look first and then await — `put` for a
|
|
419
|
+
// slot, a revival for the disk — and in that gap another caller can register
|
|
420
|
+
// the same index. Overwriting the entry dropped the previous block without
|
|
421
|
+
// returning it, so the pool's count climbed past its ceiling for ever and it
|
|
422
|
+
// degenerated into a fresh block per piece. A store holding THREE pieces
|
|
423
|
+
// reported 812 MB committed against 68 MB allowed, 739 blocks allocated and
|
|
424
|
+
// 676 still alive, and the process was killed at 4.37 GB.
|
|
425
|
+
const capacity = 4;
|
|
426
|
+
const { store, directory } = await makeStore(capacity);
|
|
427
|
+
try {
|
|
428
|
+
// Concurrent, because that is what produces the race: several peers deliver
|
|
429
|
+
// pieces of one torrent at the same time.
|
|
430
|
+
for (let round = 0; round < 6; round += 1) {
|
|
431
|
+
await Promise.all(
|
|
432
|
+
Array.from({ length: 16 }, (unused, index) => put(store, index, pieceOf(index)))
|
|
433
|
+
);
|
|
434
|
+
// And read them back, which revives from disk and races registration the
|
|
435
|
+
// other way about.
|
|
436
|
+
await Promise.all([0, 3, 7, 11, 15].map((index) => get(store, index)));
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
const stats = store.stats();
|
|
440
|
+
assert.equal(stats.blocksBeyondCeiling, 0, "the pool grew past what it is allowed to hold");
|
|
441
|
+
assert.ok(
|
|
442
|
+
stats.blocksAllocated <= capacity + stats.blocksFree,
|
|
443
|
+
`pool of ${stats.blocksAllocated} blocks for ${capacity} slots and ${stats.blocksFree} spare`
|
|
444
|
+
);
|
|
445
|
+
assert.equal(stats.committedBytes, stats.blocksAllocated * PIECE);
|
|
446
|
+
|
|
447
|
+
// Every piece still reads back as itself: a block returned to the pool
|
|
448
|
+
// twice, or reused while somebody held it, shows up here as wrong bytes.
|
|
449
|
+
for (let index = 0; index < 16; index += 1) {
|
|
450
|
+
const bytes = await get(store, index);
|
|
451
|
+
assert.ok(bytes.equals(pieceOf(index)), `piece ${index} came back changed`);
|
|
452
|
+
}
|
|
453
|
+
} finally {
|
|
454
|
+
store.destroy(() => undefined);
|
|
455
|
+
await fs.rm(directory, { recursive: true, force: true });
|
|
456
|
+
}
|
|
457
|
+
});
|
|
458
|
+
|
|
459
|
+
test("a piece wanted later than the one it would displace goes to disk instead", async () => {
|
|
460
|
+
const capacity = 4;
|
|
461
|
+
const { store, directory } = await makeStore(capacity);
|
|
462
|
+
try {
|
|
463
|
+
// A reader is at the start of the file and the store is full of what it
|
|
464
|
+
// wants. Its window is pieces 0-3.
|
|
465
|
+
store.protectRange("video", 0, 3);
|
|
466
|
+
for (let index = 0; index < capacity; index += 1) {
|
|
467
|
+
await put(store, index, pieceOf(index));
|
|
468
|
+
}
|
|
469
|
+
const before = store.stats().admittedToDisk;
|
|
470
|
+
|
|
471
|
+
// A piece arrives from far ahead. It IS inside a window in the field case —
|
|
472
|
+
// six readers covered the whole file — but it lies further from every read
|
|
473
|
+
// head than the piece the store would have to evict for it. Admitting it
|
|
474
|
+
// would write out the nearer piece and read that one back sooner.
|
|
475
|
+
store.protectRange("far", 90, 99);
|
|
476
|
+
await put(store, 95, pieceOf(95));
|
|
477
|
+
|
|
478
|
+
assert.ok(
|
|
479
|
+
store.stats().admittedToDisk > before,
|
|
480
|
+
"the further piece was admitted to memory and the nearer one written out"
|
|
481
|
+
);
|
|
482
|
+
// And it still reads back as itself, from the disk it was put on.
|
|
483
|
+
assert.ok((await get(store, 95)).equals(pieceOf(95)));
|
|
484
|
+
for (let index = 0; index < capacity; index += 1) {
|
|
485
|
+
assert.ok((await get(store, index)).equals(pieceOf(index)), `piece ${index} was lost`);
|
|
486
|
+
}
|
|
487
|
+
} finally {
|
|
488
|
+
store.destroy(() => undefined);
|
|
489
|
+
await fs.rm(directory, { recursive: true, force: true });
|
|
490
|
+
}
|
|
491
|
+
});
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file What the allowance has to bound, and why it is not the resident pieces.
|
|
3
|
+
*
|
|
4
|
+
* The field failure of 2026-09-02, in one sentence: a piece being written out
|
|
5
|
+
* leaves the store's own count the moment the eviction begins, while its memory
|
|
6
|
+
* stays held until the write that reads from it has finished. So every
|
|
7
|
+
* admission turned one resident block into one block held by the disk and took
|
|
8
|
+
* a fresh block for the arrival, and the memory in use went UP by one per
|
|
9
|
+
* admission for as long as the disk was behind.
|
|
10
|
+
*
|
|
11
|
+
* It was behind by a factor of two: 233 evictions started against about 119
|
|
12
|
+
* writes completed in the same minute. The store reported 203 blocks held with
|
|
13
|
+
* THREE pieces resident and 68 MB allowed, and the kernel killed the process at
|
|
14
|
+
* 4.37 GB.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import test from "node:test";
|
|
18
|
+
import assert from "node:assert/strict";
|
|
19
|
+
import fs from "node:fs/promises";
|
|
20
|
+
import os from "node:os";
|
|
21
|
+
import path from "node:path";
|
|
22
|
+
import { SharedPieceStore } from "../services/piece-store/shared-piece-store.js";
|
|
23
|
+
|
|
24
|
+
const PIECE = 1024;
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* A disk that answers only when the test lets it, so writes can be made slower
|
|
28
|
+
* than arrivals on purpose — which is the whole of the field condition.
|
|
29
|
+
*/
|
|
30
|
+
function heldDisk() {
|
|
31
|
+
const stored = new Set();
|
|
32
|
+
/** @type {Array<() => void>} */
|
|
33
|
+
const waiting = [];
|
|
34
|
+
return {
|
|
35
|
+
pending: waiting,
|
|
36
|
+
get size() {
|
|
37
|
+
return stored.size;
|
|
38
|
+
},
|
|
39
|
+
has: (index) => stored.has(index),
|
|
40
|
+
forget: (index) => stored.delete(index),
|
|
41
|
+
write(index) {
|
|
42
|
+
return new Promise((resolve) => {
|
|
43
|
+
waiting.push(() => {
|
|
44
|
+
stored.add(index);
|
|
45
|
+
resolve();
|
|
46
|
+
});
|
|
47
|
+
});
|
|
48
|
+
},
|
|
49
|
+
async read(index, target) {
|
|
50
|
+
target.fill(index % 251);
|
|
51
|
+
return target.length;
|
|
52
|
+
},
|
|
53
|
+
async close() { waiting.length = 0; },
|
|
54
|
+
async destroy() { stored.clear(); waiting.length = 0; }
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* @param {SharedPieceStore} store
|
|
60
|
+
* @param {number} index
|
|
61
|
+
* @returns {Promise<void>}
|
|
62
|
+
*/
|
|
63
|
+
const put = (store, index) =>
|
|
64
|
+
new Promise((resolve, reject) => {
|
|
65
|
+
store.put(index, Buffer.alloc(PIECE, index % 251), (error) => (error ? reject(error) : resolve()));
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
test("memory in use never runs past the allowance while the disk is behind", async () => {
|
|
69
|
+
const directory = await fs.mkdtemp(path.join(os.tmpdir(), "slow-disk-test-"));
|
|
70
|
+
const disk = heldDisk();
|
|
71
|
+
const capacity = 4;
|
|
72
|
+
const store = new SharedPieceStore(PIECE, {
|
|
73
|
+
length: PIECE * 200,
|
|
74
|
+
memoryBytes: PIECE * capacity,
|
|
75
|
+
path: directory,
|
|
76
|
+
name: "slow-disk",
|
|
77
|
+
disk
|
|
78
|
+
});
|
|
79
|
+
try {
|
|
80
|
+
// Fill it, then keep pieces coming while every write stays unfinished.
|
|
81
|
+
for (let index = 0; index < capacity; index += 1) {
|
|
82
|
+
await put(store, index);
|
|
83
|
+
}
|
|
84
|
+
const arrivals = [];
|
|
85
|
+
for (let index = capacity; index < capacity + 40; index += 1) {
|
|
86
|
+
arrivals.push(put(store, index).catch(() => undefined));
|
|
87
|
+
}
|
|
88
|
+
// Let them get as far as they can with the disk answering nothing.
|
|
89
|
+
await new Promise((resolve) => setTimeout(resolve, 200));
|
|
90
|
+
|
|
91
|
+
const held = store.stats();
|
|
92
|
+
// The check the field failure would fail: a store allowed four pieces held
|
|
93
|
+
// two hundred and three. Blocks held by a pending write count here, because
|
|
94
|
+
// they are memory whatever the piece count says.
|
|
95
|
+
assert.ok(
|
|
96
|
+
held.blocksInUse <= capacity,
|
|
97
|
+
`${held.blocksInUse} blocks in use with ${capacity} allowed `
|
|
98
|
+
+ `(${held.blocksInFlight} of them held by writes that have not finished)`
|
|
99
|
+
);
|
|
100
|
+
assert.ok(held.blocksInFlight > 0, "the disk was supposed to be holding some");
|
|
101
|
+
assert.ok(held.waitedForDisk > 0, "admission was supposed to wait for the disk, not evict");
|
|
102
|
+
|
|
103
|
+
// Let the disk answer: the blocks come back and everything settles.
|
|
104
|
+
while (disk.pending.length > 0) {
|
|
105
|
+
disk.pending.shift()?.();
|
|
106
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
107
|
+
}
|
|
108
|
+
await Promise.all(arrivals);
|
|
109
|
+
assert.ok(store.stats().blocksInUse <= capacity + 1, "the pool did not settle");
|
|
110
|
+
} finally {
|
|
111
|
+
store.destroy(() => undefined);
|
|
112
|
+
await fs.rm(directory, { recursive: true, force: true });
|
|
113
|
+
}
|
|
114
|
+
});
|