@torrent-tv/proxy 2.101.6 → 2.101.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/routes/encode-input/get.js +5 -2
- package/server.js +6 -1
- package/services/encode/EncodeInputs.js +84 -35
- package/services/encode/EncodeRuns.js +5 -1
- package/services/encode/OriginalInput.js +55 -11
- package/services/encode/OriginalInputStore.js +217 -0
- package/services/media/container/AviContainer.js +5 -4
- package/services/media/container/MatroskaContainer.js +6 -4
- package/services/server/wire-outputs.js +2 -0
- package/services/storage/files/CompletedFiles.js +29 -5
- package/services/storage/wire.js +1 -0
- package/services/torrent/worker/client.js +24 -0
- package/services/torrent/worker/completed-input-file.js +50 -0
- package/services/torrent/worker/pool-adapter.js +4 -0
- package/services/torrent/worker/protocol.js +3 -0
- package/services/torrent/worker/worker.js +17 -0
package/package.json
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { parseRange } from "../../utils/parse-range.js";
|
|
2
|
+
import { Readable } from "node:stream";
|
|
2
3
|
|
|
3
4
|
/** FFmpeg reads only original byte ranges already owned by its live run. */
|
|
4
5
|
export async function handleEncodeInputGet(req, reply, { inputOf, refused = () => {} }) {
|
|
@@ -53,10 +54,12 @@ export async function handleEncodeInputGet(req, reply, { inputOf, refused = () =
|
|
|
53
54
|
const socket = req.raw?.socket;
|
|
54
55
|
const writtenBefore = socket?.bytesWritten ?? 0;
|
|
55
56
|
reply.raw?.once?.("close", () => {
|
|
57
|
+
held.release?.();
|
|
56
58
|
input.noteAnswer?.({ fileIndex, start: range.start, end: held.end,
|
|
57
59
|
written: Math.max(0, (socket?.bytesWritten ?? writtenBefore) - writtenBefore),
|
|
58
60
|
handedOver: reply.raw?.writableFinished === true });
|
|
59
61
|
});
|
|
60
|
-
return reply.code(206).header("Content-Length", String(held.bytes.length))
|
|
61
|
-
.header("Content-Range", `bytes ${range.start}-${held.end}/${length}`)
|
|
62
|
+
return reply.code(206).header("Content-Length", String(held.size ?? held.bytes.length))
|
|
63
|
+
.header("Content-Range", `bytes ${range.start}-${held.end}/${length}`)
|
|
64
|
+
.send(held.stream ?? (held.chunks ? Readable.from(held.chunks, { objectMode: false }) : held.bytes));
|
|
62
65
|
}
|
package/server.js
CHANGED
|
@@ -978,22 +978,26 @@ export async function startProxyServer({
|
|
|
978
978
|
return answer && typeof answer.then !== "function" && answer.end === end ? answer.bytes : null;
|
|
979
979
|
};
|
|
980
980
|
const holdsUntil = start => {
|
|
981
|
+
if (input.heldUntil) return input.heldUntil(source.fileIndex, start);
|
|
981
982
|
const range = source.input.ranges.find(([first, last]) => first <= start && start <= last);
|
|
982
983
|
return range ? range[1] + 1 : null;
|
|
983
984
|
};
|
|
984
985
|
const startedAt = Date.now();
|
|
986
|
+
const releaseInput = input.retain?.();
|
|
985
987
|
void container.learnPackets({ from: source.input.from, to: source.input.to, trackIds: [...sounds] }, read, holdsUntil).then(learned =>
|
|
986
988
|
logger.info(`encode input #${from}..#${to} of ${output.outputKey}: soundtrack packets of track(s) ${[...sounds].join(",")} ` +
|
|
987
989
|
`over ${source.input.from.toFixed(3)}-${source.input.to.toFixed(3)}s ` +
|
|
988
990
|
(learned === "walked" ? "walked" : `not all walked (${learned === "memory" ? "memory refused" : "the copy does not hold the rest"})`) +
|
|
989
991
|
` from its copy in ${Date.now() - startedAt}ms`),
|
|
990
|
-
error => logger.warn(`encode input #${from}..#${to} of ${output.outputKey}: walking the soundtrack from its copy failed: ${error?.message ?? error}`))
|
|
992
|
+
error => logger.warn(`encode input #${from}..#${to} of ${output.outputKey}: walking the soundtrack from its copy failed: ${error?.message ?? error}`))
|
|
993
|
+
.finally(() => releaseInput?.());
|
|
991
994
|
}
|
|
992
995
|
},
|
|
993
996
|
readEncodeRanges: async (source, ranges, maxBytes) => {
|
|
994
997
|
const torrent = torrentPool.knownTorrent(source.sourceKey);
|
|
995
998
|
return torrent ? torrentPool.readHeldRangesOf(torrent, source.fileIndex, ranges, maxBytes) : null;
|
|
996
999
|
},
|
|
1000
|
+
openEncodeWholeFile: source => torrentPool.acquireWholeFile(source),
|
|
997
1001
|
// Which bytes of a file are downloaded whole now — what an original
|
|
998
1002
|
// input's stretch may extend over. A snapshot: the copy that follows
|
|
999
1003
|
// holds the pieces it reads, and refuses where one has gone since.
|
|
@@ -1036,6 +1040,7 @@ export async function startProxyServer({
|
|
|
1036
1040
|
wholeFiles: typeof torrentPool.allowWholeFileBytes === "function"
|
|
1037
1041
|
? {
|
|
1038
1042
|
held: () => torrentPool.wholeFileBytes ?? 0,
|
|
1043
|
+
required: () => torrentPool.wholeFileRequiredBytes ?? 0,
|
|
1039
1044
|
allow: (bytes) => torrentPool.allowWholeFileBytes(bytes),
|
|
1040
1045
|
root: completedFilesRoot()
|
|
1041
1046
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { admitInput } from "./AdmittedInput.js";
|
|
2
2
|
import { admitOriginalInput, originalInputBytes } from "./OriginalInput.js";
|
|
3
|
+
import { OriginalInputStore } from "./OriginalInputStore.js";
|
|
3
4
|
|
|
4
5
|
/**
|
|
5
6
|
* How many recent readings a learned figure here is taken from: long enough
|
|
@@ -32,6 +33,7 @@ export class EncodeInputs {
|
|
|
32
33
|
#preparedIn = new WeakMap();
|
|
33
34
|
#memoryRevision = 0;
|
|
34
35
|
#held = 0;
|
|
36
|
+
#shared = new OriginalInputStore({ changed: () => queueMicrotask(() => this.memoryChanged()) });
|
|
35
37
|
#allowed = 0;
|
|
36
38
|
/** What each preparing request asks of memory: all it would use, and the least it can run with. */
|
|
37
39
|
#wanted = new Map();
|
|
@@ -46,6 +48,7 @@ export class EncodeInputs {
|
|
|
46
48
|
#openOf;
|
|
47
49
|
#resolve;
|
|
48
50
|
#read;
|
|
51
|
+
#openWholeFile;
|
|
49
52
|
#heldRanges;
|
|
50
53
|
#revise;
|
|
51
54
|
#changed;
|
|
@@ -63,12 +66,13 @@ export class EncodeInputs {
|
|
|
63
66
|
*/
|
|
64
67
|
#stretchCopied;
|
|
65
68
|
|
|
66
|
-
constructor({ resolve, readRanges, heldRanges = null, reviseBudget, changed, failed, capacity = () => null, urgent = () => true,
|
|
69
|
+
constructor({ resolve, readRanges, openWholeFile = null, heldRanges = null, reviseBudget, changed, failed, capacity = () => null, urgent = () => true,
|
|
67
70
|
rankAt = () => null, openOf = () => null, log = () => {}, now = Date.now, stretchCopied = () => {} }) {
|
|
68
71
|
this.#stretchCopied = stretchCopied;
|
|
69
72
|
this.#openOf = openOf;
|
|
70
73
|
this.#resolve = resolve;
|
|
71
74
|
this.#read = readRanges;
|
|
75
|
+
this.#openWholeFile = openWholeFile;
|
|
72
76
|
this.#heldRanges = heldRanges;
|
|
73
77
|
this.#revise = reviseBudget;
|
|
74
78
|
this.#changed = changed;
|
|
@@ -95,7 +99,7 @@ export class EncodeInputs {
|
|
|
95
99
|
const serves = request.serves ?? request.from;
|
|
96
100
|
const rank = this.#rankAt(request.output, serves);
|
|
97
101
|
this.#log(`encode input #${request.from}..#${request.to} of ${request.output.outputKey ?? request.output.id}: ${state}` +
|
|
98
|
-
` (held ${this
|
|
102
|
+
` (held ${this.held()} of ${this.#allowed} allowed; for #${serves}` +
|
|
99
103
|
`${rank ? `, rank ${rank.rank} of ${rank.topRank}` : ""})${detail}`);
|
|
100
104
|
}
|
|
101
105
|
|
|
@@ -120,10 +124,10 @@ export class EncodeInputs {
|
|
|
120
124
|
}
|
|
121
125
|
}
|
|
122
126
|
|
|
123
|
-
held() { return this.#held; }
|
|
124
|
-
wanted() { return this
|
|
127
|
+
held() { return this.#held + this.#shared.bytes; }
|
|
128
|
+
wanted() { return this.held() + [...this.#wanted.values()].reduce((sum, asked) => sum + asked.wanted, 0); }
|
|
125
129
|
required() {
|
|
126
|
-
let bytes = this.#held;
|
|
130
|
+
let bytes = this.#held + this.#shared.required;
|
|
127
131
|
for (const [key, asked] of this.#wanted) {
|
|
128
132
|
// The encode plan already chose this finite input. Its least runnable
|
|
129
133
|
// part must fit before production can advance, even when playback has
|
|
@@ -137,6 +141,7 @@ export class EncodeInputs {
|
|
|
137
141
|
allow(bytes) {
|
|
138
142
|
const changed = this.#allowed !== Math.max(0, bytes);
|
|
139
143
|
this.#allowed = Math.max(0, bytes);
|
|
144
|
+
this.#shared.trim(this.#allowed - this.#held);
|
|
140
145
|
if (changed) this.memoryChanged();
|
|
141
146
|
}
|
|
142
147
|
|
|
@@ -320,6 +325,7 @@ export class EncodeInputs {
|
|
|
320
325
|
this.#wanted.delete(key);
|
|
321
326
|
request.result?.release?.();
|
|
322
327
|
}
|
|
328
|
+
this.#shared.forget(output);
|
|
323
329
|
}
|
|
324
330
|
|
|
325
331
|
/**
|
|
@@ -409,7 +415,23 @@ export class EncodeInputs {
|
|
|
409
415
|
if (resolved.kind !== "result") {
|
|
410
416
|
result = resolved;
|
|
411
417
|
} else if (resolved.sources?.every(source => source.input.original === true)) {
|
|
412
|
-
|
|
418
|
+
const sharing = resolved.sources.every(source => Array.isArray(source.input.commonRanges));
|
|
419
|
+
if (sharing) this.#shared.retain(request.output, resolved.sources);
|
|
420
|
+
const files = new Map();
|
|
421
|
+
let unpin = () => {};
|
|
422
|
+
try {
|
|
423
|
+
for (const source of resolved.sources) {
|
|
424
|
+
const file = await this.#openWholeFile?.(source);
|
|
425
|
+
if (file && file.length === source.input.fileLength) files.set(source.fileIndex, file);
|
|
426
|
+
else if (file) await file.release();
|
|
427
|
+
if (this.#requests.get(request.key) !== request) return;
|
|
428
|
+
}
|
|
429
|
+
if (sharing) unpin = this.#shared.protect(resolved.sources.filter(source => !files.has(source.fileIndex)));
|
|
430
|
+
result = await this.#admitStretch(request, resolved.sources, files);
|
|
431
|
+
} finally {
|
|
432
|
+
unpin();
|
|
433
|
+
if (result?.kind !== "result") await Promise.all([...files.values()].map(file => file.release()));
|
|
434
|
+
}
|
|
413
435
|
if (!result) return;
|
|
414
436
|
} else {
|
|
415
437
|
result = await admitInput({ sources: resolved.sources, readRanges: this.#read, reserve: this.#reserveFor(request) });
|
|
@@ -475,7 +497,9 @@ export class EncodeInputs {
|
|
|
475
497
|
|
|
476
498
|
/** Take `bytes` of the allowance for this request now, or answer null. */
|
|
477
499
|
#hold(request, bytes) {
|
|
478
|
-
if (this.#requests.get(request.key) !== request
|
|
500
|
+
if (this.#requests.get(request.key) !== request) return null;
|
|
501
|
+
this.#shared.trim(this.#allowed - this.#held - bytes);
|
|
502
|
+
if (this.#allowed - this.held() < bytes) return null;
|
|
479
503
|
this.#wanted.delete(request.key);
|
|
480
504
|
this.#held += bytes;
|
|
481
505
|
let released = false;
|
|
@@ -497,8 +521,11 @@ export class EncodeInputs {
|
|
|
497
521
|
*
|
|
498
522
|
* @returns {Promise<object | undefined>} Undefined when the request was withdrawn.
|
|
499
523
|
*/
|
|
500
|
-
async #admitStretch(request, firstSources) {
|
|
524
|
+
async #admitStretch(request, firstSources, files = new Map()) {
|
|
501
525
|
const { output, from } = request;
|
|
526
|
+
const sharing = files.size > 0 || firstSources.every(source => Array.isArray(source.input.commonRanges));
|
|
527
|
+
const inMemory = sources => sources.filter(source => !files.has(source.fileIndex));
|
|
528
|
+
const memoryBytes = sources => sharing ? this.#shared.bytesNeeded(inMemory(sources)) : originalInputBytes(sources);
|
|
502
529
|
// Record existing decisions only: diagnostics must not resolve another
|
|
503
530
|
// interval or ask storage for an additional availability reading.
|
|
504
531
|
const probes = [], reservations = [];
|
|
@@ -515,12 +542,13 @@ export class EncodeInputs {
|
|
|
515
542
|
}
|
|
516
543
|
return stretches.get(end);
|
|
517
544
|
};
|
|
518
|
-
const fits = async (end, maxBytes) => {
|
|
545
|
+
const fits = async (end, maxBytes, memory = false) => {
|
|
519
546
|
const stretch = await at(end);
|
|
520
|
-
const withinLimit = Boolean(stretch) && stretch.bytes <= maxBytes;
|
|
547
|
+
const withinLimit = Boolean(stretch) && (memory ? memoryBytes(stretch.sources) <= maxBytes && stretch.bytes <= (target ?? minimum) : stretch.bytes <= maxBytes);
|
|
521
548
|
const result = !stretch ? "unresolved" : !withinLimit ? "size"
|
|
522
|
-
: await this.#present(stretch.sources, held, request.timing) ? "fits" : "missing-bytes";
|
|
523
|
-
probes.push({ end, limitBytes: maxBytes, bytes: stretch?.bytes ?? null, result
|
|
549
|
+
: await this.#present(stretch.sources, held, request.timing, files) ? "fits" : "missing-bytes";
|
|
550
|
+
probes.push({ end, limitBytes: maxBytes, bytes: stretch?.bytes ?? null, result,
|
|
551
|
+
...(sharing ? { bound: memory ? "memory" : "target", memoryBytes: stretch ? memoryBytes(stretch.sources) : null } : {}) });
|
|
524
552
|
return result === "fits";
|
|
525
553
|
};
|
|
526
554
|
// Grows by doubling and then halves back: a handful of resolutions find
|
|
@@ -534,17 +562,17 @@ export class EncodeInputs {
|
|
|
534
562
|
const covering = await at(serves);
|
|
535
563
|
if (covering) { floor = serves; minimum = covering.bytes; }
|
|
536
564
|
}
|
|
537
|
-
const longest = async maxBytes => {
|
|
565
|
+
const longest = async (maxBytes, memory = false) => {
|
|
538
566
|
let good = floor, bad = null, step = 1;
|
|
539
567
|
while (bad === null) {
|
|
540
568
|
const end = good + step;
|
|
541
569
|
if (end > request.to) bad = request.to + 1;
|
|
542
|
-
else if (await fits(end, maxBytes)) { good = end; step *= 2; }
|
|
570
|
+
else if (await fits(end, maxBytes, memory)) { good = end; step *= 2; }
|
|
543
571
|
else bad = end;
|
|
544
572
|
}
|
|
545
573
|
while (bad - good > 1) {
|
|
546
574
|
const middle = Math.floor((good + bad) / 2);
|
|
547
|
-
if (await fits(middle, maxBytes)) good = middle; else bad = middle;
|
|
575
|
+
if (await fits(middle, maxBytes, memory)) good = middle; else bad = middle;
|
|
548
576
|
}
|
|
549
577
|
return good;
|
|
550
578
|
};
|
|
@@ -554,24 +582,26 @@ export class EncodeInputs {
|
|
|
554
582
|
let end = target === null || this.#heldRanges === null ? floor : await longest(target);
|
|
555
583
|
const targetEnd = end;
|
|
556
584
|
if (this.#requests.get(request.key) !== request) return undefined;
|
|
557
|
-
|
|
585
|
+
const minimumMemory = () => memoryBytes(stretches.get(floor).sources);
|
|
586
|
+
this.#wanted.set(request.key, { wanted: memoryBytes(stretches.get(end).sources), required: minimumMemory() });
|
|
558
587
|
await this.#spent(request.timing, "budgetMs", () => this.#revise());
|
|
559
588
|
if (this.#requests.get(request.key) !== request) return undefined;
|
|
560
589
|
const capacity = this.#capacity();
|
|
561
|
-
|
|
562
|
-
|
|
590
|
+
const minimumCapacity = originalInputBytes(inMemory(stretches.get(floor).sources));
|
|
591
|
+
if (capacity !== null && minimumCapacity > capacity && this.#urgent(output, from)) {
|
|
592
|
+
return { kind: "terminal", reason: "source-input-exceeds-memory-capacity", bytes: minimumCapacity, capacity };
|
|
563
593
|
}
|
|
564
|
-
const roomNow = () => Math.min(capacity ?? Number.POSITIVE_INFINITY, this.#allowed - this
|
|
594
|
+
const roomNow = () => Math.min(capacity ?? Number.POSITIVE_INFINITY, this.#allowed - this.held());
|
|
565
595
|
let room = roomNow();
|
|
566
596
|
let result;
|
|
567
597
|
for (;;) {
|
|
568
|
-
if (
|
|
598
|
+
if (minimumMemory() > room) {
|
|
569
599
|
// Wait for memory with the least it can run with, not the larger stretch.
|
|
570
|
-
this.#wanted.set(request.key, { wanted:
|
|
571
|
-
return { kind: "needs-memory", bytes:
|
|
600
|
+
this.#wanted.set(request.key, { wanted: minimumMemory(), required: minimumMemory() });
|
|
601
|
+
return { kind: "needs-memory", bytes: minimumMemory() };
|
|
572
602
|
}
|
|
573
|
-
if (stretches.get(end).
|
|
574
|
-
end = await longest(Math.min(target ?? minimum, room));
|
|
603
|
+
if (memoryBytes(stretches.get(end).sources) > room) {
|
|
604
|
+
end = await longest(sharing ? room : Math.min(target ?? minimum, room), sharing);
|
|
575
605
|
if (this.#requests.get(request.key) !== request) return undefined;
|
|
576
606
|
}
|
|
577
607
|
const chosen = stretches.get(end);
|
|
@@ -581,17 +611,26 @@ export class EncodeInputs {
|
|
|
581
611
|
// it waits for the bytes as one read of everything used to. Field
|
|
582
612
|
// 2026-10-12: a cold MKV's first piece was answered, its stretch read
|
|
583
613
|
// found pieces missing, and the run was refused for good (torrent-tv/meta#178).
|
|
584
|
-
if (this.#heldRanges !== null && !await this.#present(chosen.sources, held, request.timing)) {
|
|
614
|
+
if (this.#heldRanges !== null && !await this.#present(chosen.sources, held, request.timing, files)) {
|
|
585
615
|
const [first] = chosen.sources;
|
|
586
616
|
return { kind: "needs-bytes", sourceKey: first.sourceKey, fileIndex: first.fileIndex, ranges: first.input.ranges,
|
|
587
617
|
// Every source of the stretch, so the bytes of any of them wake it.
|
|
588
618
|
awaiting: chosen.sources.map(source => ({ sourceKey: source.sourceKey, fileIndex: source.fileIndex, ranges: source.input.ranges })) };
|
|
589
619
|
}
|
|
590
620
|
if (this.#requests.get(request.key) !== request) return undefined;
|
|
591
|
-
|
|
592
|
-
|
|
621
|
+
const needed = memoryBytes(chosen.sources);
|
|
622
|
+
this.#wanted.set(request.key, { wanted: needed, required: needed });
|
|
623
|
+
if (sharing) {
|
|
624
|
+
const reservation = { bytes: needed, minimumMemoryBytes: minimumMemory(), allowed: this.#allowed, held: this.held(), capacity, room: roomNow(), end };
|
|
625
|
+
const storage = this.#shared.acquire(inMemory(chosen.sources), roomNow(), { readRanges: this.#read, now: this.#now });
|
|
626
|
+
reservations.push({ ...reservation, admitted: storage !== null });
|
|
627
|
+
if (storage) {
|
|
628
|
+
this.#wanted.delete(request.key);
|
|
629
|
+
result = await admitOriginalInput({ sources: chosen.sources, storage, files, now: this.#now });
|
|
630
|
+
} else result = { kind: "needs-memory", bytes: needed };
|
|
631
|
+
} else result = await admitOriginalInput({ sources: chosen.sources, readRanges: this.#read, now: this.#now,
|
|
593
632
|
reserve: async bytes => {
|
|
594
|
-
const reservation = { bytes, allowed: this.#allowed, held: this
|
|
633
|
+
const reservation = { bytes, allowed: this.#allowed, held: this.held(), capacity,
|
|
595
634
|
room: roomNow(), end };
|
|
596
635
|
const release = this.#hold(request, bytes);
|
|
597
636
|
reservations.push({ ...reservation, admitted: typeof release === "function" });
|
|
@@ -603,11 +642,12 @@ export class EncodeInputs {
|
|
|
603
642
|
if (result.kind !== "needs-memory" || this.#requests.get(request.key) !== request || roomNow() >= room) break;
|
|
604
643
|
room = roomNow();
|
|
605
644
|
}
|
|
606
|
-
if (result.kind === "needs-memory") this.#wanted.set(request.key, { wanted:
|
|
645
|
+
if (result.kind === "needs-memory") this.#wanted.set(request.key, { wanted: minimumMemory(), required: minimumMemory() });
|
|
607
646
|
if (result.kind === "result") {
|
|
608
647
|
this.#log(`encode input selection of ${addressOf(output)}: ${JSON.stringify({
|
|
609
648
|
from, requestedTo: request.to, serves, floor, targetEnd, chosenEnd: end,
|
|
610
649
|
targetBytes: target, minimumBytes: minimum, inputBytes: result.bytes,
|
|
650
|
+
...(sharing ? { minimumMemoryBytes: reservations.at(-1)?.minimumMemoryBytes ?? null, reservedMemoryBytes: reservations.at(-1)?.bytes ?? null } : {}),
|
|
611
651
|
copyBytesPerMs: sizing?.copyBytesPerMs ?? null, openSeconds: sizing?.openSeconds ?? null,
|
|
612
652
|
openMeasured: sizing?.openMeasured ?? null, reservations, probes
|
|
613
653
|
})}`);
|
|
@@ -616,13 +656,13 @@ export class EncodeInputs {
|
|
|
616
656
|
// is the whole copy's, learned once it is done.
|
|
617
657
|
const copied = result.whole ?? Promise.resolve(true);
|
|
618
658
|
void copied.then(whole => {
|
|
619
|
-
if (whole && result.copyMs > 0) keep(this.#copyRates, result.
|
|
620
|
-
if (whole) {
|
|
659
|
+
if (whole && result.copyMs > 0 && result.copiedBytes > 0) keep(this.#copyRates, result.copiedBytes / result.copyMs);
|
|
660
|
+
if (whole && result.copiedBytes + result.sharedBytes > 0) {
|
|
621
661
|
try { this.#stretchCopied(output, from, end, result); }
|
|
622
662
|
catch (error) { this.#log(`encode input #${from}..#${end} of ${addressOf(output)}: what was told of its copy failed: ${error?.message ?? error}`); }
|
|
623
663
|
}
|
|
624
664
|
this.#log(`encode input #${from}..#${end} of ${addressOf(output)}: ` + (whole
|
|
625
|
-
? `stretch copied, ${result.bytes} bytes in ${result.copyMs}ms of copying since it was admitted`
|
|
665
|
+
? `stretch copied, ${result.bytes} bytes in ${result.copyMs}ms of copying since it was admitted; copied ${result.copiedBytes}, shared ${result.sharedBytes}, disk ${result.diskBytes} bytes`
|
|
626
666
|
: result.copyFailed?.() ? "the stretch could not be copied: a piece it needs is no longer held"
|
|
627
667
|
: "released before its stretch was copied"));
|
|
628
668
|
});
|
|
@@ -631,12 +671,21 @@ export class EncodeInputs {
|
|
|
631
671
|
}
|
|
632
672
|
|
|
633
673
|
/** Whether every range of these sources is downloaded whole now. */
|
|
634
|
-
async #present(sources, held, timing) {
|
|
674
|
+
async #present(sources, held, timing, files = new Map()) {
|
|
635
675
|
for (const source of sources) {
|
|
676
|
+
if (files.has(source.fileIndex)) continue;
|
|
636
677
|
const name = `${source.sourceKey}:${source.fileIndex}`;
|
|
637
678
|
if (!held.has(name)) held.set(name, await this.#spent(timing, "heldMs", () => Promise.resolve().then(() => this.#heldRanges(source)).catch(() => null)) ?? []);
|
|
638
|
-
const ranges = held.get(name);
|
|
639
|
-
if (!source.input.ranges.every(([start, end]) =>
|
|
679
|
+
const ranges = [...held.get(name), ...this.#shared.available(source)].sort((a, b) => a[0] - b[0]);
|
|
680
|
+
if (!source.input.ranges.every(([start, end]) => {
|
|
681
|
+
let at = start;
|
|
682
|
+
for (const [first, last] of ranges) {
|
|
683
|
+
if (first > at) return false;
|
|
684
|
+
if (last >= at) at = last + 1;
|
|
685
|
+
if (at > end) return true;
|
|
686
|
+
}
|
|
687
|
+
return false;
|
|
688
|
+
})) return false;
|
|
640
689
|
}
|
|
641
690
|
return true;
|
|
642
691
|
}
|
|
@@ -1220,7 +1220,11 @@ export class EncodeRuns {
|
|
|
1220
1220
|
`ended at ${Date.now() - run.startedAt}ms` + describeTaken(reads.answers, run.startedAt, ended.firstNamedWallMs));
|
|
1221
1221
|
}
|
|
1222
1222
|
this.#noteOpen(session, run, ended);
|
|
1223
|
-
admittedInput.release()
|
|
1223
|
+
void Promise.resolve(admittedInput.release()).then(() => {
|
|
1224
|
+
this.#host.logger.info(`encode input storage of run #${run.from}..#${run.to} on ${session.outputKey}: ` +
|
|
1225
|
+
`input ${admittedInput.bytes} bytes, copied ${admittedInput.copiedBytes ?? 0}, ` +
|
|
1226
|
+
`shared ${admittedInput.sharedBytes ?? 0}, disk ${admittedInput.diskBytes ?? 0} bytes`);
|
|
1227
|
+
});
|
|
1224
1228
|
}
|
|
1225
1229
|
this.#sayRunCpu(session, run);
|
|
1226
1230
|
this.noteRunEnded(session, run, ended);
|
|
@@ -13,10 +13,10 @@ import { createHash } from "node:crypto";
|
|
|
13
13
|
* (torrent-tv/meta#178). `whole` settles when everything is copied — true — or
|
|
14
14
|
* when the copy failed or the input was released first — false.
|
|
15
15
|
*/
|
|
16
|
-
export async function admitOriginalInput({ sources, reserve, readRanges, now = Date.now }) {
|
|
16
|
+
export async function admitOriginalInput({ sources, reserve, readRanges, now = Date.now, storage = null, files = new Map() }) {
|
|
17
17
|
const bytes = originalInputBytes(sources);
|
|
18
18
|
if (!Number.isSafeInteger(bytes) || bytes <= 0) throw new TypeError("Original input requires complete finite byte ranges.");
|
|
19
|
-
const releaseBudget = await reserve(bytes);
|
|
19
|
+
const releaseBudget = storage ? storage.release : await reserve(bytes);
|
|
20
20
|
if (typeof releaseBudget !== "function") return releaseBudget?.kind === "terminal" ? releaseBudget : { kind: "needs-memory", bytes };
|
|
21
21
|
const held = new Map();
|
|
22
22
|
const hash = createHash("sha256");
|
|
@@ -24,10 +24,13 @@ export async function admitOriginalInput({ sources, reserve, readRanges, now = D
|
|
|
24
24
|
// How long the copy itself took, apart from any wait for memory or bytes
|
|
25
25
|
// before it: the figure a run's stretch is sized from, read where the copy is.
|
|
26
26
|
let copyMs = 0;
|
|
27
|
+
let diskBytes = 0;
|
|
27
28
|
// Per source, the range copied after the answer: its position in the
|
|
28
29
|
// source's ranges, and the copy's promise once it has begun.
|
|
29
30
|
const later = [];
|
|
30
31
|
const read = async (source, ranges) => {
|
|
32
|
+
if (files.has(source.fileIndex)) return ranges.map(() => true);
|
|
33
|
+
if (storage) return await storage.copy(source, ranges) ? ranges.map(() => true) : null;
|
|
31
34
|
const startedAt = now();
|
|
32
35
|
const buffers = await readRanges(source, ranges, bytes);
|
|
33
36
|
copyMs += Math.max(0, now() - startedAt);
|
|
@@ -43,7 +46,7 @@ export async function admitOriginalInput({ sources, reserve, readRanges, now = D
|
|
|
43
46
|
const { ranges } = source.input;
|
|
44
47
|
const largest = ranges.reduce((best, range, index) =>
|
|
45
48
|
range[1] - range[0] > ranges[best][1] - ranges[best][0] ? index : best, 0);
|
|
46
|
-
const deferred = ranges.length > 1 ? largest : -1;
|
|
49
|
+
const deferred = !files.has(source.fileIndex) && ranges.length > 1 ? largest : -1;
|
|
47
50
|
const first = ranges.filter((_, index) => index !== deferred);
|
|
48
51
|
const buffers = await read(source, first);
|
|
49
52
|
if (buffers === null) return { kind: "needs-bytes", sourceKey: source.sourceKey, fileIndex: source.fileIndex, ranges };
|
|
@@ -59,6 +62,8 @@ export async function admitOriginalInput({ sources, reserve, readRanges, now = D
|
|
|
59
62
|
if (deferred >= 0) later.push({ entry, position: deferred });
|
|
60
63
|
}
|
|
61
64
|
let released = false;
|
|
65
|
+
let runReleased = false, owners = 1, finishClose;
|
|
66
|
+
const closed = new Promise(resolve => { finishClose = resolve; });
|
|
62
67
|
let failed = false;
|
|
63
68
|
retained = true;
|
|
64
69
|
// ONE READ FOR THE STRETCH. Copied a torrent piece per read (2.97.12), a
|
|
@@ -102,19 +107,46 @@ export async function admitOriginalInput({ sources, reserve, readRanges, now = D
|
|
|
102
107
|
// When FFmpeg's reads waited for a stretch still being copied.
|
|
103
108
|
/** @type {{ from: number, to: number | null }[]} */
|
|
104
109
|
const waits = [];
|
|
105
|
-
const answer = (entry, position, start, stop) => {
|
|
110
|
+
const answer = (entry, position, start, stop, answeringHttp = false) => {
|
|
106
111
|
const now = Date.now();
|
|
107
112
|
reads.count += 1;
|
|
108
113
|
reads.bytes += stop - start + 1;
|
|
109
114
|
reads.firstAt ||= now;
|
|
110
115
|
reads.lastAt = now;
|
|
111
116
|
const [from] = entry.source.input.ranges[position];
|
|
117
|
+
const file = files.get(entry.source.fileIndex);
|
|
118
|
+
if (file) {
|
|
119
|
+
// Packet learning reads only an already-owned memory copy; disk has no
|
|
120
|
+
// such copy. Its ordinary container reader already reads this file.
|
|
121
|
+
if (!answeringHttp) return null;
|
|
122
|
+
const stream = file.stream(start, stop, bytes => { diskBytes += bytes; });
|
|
123
|
+
return stream ? { length: file.length, end: stop, size: stop - start + 1, stream } : null;
|
|
124
|
+
}
|
|
125
|
+
if (storage) {
|
|
126
|
+
const held = storage.slices(entry.source, start, stop, answeringHttp);
|
|
127
|
+
if (!held) return null;
|
|
128
|
+
return { length: entry.source.input.fileLength, end: stop, size: stop - start + 1,
|
|
129
|
+
...(answeringHttp && held.buffers.length > 1 ? { chunks: held.buffers }
|
|
130
|
+
: { bytes: held.buffers.length === 1 ? held.buffers[0] : Buffer.concat(held.buffers) }),
|
|
131
|
+
release: held.release };
|
|
132
|
+
}
|
|
112
133
|
return { length: entry.source.input.fileLength, end: stop, bytes: entry.buffers[position].subarray(start - from, stop - from + 1) };
|
|
113
134
|
};
|
|
135
|
+
const drop = () => {
|
|
136
|
+
owners--;
|
|
137
|
+
if (owners || released) return;
|
|
138
|
+
released = true;
|
|
139
|
+
held.clear();
|
|
140
|
+
releaseBudget();
|
|
141
|
+
void Promise.all([whole, ...[...files.values()].map(file => file.release())]).then(finishClose);
|
|
142
|
+
};
|
|
114
143
|
return {
|
|
115
144
|
kind: "result", original: true, sources, bytes, fingerprint: hash.digest("hex"),
|
|
116
145
|
// What the copy took so far; final once `whole` has settled.
|
|
117
|
-
get copyMs() { return copyMs; },
|
|
146
|
+
get copyMs() { return storage ? storage.stats.copyMs : copyMs; },
|
|
147
|
+
get copiedBytes() { return storage ? storage.stats.copiedBytes : bytes; },
|
|
148
|
+
get sharedBytes() { return storage ? storage.stats.sharedBytes : 0; },
|
|
149
|
+
get diskBytes() { return diskBytes; },
|
|
118
150
|
whole,
|
|
119
151
|
// Bytes of the stretches still to be copied when the answer was given.
|
|
120
152
|
deferredBytes: later.reduce((sum, job) => {
|
|
@@ -126,6 +158,13 @@ export async function admitOriginalInput({ sources, reserve, readRanges, now = D
|
|
|
126
158
|
// not where each track stops. The final piece is judged by the input
|
|
127
159
|
// reaching the end of the file instead (`piece-completeness.js`).
|
|
128
160
|
tracks: sources.flatMap(source => (source.input.selections ?? []).map(selection => ({ track: selection.track }))),
|
|
161
|
+
heldUntil(fileIndex, start) {
|
|
162
|
+
if (released) return null;
|
|
163
|
+
const entry = held.get(fileIndex);
|
|
164
|
+
if (!entry || files.has(fileIndex)) return null;
|
|
165
|
+
const range = entry.source.input.ranges.find(([first, last]) => first <= start && last >= start);
|
|
166
|
+
return range ? Math.min(range[1] + 1, storage?.endOf(entry.source, start) ?? range[1] + 1) : null;
|
|
167
|
+
},
|
|
129
168
|
/**
|
|
130
169
|
* The answer to a read from `start`, or null when it lies outside what
|
|
131
170
|
* the run holds. Inside a stretch still being copied the answer is a
|
|
@@ -141,12 +180,12 @@ export async function admitOriginalInput({ sources, reserve, readRanges, now = D
|
|
|
141
180
|
const position = ranges.findIndex(([from, to]) => from <= start && (partial ? start : stop) <= to);
|
|
142
181
|
if (position < 0) return null;
|
|
143
182
|
stop = Math.min(stop, ranges[position][1]);
|
|
144
|
-
if (entry.buffers[position] !== null) return answer(entry, position, start, stop);
|
|
183
|
+
if (entry.buffers[position] !== null) return answer(entry, position, start, stop, partial);
|
|
145
184
|
const wait = { from: Date.now(), to: null };
|
|
146
185
|
waits.push(wait);
|
|
147
186
|
return copied(entry, position).then(ok => {
|
|
148
187
|
wait.to = Date.now();
|
|
149
|
-
return ok && !released ? answer(entry, position, start, stop) : null;
|
|
188
|
+
return ok && !released ? answer(entry, position, start, stop, partial) : null;
|
|
150
189
|
});
|
|
151
190
|
},
|
|
152
191
|
/**
|
|
@@ -163,11 +202,16 @@ export async function admitOriginalInput({ sources, reserve, readRanges, now = D
|
|
|
163
202
|
noteAnswer(answer) {
|
|
164
203
|
if (!released) answers.push({ ...answer, at: Date.now() });
|
|
165
204
|
},
|
|
205
|
+
/** Packet learning can finish after the FFmpeg run that supplied it. */
|
|
206
|
+
retain() {
|
|
207
|
+
if (released || runReleased) return null;
|
|
208
|
+
owners++;
|
|
209
|
+
let dropped = false;
|
|
210
|
+
return () => { if (!dropped) { dropped = true; drop(); } };
|
|
211
|
+
},
|
|
166
212
|
release() {
|
|
167
|
-
if (
|
|
168
|
-
|
|
169
|
-
held.clear();
|
|
170
|
-
releaseBudget();
|
|
213
|
+
if (!runReleased) { runReleased = true; drop(); }
|
|
214
|
+
return closed;
|
|
171
215
|
}
|
|
172
216
|
};
|
|
173
217
|
} finally {
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
/** Copied, admitted file ranges shared by live inputs and HTTP responses. */
|
|
2
|
+
export class OriginalInputStore {
|
|
3
|
+
#files = new Map();
|
|
4
|
+
#owners = new Map();
|
|
5
|
+
#parts = new Set();
|
|
6
|
+
#changed;
|
|
7
|
+
|
|
8
|
+
constructor({ changed = () => {} } = {}) { this.#changed = changed; }
|
|
9
|
+
|
|
10
|
+
get bytes() { return [...this.#parts].reduce((sum, part) => sum + (part.charged ? sizeOf(part) : 0), 0); }
|
|
11
|
+
get required() { return [...this.#parts].reduce((sum, part) => sum + (part.charged && (part.refs || part.loading) ? sizeOf(part) : 0), 0); }
|
|
12
|
+
|
|
13
|
+
#file(source) {
|
|
14
|
+
const key = JSON.stringify([source.sourceKey, source.fileIndex]);
|
|
15
|
+
if (!this.#files.has(key)) this.#files.set(key, { key, parts: [], owners: new Set() });
|
|
16
|
+
return this.#files.get(key);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/** Keep common parts across seeks while an output of this file exists. */
|
|
20
|
+
retain(owner, sources) {
|
|
21
|
+
const files = this.#owners.get(owner) ?? new Set();
|
|
22
|
+
for (const source of sources) {
|
|
23
|
+
const file = this.#file(source);
|
|
24
|
+
file.owners.add(owner);
|
|
25
|
+
files.add(file);
|
|
26
|
+
}
|
|
27
|
+
this.#owners.set(owner, files);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
forget(owner) {
|
|
31
|
+
for (const file of this.#owners.get(owner) ?? []) {
|
|
32
|
+
file.owners.delete(owner);
|
|
33
|
+
for (const part of [...file.parts]) this.#discardUnused(part);
|
|
34
|
+
this.#prune(file);
|
|
35
|
+
}
|
|
36
|
+
this.#owners.delete(owner);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** Unused common bytes are wanted, never required, and can leave for room. */
|
|
40
|
+
trim(limit) {
|
|
41
|
+
let bytes = this.bytes;
|
|
42
|
+
for (const part of this.#parts) {
|
|
43
|
+
if (bytes <= limit) break;
|
|
44
|
+
if (part.refs || part.loading) continue;
|
|
45
|
+
bytes -= part.charged ? sizeOf(part) : 0;
|
|
46
|
+
this.#discard(part);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
#prune(file) {
|
|
51
|
+
if (!file.parts.length && !file.owners.size) this.#files.delete(file.key);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
#discard(part) {
|
|
55
|
+
part.charged = false;
|
|
56
|
+
part.buffer = null;
|
|
57
|
+
part.ensure = null;
|
|
58
|
+
this.#parts.delete(part);
|
|
59
|
+
const position = part.file.parts.indexOf(part);
|
|
60
|
+
if (position >= 0) part.file.parts.splice(position, 1);
|
|
61
|
+
this.#prune(part.file);
|
|
62
|
+
this.#changed();
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
#discardUnused(part) {
|
|
66
|
+
if (part.refs || part.loading) return;
|
|
67
|
+
if (!part.failed && part.buffer && part.common && part.file.owners.size) return;
|
|
68
|
+
this.#discard(part);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
#unpin(parts) {
|
|
72
|
+
let released = false;
|
|
73
|
+
return () => {
|
|
74
|
+
if (released) return;
|
|
75
|
+
released = true;
|
|
76
|
+
for (const part of parts) { part.refs--; this.#discardUnused(part); }
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Protect existing bytes during an asynchronous budget revision. */
|
|
81
|
+
protect(sources) {
|
|
82
|
+
const parts = new Set();
|
|
83
|
+
for (const source of sources) {
|
|
84
|
+
for (const part of this.#file(source).parts) {
|
|
85
|
+
if (source.input.ranges.some(([start, end]) => start <= part.end && end >= part.start)) parts.add(part);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
for (const part of parts) part.refs++;
|
|
89
|
+
return this.#unpin(parts);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
available(source) {
|
|
93
|
+
const file = this.#files.get(JSON.stringify([source.sourceKey, source.fileIndex]));
|
|
94
|
+
return (file?.parts ?? []).filter(part => !part.failed).map(part => [part.start, part.end]);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
#missing(source) {
|
|
98
|
+
const file = this.#file(source);
|
|
99
|
+
const missing = [];
|
|
100
|
+
for (const [start, end] of source.input.ranges) {
|
|
101
|
+
let at = start;
|
|
102
|
+
for (const part of file.parts) {
|
|
103
|
+
if (part.end < at || part.failed) continue;
|
|
104
|
+
if (part.start > end) break;
|
|
105
|
+
if (at < part.start) missing.push([at, Math.min(end, part.start - 1)]);
|
|
106
|
+
at = Math.max(at, part.end + 1);
|
|
107
|
+
if (at > end) break;
|
|
108
|
+
}
|
|
109
|
+
if (at <= end) missing.push([at, end]);
|
|
110
|
+
}
|
|
111
|
+
return missing;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
bytesNeeded(sources) { return sources.reduce((sum, source) => sum + this.#missing(source).reduce((n, [start, end]) => n + end - start + 1, 0), 0); }
|
|
115
|
+
|
|
116
|
+
/** Reserve missing ranges synchronously; a second admission sees them pending. */
|
|
117
|
+
acquire(sources, room, { readRanges, now = Date.now }) {
|
|
118
|
+
const additionalBytes = this.bytesNeeded(sources);
|
|
119
|
+
if (additionalBytes > room) return null;
|
|
120
|
+
const parts = new Set();
|
|
121
|
+
const owned = new Set();
|
|
122
|
+
const stats = { copiedBytes: 0, sharedBytes: 0, copyMs: 0 };
|
|
123
|
+
for (const source of sources) {
|
|
124
|
+
const file = this.#file(source);
|
|
125
|
+
for (const [start, end] of this.#missing(source)) {
|
|
126
|
+
// A common prefix must never retain the rest of a large media buffer.
|
|
127
|
+
const cuts = new Set([start, end + 1]);
|
|
128
|
+
for (const [first, last] of source.input.commonRanges ?? []) {
|
|
129
|
+
if (first > start && first <= end) cuts.add(first);
|
|
130
|
+
if (last >= start && last < end) cuts.add(last + 1);
|
|
131
|
+
}
|
|
132
|
+
const offsets = [...cuts].sort((a, b) => a - b);
|
|
133
|
+
for (let i = 1; i < offsets.length; i++) {
|
|
134
|
+
const first = offsets[i - 1], last = offsets[i] - 1;
|
|
135
|
+
const part = { file, source, start: first, end: last, refs: 0, charged: true,
|
|
136
|
+
common: (source.input.commonRanges ?? []).some(([a, b]) => a <= first && b >= last),
|
|
137
|
+
buffer: null, loading: false, failed: false, promise: null, ensure: null };
|
|
138
|
+
file.parts.push(part);
|
|
139
|
+
this.#parts.add(part);
|
|
140
|
+
owned.add(part);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
file.parts.sort((a, b) => a.start - b.start);
|
|
144
|
+
for (const part of file.parts) {
|
|
145
|
+
for (const [start, end] of source.input.ranges) {
|
|
146
|
+
const bytes = Math.max(0, Math.min(end, part.end) - Math.max(start, part.start) + 1);
|
|
147
|
+
if (!bytes) continue;
|
|
148
|
+
parts.add(part);
|
|
149
|
+
if (!owned.has(part)) stats.sharedBytes += bytes;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
for (const part of parts) part.refs++;
|
|
154
|
+
let released = false;
|
|
155
|
+
const unpin = this.#unpin(parts);
|
|
156
|
+
const copy = async (source, ranges) => {
|
|
157
|
+
const selected = [...parts].filter(part => part.file.key === JSON.stringify([source.sourceKey, source.fileIndex]) &&
|
|
158
|
+
ranges.some(([start, end]) => start <= part.end && end >= part.start));
|
|
159
|
+
const fresh = selected.filter(part => owned.has(part) && !part.promise);
|
|
160
|
+
if (fresh.length) {
|
|
161
|
+
for (const part of fresh) part.loading = true;
|
|
162
|
+
const startedAt = now();
|
|
163
|
+
// Defer the read until every part names the same pending operation.
|
|
164
|
+
const operation = Promise.resolve().then(() => readRanges(source, fresh.map(part => [part.start, part.end]), additionalBytes))
|
|
165
|
+
.then(buffers => {
|
|
166
|
+
if (buffers === null) return false;
|
|
167
|
+
if (!Array.isArray(buffers) || buffers.length !== fresh.length || buffers.some((buffer, i) => !Buffer.isBuffer(buffer) || buffer.length !== sizeOf(fresh[i]))) {
|
|
168
|
+
throw new Error("Original input reader returned incomplete ranges.");
|
|
169
|
+
}
|
|
170
|
+
for (let i = 0; i < fresh.length; i++) fresh[i].buffer = buffers[i];
|
|
171
|
+
stats.copiedBytes += fresh.reduce((sum, part) => sum + sizeOf(part), 0);
|
|
172
|
+
return true;
|
|
173
|
+
}).catch(error => { for (const part of fresh) part.error = error; return false; }).then(ok => {
|
|
174
|
+
stats.copyMs += Math.max(0, now() - startedAt);
|
|
175
|
+
for (const part of fresh) {
|
|
176
|
+
part.loading = false;
|
|
177
|
+
if (!ok) { part.failed = true; this.#discard(part); }
|
|
178
|
+
else this.#discardUnused(part);
|
|
179
|
+
}
|
|
180
|
+
return ok;
|
|
181
|
+
});
|
|
182
|
+
for (const part of fresh) { part.promise = operation; part.ensure = null; }
|
|
183
|
+
}
|
|
184
|
+
for (const part of selected) if (!part.promise) part.ensure();
|
|
185
|
+
const answers = await Promise.all(selected.map(part => part.promise));
|
|
186
|
+
const failure = selected.find(part => part.error);
|
|
187
|
+
if (failure) throw failure.error;
|
|
188
|
+
return !released && answers.every(Boolean);
|
|
189
|
+
};
|
|
190
|
+
for (const part of owned) part.ensure = () => copy(part.source, [[part.start, part.end]]);
|
|
191
|
+
return {
|
|
192
|
+
stats, additionalBytes,
|
|
193
|
+
copy,
|
|
194
|
+
endOf(source, start) {
|
|
195
|
+
if (released) return null;
|
|
196
|
+
const part = [...parts].find(part => part.file.key === JSON.stringify([source.sourceKey, source.fileIndex]) &&
|
|
197
|
+
part.buffer && part.start <= start && part.end >= start);
|
|
198
|
+
return part ? part.end + 1 : null;
|
|
199
|
+
},
|
|
200
|
+
slices(source, start, end, pin = false) {
|
|
201
|
+
if (released) return null;
|
|
202
|
+
const selected = [...parts].filter(part => part.file.key === JSON.stringify([source.sourceKey, source.fileIndex]) && part.start <= end && part.end >= start)
|
|
203
|
+
.sort((a, b) => a.start - b.start);
|
|
204
|
+
if (selected.some(part => !part.buffer || part.failed)) return null;
|
|
205
|
+
const buffers = selected.map(part => part.buffer.subarray(Math.max(start, part.start) - part.start, Math.min(end, part.end) - part.start + 1));
|
|
206
|
+
if (buffers.reduce((sum, buffer) => sum + buffer.length, 0) !== end - start + 1) return null;
|
|
207
|
+
if (pin) for (const part of selected) part.refs++;
|
|
208
|
+
return { buffers, release: pin ? this.releaseParts(selected) : () => {} };
|
|
209
|
+
},
|
|
210
|
+
// Bound separately because slices' `this` is the lease, not the store.
|
|
211
|
+
releaseParts: selected => this.#unpin(selected),
|
|
212
|
+
release() { if (!released) { released = true; unpin(); } }
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
const sizeOf = part => part.end - part.start + 1;
|
|
@@ -275,9 +275,10 @@ export class AviContainer extends Container {
|
|
|
275
275
|
// media time: every stream's packets in file order, from the first.
|
|
276
276
|
const search = index.mediaRanges({ from: 0, to: FFMPEG_STREAM_SEARCH_SECONDS, picture: null,
|
|
277
277
|
streams: requested.map(track => track.trackNumber) });
|
|
278
|
-
const
|
|
279
|
-
|
|
280
|
-
|
|
278
|
+
const commonRanges = [[0, movi.start + 3], ...index.indexRanges, ...index.firstPackets(), ...search].map(padded);
|
|
279
|
+
const intervalRanges = index.mediaRanges({ from: interval.from, to: interval.to, picture: picture?.trackNumber ?? null,
|
|
280
|
+
streams: requested.map(track => track.trackNumber) }).map(padded);
|
|
281
|
+
const ranges = [...commonRanges, ...intervalRanges];
|
|
281
282
|
ranges.sort((left, right) => left[0] - right[0]);
|
|
282
283
|
const union = [];
|
|
283
284
|
for (const range of ranges) {
|
|
@@ -285,7 +286,7 @@ export class AviContainer extends Container {
|
|
|
285
286
|
if (previous && range[0] <= previous[1] + 1) previous[1] = Math.max(previous[1], range[1]);
|
|
286
287
|
else union.push([...range]);
|
|
287
288
|
}
|
|
288
|
-
return { kind: "result", from: interval.from, to: interval.to, ranges: union, fileLength: this.fileSize,
|
|
289
|
+
return { kind: "result", from: interval.from, to: interval.to, ranges: union, commonRanges, intervalRanges, fileLength: this.fileSize,
|
|
289
290
|
streamSearchSeconds: FFMPEG_STREAM_SEARCH_SECONDS };
|
|
290
291
|
}
|
|
291
292
|
|
|
@@ -761,8 +761,8 @@ export class MatroskaContainer extends Container {
|
|
|
761
761
|
// media time and stops on the first packet past it, which lies before the
|
|
762
762
|
// first Cue later than that time; the padding holds its read-ahead.
|
|
763
763
|
const searchEnd = points[firstAt(points[0].seconds + FFMPEG_STREAM_SEARCH_SECONDS, true)];
|
|
764
|
-
const
|
|
765
|
-
[
|
|
764
|
+
const commonRanges = [[0, paddedEnd((searchEnd ? searchEnd.at : layout.segmentEnd) - 1)],
|
|
765
|
+
[layout.cuesAt, paddedEnd(cuesEnd - 1)],
|
|
766
766
|
[Math.max(0, this.fileSize - requestBytes), this.fileSize - 1]];
|
|
767
767
|
// SeekHead may name top-level declarations after media clusters. Preserve
|
|
768
768
|
// their original bytes instead of reconstructing declarations for FFmpeg.
|
|
@@ -770,8 +770,10 @@ export class MatroskaContainer extends Container {
|
|
|
770
770
|
if (at === null || at < layout.firstClusterAt) continue;
|
|
771
771
|
const end = await elementEnd(at);
|
|
772
772
|
if (end === null) return { kind: "terminal", reason: "source-declarations-are-unbounded" };
|
|
773
|
-
|
|
773
|
+
commonRanges.push([at, paddedEnd(end - 1)]);
|
|
774
774
|
}
|
|
775
|
+
const intervalRanges = [[points[first].at, paddedEnd(mediaEnd)]];
|
|
776
|
+
const ranges = [...commonRanges, ...intervalRanges];
|
|
775
777
|
ranges.sort((left, right) => left[0] - right[0]);
|
|
776
778
|
const union = [];
|
|
777
779
|
for (const range of ranges) {
|
|
@@ -779,7 +781,7 @@ export class MatroskaContainer extends Container {
|
|
|
779
781
|
if (previous && range[0] <= previous[1] + 1) previous[1] = Math.max(previous[1], range[1]);
|
|
780
782
|
else union.push([...range]);
|
|
781
783
|
}
|
|
782
|
-
return { kind: "result", from: interval.from, to: interval.to, ranges: union, fileLength: this.fileSize,
|
|
784
|
+
return { kind: "result", from: interval.from, to: interval.to, ranges: union, commonRanges, intervalRanges, fileLength: this.fileSize,
|
|
783
785
|
streamSearchSeconds: FFMPEG_STREAM_SEARCH_SECONDS };
|
|
784
786
|
}
|
|
785
787
|
|
|
@@ -230,6 +230,7 @@ export function wireOutputs({
|
|
|
230
230
|
readMetadataActivity = null,
|
|
231
231
|
sourceInputsFor = null,
|
|
232
232
|
readEncodeRanges = null,
|
|
233
|
+
openEncodeWholeFile = null,
|
|
233
234
|
stretchCopied = () => {},
|
|
234
235
|
readEncodeHeldRanges = null,
|
|
235
236
|
encodeInputs = null,
|
|
@@ -1117,6 +1118,7 @@ export function wireOutputs({
|
|
|
1117
1118
|
parts.encodeInputs = encodeInputs ?? (typeof resolveEncodeInput === "function" ? new EncodeInputs({
|
|
1118
1119
|
resolve: resolveEncodeInput,
|
|
1119
1120
|
readRanges: readEncodeRanges,
|
|
1121
|
+
openWholeFile: openEncodeWholeFile,
|
|
1120
1122
|
heldRanges: readEncodeHeldRanges,
|
|
1121
1123
|
stretchCopied,
|
|
1122
1124
|
reviseBudget: () => parts.machineBudget.revise("memory", "admitted encode inputs"),
|
|
@@ -54,6 +54,8 @@ export class CompletedFiles {
|
|
|
54
54
|
|
|
55
55
|
/** Assemblies in flight, so two passes cannot write one file at once. @type {Set<string>} */
|
|
56
56
|
#writing = new Set();
|
|
57
|
+
#readers = new Map();
|
|
58
|
+
#removeAfterRead = new Set();
|
|
57
59
|
|
|
58
60
|
/**
|
|
59
61
|
* What this store may hold, handed to it by the owner of the disk.
|
|
@@ -140,6 +142,7 @@ export class CompletedFiles {
|
|
|
140
142
|
*/
|
|
141
143
|
find(infoHash, fileIndex) {
|
|
142
144
|
const key = this.#keyOf(infoHash, fileIndex);
|
|
145
|
+
if (this.#removeAfterRead.has(key)) return null;
|
|
143
146
|
const file = this.#held.get(key) ?? null;
|
|
144
147
|
if (file) {
|
|
145
148
|
// WHEN IT WAS LAST WANTED, which is the order it leaves in. A whole file
|
|
@@ -150,6 +153,26 @@ export class CompletedFiles {
|
|
|
150
153
|
return file;
|
|
151
154
|
}
|
|
152
155
|
|
|
156
|
+
/** Pin a completed file against eviction until all its offset reads finish. */
|
|
157
|
+
acquire(infoHash, fileIndex) {
|
|
158
|
+
const key = this.#keyOf(infoHash, fileIndex);
|
|
159
|
+
const file = this.find(infoHash, fileIndex);
|
|
160
|
+
if (!file) return null;
|
|
161
|
+
this.#readers.set(key, (this.#readers.get(key) ?? 0) + 1);
|
|
162
|
+
let released = false;
|
|
163
|
+
return { ...file, release: async () => {
|
|
164
|
+
if (released) return { bytes: this.bytes, removed: 0 };
|
|
165
|
+
released = true;
|
|
166
|
+
const remaining = this.#readers.get(key) - 1;
|
|
167
|
+
if (remaining) this.#readers.set(key, remaining);
|
|
168
|
+
else {
|
|
169
|
+
this.#readers.delete(key);
|
|
170
|
+
if (this.#removeAfterRead.delete(key)) await this.#remove(key);
|
|
171
|
+
}
|
|
172
|
+
return this.allow(this.#allowanceBytes);
|
|
173
|
+
} };
|
|
174
|
+
}
|
|
175
|
+
|
|
153
176
|
/**
|
|
154
177
|
* Say how much disk these files may take between them.
|
|
155
178
|
*
|
|
@@ -204,7 +227,7 @@ export class CompletedFiles {
|
|
|
204
227
|
let oldest = null;
|
|
205
228
|
let oldestAt = Number.POSITIVE_INFINITY;
|
|
206
229
|
for (const key of this.#held.keys()) {
|
|
207
|
-
if (this.#writing.has(key)) {
|
|
230
|
+
if (this.#writing.has(key) || this.#readers.has(key)) {
|
|
208
231
|
continue;
|
|
209
232
|
}
|
|
210
233
|
const at = this.#lastRead.get(key) ?? 0;
|
|
@@ -223,6 +246,7 @@ export class CompletedFiles {
|
|
|
223
246
|
* @returns {Promise<void>}
|
|
224
247
|
*/
|
|
225
248
|
async #remove(key) {
|
|
249
|
+
if (this.#readers.has(key)) { this.#removeAfterRead.add(key); return; }
|
|
226
250
|
const file = this.#held.get(key);
|
|
227
251
|
this.#held.delete(key);
|
|
228
252
|
this.#lastRead.delete(key);
|
|
@@ -400,11 +424,11 @@ export class CompletedFiles {
|
|
|
400
424
|
async forget(infoHash) {
|
|
401
425
|
for (const key of [...this.#held.keys()]) {
|
|
402
426
|
if (key.startsWith(`${infoHash}/`)) {
|
|
403
|
-
this.#
|
|
427
|
+
await this.#remove(key);
|
|
404
428
|
}
|
|
405
429
|
}
|
|
406
|
-
|
|
407
|
-
() => undefined
|
|
408
|
-
|
|
430
|
+
if (![...this.#held.keys()].some(key => key.startsWith(`${infoHash}/`))) {
|
|
431
|
+
await fs.rm(path.join(this.#root, String(infoHash)), { ...REMOVAL, recursive: true }).catch(() => undefined);
|
|
432
|
+
}
|
|
409
433
|
}
|
|
410
434
|
}
|
package/services/storage/wire.js
CHANGED
|
@@ -147,6 +147,7 @@ export function wireMachineBudget({
|
|
|
147
147
|
name: "whole files",
|
|
148
148
|
resource: filesOn,
|
|
149
149
|
held: () => wholeFiles.held?.() ?? 0,
|
|
150
|
+
required: () => wholeFiles.required?.() ?? 0,
|
|
150
151
|
// A whole file is worth keeping while anybody may read it again, and
|
|
151
152
|
// there is no smaller honest answer than "all of them" — like the rest,
|
|
152
153
|
// it is cut in proportion.
|
|
@@ -21,6 +21,7 @@ import { fileURLToPath } from "node:url";
|
|
|
21
21
|
import { logger, writeAlreadyDecided } from "../../../utils/logger.js";
|
|
22
22
|
import { createCaller, createReceiveStream, workerReplyError } from "./channel.js";
|
|
23
23
|
import { Command, Event } from "./protocol.js";
|
|
24
|
+
import { openCompletedInputFile } from "./completed-input-file.js";
|
|
24
25
|
|
|
25
26
|
const WORKER_URL = new URL("./worker.js", import.meta.url);
|
|
26
27
|
/**
|
|
@@ -106,6 +107,29 @@ export class TorrentWorkerClient {
|
|
|
106
107
|
* @type {Map<string, { path: string, length: number, name: string }>}
|
|
107
108
|
*/
|
|
108
109
|
wholeFiles = new Map();
|
|
110
|
+
#wholeInputPins = new Map();
|
|
111
|
+
|
|
112
|
+
get wholeFileRequiredBytes() {
|
|
113
|
+
const files = new Map();
|
|
114
|
+
for (const file of this.#wholeInputPins.values()) files.set(JSON.stringify([file.sourceKey, file.fileIndex]), file.length);
|
|
115
|
+
return [...files.values()].reduce((sum, length) => sum + length, 0);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** No source is opened or revived: only the worker's completed-file store answers. */
|
|
119
|
+
async acquireWholeFile({ sourceKey, fileIndex }) {
|
|
120
|
+
if (!this.#wholeFileOf(sourceKey, fileIndex)) return null;
|
|
121
|
+
const file = await this.#caller.call(Command.ACQUIRE_WHOLE_FILE, { sourceKey, fileIndex });
|
|
122
|
+
if (!file) {
|
|
123
|
+
this.wholeFiles.delete(`${String(sourceKey).slice("torrent:".length).toLowerCase()}/${fileIndex}`);
|
|
124
|
+
return null;
|
|
125
|
+
}
|
|
126
|
+
this.#wholeInputPins.set(file.token, { sourceKey, fileIndex, length: file.length });
|
|
127
|
+
this.wholeFileBytes = file.heldBytes;
|
|
128
|
+
return openCompletedInputFile(file, async () => {
|
|
129
|
+
try { this.wholeFileBytes = await this.#caller.call(Command.RELEASE_WHOLE_FILE, { token: file.token }); }
|
|
130
|
+
finally { this.#wholeInputPins.delete(file.token); }
|
|
131
|
+
});
|
|
132
|
+
}
|
|
109
133
|
|
|
110
134
|
/** @type {(event: { sourceKey: string, fileIndex: number, trackIndex: number, cues: object[], language: string }) => void} */
|
|
111
135
|
/** @type {(event: { sourceKey: string, fileIndexes: number[] }) => void} */
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { open } from "node:fs/promises";
|
|
2
|
+
import { Readable } from "node:stream";
|
|
3
|
+
|
|
4
|
+
/** Open one pinned completed file; every HTTP response owns a separate read. */
|
|
5
|
+
export async function openCompletedInputFile(file, release) {
|
|
6
|
+
let handle;
|
|
7
|
+
try {
|
|
8
|
+
handle = await open(file.path, "r");
|
|
9
|
+
if ((await handle.stat()).size !== file.length) throw new Error("Completed file length changed.");
|
|
10
|
+
} catch {
|
|
11
|
+
await handle?.close();
|
|
12
|
+
await release();
|
|
13
|
+
return null;
|
|
14
|
+
}
|
|
15
|
+
let references = 1, released = false, closing = false;
|
|
16
|
+
let finish;
|
|
17
|
+
const closed = new Promise(resolve => { finish = resolve; });
|
|
18
|
+
const drop = () => {
|
|
19
|
+
references--;
|
|
20
|
+
if (references || closing) return;
|
|
21
|
+
closing = true;
|
|
22
|
+
void handle.close().finally(release).then(() => finish(), error => finish(error));
|
|
23
|
+
};
|
|
24
|
+
return {
|
|
25
|
+
length: file.length, closed,
|
|
26
|
+
stream(start, end, onRead) {
|
|
27
|
+
if (released || !Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start < 0 || end < start || end >= file.length) return null;
|
|
28
|
+
references++;
|
|
29
|
+
let dropped = false;
|
|
30
|
+
const done = () => { if (!dropped) { dropped = true; drop(); } };
|
|
31
|
+
const stream = Readable.from((async function* () {
|
|
32
|
+
for (let at = start; at <= end;) {
|
|
33
|
+
const buffer = Buffer.allocUnsafe(Math.min(64 * 1024, end - at + 1));
|
|
34
|
+
const { bytesRead } = await handle.read(buffer, 0, buffer.length, at);
|
|
35
|
+
if (!bytesRead) throw new Error("Completed file ended inside an admitted range.");
|
|
36
|
+
at += bytesRead;
|
|
37
|
+
onRead(bytesRead);
|
|
38
|
+
yield buffer.subarray(0, bytesRead);
|
|
39
|
+
}
|
|
40
|
+
})(), { objectMode: false, highWaterMark: 64 * 1024 });
|
|
41
|
+
stream.once("end", done);
|
|
42
|
+
stream.once("close", done);
|
|
43
|
+
return stream;
|
|
44
|
+
},
|
|
45
|
+
release() {
|
|
46
|
+
if (!released) { released = true; drop(); }
|
|
47
|
+
return closed;
|
|
48
|
+
}
|
|
49
|
+
};
|
|
50
|
+
}
|
|
@@ -122,6 +122,10 @@ export class WorkerTorrentPool {
|
|
|
122
122
|
return this.#client.wholeFiles;
|
|
123
123
|
}
|
|
124
124
|
|
|
125
|
+
get wholeFileRequiredBytes() { return this.#client.wholeFileRequiredBytes ?? 0; }
|
|
126
|
+
|
|
127
|
+
acquireWholeFile(source) { return this.#client.acquireWholeFile(source); }
|
|
128
|
+
|
|
125
129
|
allowWholeFileBytes(bytes) {
|
|
126
130
|
return this.#client.allowWholeFileBytes(bytes);
|
|
127
131
|
}
|
|
@@ -94,6 +94,9 @@ export const Command = {
|
|
|
94
94
|
READ_HELD: "read-held",
|
|
95
95
|
/** Acquire all source pieces of an admitted input before copying its ranges. */
|
|
96
96
|
READ_HELD_RANGES: "read-held-ranges",
|
|
97
|
+
/** Pin a completed file; only a matching release permits its eviction. */
|
|
98
|
+
ACQUIRE_WHOLE_FILE: "acquire-whole-file",
|
|
99
|
+
RELEASE_WHOLE_FILE: "release-whole-file",
|
|
97
100
|
|
|
98
101
|
/** Shut the client down, optionally deleting downloaded data. */
|
|
99
102
|
/**
|
|
@@ -595,6 +595,22 @@ async function runCommand(command, params, id) {
|
|
|
595
595
|
return memoryClaim();
|
|
596
596
|
}
|
|
597
597
|
|
|
598
|
+
case Command.ACQUIRE_WHOLE_FILE: {
|
|
599
|
+
const hash = String(params.sourceKey).startsWith("torrent:") ? params.sourceKey.slice("torrent:".length).toLowerCase() : null;
|
|
600
|
+
const lease = hash === null ? null : completedFiles.acquire(hash, params.fileIndex);
|
|
601
|
+
if (!lease) return null;
|
|
602
|
+
wholeFileReaders.set(id, lease);
|
|
603
|
+
return { token: id, path: lease.path, length: lease.length, name: lease.name, heldBytes: completedFiles.bytes };
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
case Command.RELEASE_WHOLE_FILE: {
|
|
607
|
+
const lease = wholeFileReaders.get(params.token);
|
|
608
|
+
wholeFileReaders.delete(params.token);
|
|
609
|
+
const after = await lease?.release();
|
|
610
|
+
if (after?.removed) holdingsChanged();
|
|
611
|
+
return completedFiles.bytes;
|
|
612
|
+
}
|
|
613
|
+
|
|
598
614
|
case Command.WHOLE_FILES_ALLOWANCE: {
|
|
599
615
|
// Whole files are held on this thread and the disk has one owner, on the
|
|
600
616
|
// other. What arrives is their share; what goes back is what they hold
|
|
@@ -980,6 +996,7 @@ function holdingsChanged() {
|
|
|
980
996
|
* thread writes them, the main thread serves them without asking anybody.
|
|
981
997
|
*/
|
|
982
998
|
const completedFiles = new CompletedFiles({ root: completedFilesRoot() });
|
|
999
|
+
const wholeFileReaders = new Map();
|
|
983
1000
|
const wholeSources = createWholeSources({ find: (infoHash, fileIndex) => completedFiles.find(infoHash, fileIndex) });
|
|
984
1001
|
|
|
985
1002
|
/** Adopted files and newly completed files state the same availability event. */
|