@torrent-tv/proxy 2.97.3 → 2.97.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/services/encode/AdmittedInput.js +6 -6
- package/services/encode/EncodeRuns.js +35 -11
- package/services/encode/InputFailures.js +51 -4
- package/services/encode/OriginalInput.js +6 -1
- package/services/encode/piece-completeness.js +23 -0
- package/services/media/container/AviContainer.js +20 -1
- package/services/storage/IndexMemory.js +33 -26
- package/services/viewer/DownloadMaps.js +54 -7
package/package.json
CHANGED
|
@@ -28,12 +28,12 @@ export async function admitInput({ sources, reserve, readRanges }) {
|
|
|
28
28
|
!Buffer.isBuffer(buffer) || buffer.length !== source.input.ranges[index][1] - source.input.ranges[index][0] + 1)) {
|
|
29
29
|
throw new Error("Available-only input reader returned incomplete ranges.");
|
|
30
30
|
}
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
31
|
+
// Named by the source and what is stated about its packets below, not by
|
|
32
|
+
// the bytes: verified pieces and strict reads make the same ranges of the
|
|
33
|
+
// same source the same bytes (torrent-tv/meta#178). The check that each
|
|
34
|
+
// packet's address holds its expected payload (`expectedHash`) stays: a
|
|
35
|
+
// verified piece does not prove an address inside it was worked out right.
|
|
36
|
+
hash.update(JSON.stringify([source.sourceKey, source.fileIndex, source.input.ranges]));
|
|
37
37
|
for (const input of source.input.tracks) {
|
|
38
38
|
const shifted = { ...input,
|
|
39
39
|
...(Number.isFinite(input.sourceEndSeconds) ? { sourceEndSeconds: input.sourceEndSeconds - (source.timeShiftSeconds ?? 0) } : {}),
|
|
@@ -17,7 +17,7 @@ import { computeOutputDimensions } from "./args.js";
|
|
|
17
17
|
import { buildAdmittedCommand } from "./admitted-command.js";
|
|
18
18
|
import { buildOriginalCommand } from "./source-command.js";
|
|
19
19
|
import { writeAdmittedInput } from "./AdmittedInput.js";
|
|
20
|
-
import { cutOf, cutShiftOf, judgeNeighbors, judgePiece } from "./piece-completeness.js";
|
|
20
|
+
import { cutOf, cutShiftOf, judgeNeighbors, judgePiece, onItsCutClock } from "./piece-completeness.js";
|
|
21
21
|
import { presentationSegment } from "./segment-formats/presentation-segment.js";
|
|
22
22
|
import { InputFailures, failedAdmittedInput } from "./InputFailures.js";
|
|
23
23
|
import { readProcessCpuSeconds } from "./host-load.js";
|
|
@@ -302,7 +302,13 @@ export class EncodeRuns {
|
|
|
302
302
|
runStateOf(output) {
|
|
303
303
|
if (this.#host.encodeInputs?.failureOf(output)) return ENCODE_RUN_STATE.ENDED_FAILED;
|
|
304
304
|
if (output && this.liveRunsOf(output).length === 0 && this.#inputFailures.reason(output.outputKey)) return ENCODE_RUN_STATE.ENDED_FAILED;
|
|
305
|
-
|
|
305
|
+
const state = output ? this.#host.encodeOrchestrator.stateOf(output.outputKey) : runStateOf([]);
|
|
306
|
+
// A piece of a longer input failed and is to be made from an input holding
|
|
307
|
+
// it alone: production has not ended, and a request for it is held rather
|
|
308
|
+
// than refused (torrent-tv/meta#178).
|
|
309
|
+
if (state === ENCODE_RUN_STATE.ENDED_FAILED && this.liveRunsOf(output).length === 0 &&
|
|
310
|
+
this.#inputFailures.retrying(output.outputKey)) return ENCODE_RUN_STATE.IDLE;
|
|
311
|
+
return state;
|
|
306
312
|
}
|
|
307
313
|
|
|
308
314
|
lastErrorOf(output) {
|
|
@@ -374,6 +380,9 @@ export class EncodeRuns {
|
|
|
374
380
|
return null;
|
|
375
381
|
}
|
|
376
382
|
const address = session.outputKey ?? "";
|
|
383
|
+
// `bytes` is the piece the proxy cut again by presentation time at the
|
|
384
|
+
// published grid; without it the piece is as the muxer cut it.
|
|
385
|
+
const cutByProxy = bytes !== null;
|
|
377
386
|
try {
|
|
378
387
|
const mediaRanges = format.readMediaRanges(bytes ?? this.#host.segmentFiles.closedBytesOf(address, name));
|
|
379
388
|
|
|
@@ -390,16 +399,27 @@ export class EncodeRuns {
|
|
|
390
399
|
// piece has closed, and holds for every piece after it. It is a fact
|
|
391
400
|
// of each piece the run makes, so it travels with the piece.
|
|
392
401
|
if (run.cutShiftSeconds === null && index === run.from) {
|
|
393
|
-
|
|
402
|
+
// Read from the piece as the muxer closed it: a piece cut again by
|
|
403
|
+
// the proxy starts where the grid says, whatever the muxer did.
|
|
404
|
+
const asMuxed = cutByProxy ? format.readMediaRanges(this.#host.segmentFiles.closedBytesOf(address, name)) : mediaRanges;
|
|
405
|
+
run.cutShiftSeconds = cutShiftOf(asMuxed, grid?.[run.from], session.spec.carries !== "audio-only" ? "vide" : "soun",
|
|
394
406
|
timing?.endMicros ?? null);
|
|
407
|
+
// Said once per run: a piece judged on the wrong clock is refused by
|
|
408
|
+
// a millisecond, and nothing else in the log says which clock it was
|
|
409
|
+
// judged on (torrent-tv/meta#178).
|
|
410
|
+
this.#host.logger.info(`encode-run #${run.from}..#${run.to} on ${address}: its muxer counts cuts ` +
|
|
411
|
+
(run.cutShiftSeconds === null ? "from a first packet more than a frame away from its start (shift unknown)"
|
|
412
|
+
: `${run.cutShiftSeconds.toFixed(6)}s from its start at ${grid?.[run.from]}s`) +
|
|
413
|
+
(cutByProxy ? "; its pieces are cut again by the proxy at the published grid, so the shift does not apply to them" : ""));
|
|
395
414
|
}
|
|
396
|
-
|
|
415
|
+
onItsCutClock(mediaRanges, run.cutShiftSeconds, cutByProxy);
|
|
397
416
|
}
|
|
398
417
|
const { whole, throughSeconds, reason } = judgePiece(format, mediaRanges, cut, interval);
|
|
399
418
|
if (!whole) {
|
|
400
419
|
this.#host.logger.warn(
|
|
401
420
|
`encode: not publishing piece ${index} of ${address.slice(0, 60)}: ` +
|
|
402
421
|
(reason ?? (throughSeconds === null ? "it holds no playable media" : `produced through ${throughSeconds}s, cut ${cut}s`)) +
|
|
422
|
+
` cutBy=${cutByProxy ? "proxy" : "muxer"} shift=${Number.isFinite(mediaRanges?.cutShiftSeconds) ? mediaRanges.cutShiftSeconds : "none"}` +
|
|
403
423
|
(interval ? ` expected=${interval.from}..${interval.to} tracks=${JSON.stringify(mediaRanges.tracks.map(track => ({
|
|
404
424
|
kind: track.kind, timescale: track.timescale, frame: track.productionFrame,
|
|
405
425
|
first: track.ranges[0], last: track.ranges.at(-1), count: track.ranges.length })),
|
|
@@ -809,8 +829,11 @@ export class EncodeRuns {
|
|
|
809
829
|
// really reaches is the input's answer. An original input is one copied
|
|
810
830
|
// stretch, and the run is given exactly that stretch (torrent-tv/meta#158).
|
|
811
831
|
const segmentCount = Number(session.timeline?.segmentCount) || 0;
|
|
812
|
-
|
|
813
|
-
|
|
832
|
+
// A piece whose longer input failed is made from an input holding it alone,
|
|
833
|
+
// so a run reaches no further than the number before it.
|
|
834
|
+
const mayReach = this.#inputFailures.boundFor(session.outputKey, startIndex,
|
|
835
|
+
Number.isInteger(ordered?.to) && ordered.to >= startIndex ? ordered.to
|
|
836
|
+
: segmentCount > startIndex ? segmentCount - 1 : startIndex);
|
|
814
837
|
const admittedInput = this.#host.encodeInputs?.take(session, startIndex, mayReach) ?? null;
|
|
815
838
|
if (!admittedInput) return null;
|
|
816
839
|
const inputEnd = admittedInput.original ? admittedInput.to : startIndex;
|
|
@@ -1039,7 +1062,6 @@ export class EncodeRuns {
|
|
|
1039
1062
|
});
|
|
1040
1063
|
if (admittedInput.original) {
|
|
1041
1064
|
run.originalInput = admittedInput;
|
|
1042
|
-
run.inputFingerprint = admittedInput.fingerprint;
|
|
1043
1065
|
// Every number of the stretch answers to its one key, so a failure
|
|
1044
1066
|
// anywhere inside it is recorded against the stretch that was commanded.
|
|
1045
1067
|
run.admittedInputKeys = new Map(Array.from({ length: inputEnd - safeIndex + 1 }, (_, offset) => [safeIndex + offset, inputKey]));
|
|
@@ -1052,7 +1074,6 @@ export class EncodeRuns {
|
|
|
1052
1074
|
// this run works and is ready when the plan places the next run there.
|
|
1053
1075
|
if (inputEnd < mayReach) this.#host.encodeInputs?.take(session, inputEnd + 1, mayReach);
|
|
1054
1076
|
} else if (admittedInput) {
|
|
1055
|
-
run.inputFingerprint = admittedInput.fingerprint;
|
|
1056
1077
|
run.admittedInputKeys = new Map([[safeIndex, inputKey]]);
|
|
1057
1078
|
let nextIndex = startIndex + 1;
|
|
1058
1079
|
const canAppend = index => this.isLive(session) && run.isAlive &&
|
|
@@ -1101,7 +1122,7 @@ export class EncodeRuns {
|
|
|
1101
1122
|
if (inputKey && failedAdmittedInput({ ending: ENCODE_EXIT.FAILED,
|
|
1102
1123
|
code: error.code ? null : 1, because: error.message },
|
|
1103
1124
|
session.spec.transcodesVideo && this.#host.videoEncoder.kind !== "software")) {
|
|
1104
|
-
this.#inputFailures.note(session.outputKey, startIndex, inputKey, error.message);
|
|
1125
|
+
this.#inputFailures.note(session.outputKey, startIndex, inputKey, error.message, { pieces: inputEnd - startIndex + 1 });
|
|
1105
1126
|
this.#host.invalidateWaits(session);
|
|
1106
1127
|
this.#host.productionFailed(session);
|
|
1107
1128
|
}
|
|
@@ -1236,8 +1257,11 @@ export class EncodeRuns {
|
|
|
1236
1257
|
const index = run.head;
|
|
1237
1258
|
const key = run.admittedInputKeys.get(index);
|
|
1238
1259
|
if (key) {
|
|
1239
|
-
|
|
1240
|
-
this.#
|
|
1260
|
+
const pieces = [...run.admittedInputKeys.values()].filter(held => held === key).length;
|
|
1261
|
+
this.#inputFailures.note(session.outputKey, index, key, ended.because, { pieces });
|
|
1262
|
+
this.#host.logger.warn(pieces > 1
|
|
1263
|
+
? `encode: piece ${index} will be made again from an input holding it alone output=${session.outputKey} input=${key} (held ${pieces} pieces) reason=${ended.because}`
|
|
1264
|
+
: `encode: unchanged input will not be retried output=${session.outputKey} segment=${index} input=${key} reason=${ended.because}`);
|
|
1241
1265
|
}
|
|
1242
1266
|
}
|
|
1243
1267
|
this.#host.encodeOrchestrator.noteEnded(ended);
|
|
@@ -12,7 +12,18 @@ export function failedAdmittedInput(ended, hardwareEncode = false) {
|
|
|
12
12
|
return ended.ending === ENCODE_EXIT.SHORT || ended.ending === ENCODE_EXIT.FAILED;
|
|
13
13
|
}
|
|
14
14
|
|
|
15
|
-
/**
|
|
15
|
+
/**
|
|
16
|
+
* A failed input is retried only after its identity — source, file, ranges and
|
|
17
|
+
* what is stated about them — or the output parameters change.
|
|
18
|
+
*
|
|
19
|
+
* A failure of an input that held more pieces than the one that failed is not
|
|
20
|
+
* the end of the output: the piece is made again from an input holding it
|
|
21
|
+
* alone, which is another input (`retryAlone`). Only the failure of that input
|
|
22
|
+
* — or of any input holding one piece — ends production. Field 2026-10-11,
|
|
23
|
+
* torrent-tv/meta#178: one refused piece of a six-piece copy ended the viewer's
|
|
24
|
+
* playback with a 500 while the same piece made alone was published in an
|
|
25
|
+
* earlier session.
|
|
26
|
+
*/
|
|
16
27
|
export class InputFailures {
|
|
17
28
|
#failures = new Map();
|
|
18
29
|
|
|
@@ -28,11 +39,47 @@ export class InputFailures {
|
|
|
28
39
|
return null;
|
|
29
40
|
}
|
|
30
41
|
|
|
31
|
-
|
|
42
|
+
/** Why production of this output has ended, or null while every failed piece can still be made alone. */
|
|
43
|
+
reason(address) {
|
|
44
|
+
for (const entry of this.#failures.get(address)?.values() ?? []) if (!entry.retryAlone) return entry.reason;
|
|
45
|
+
return null;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** Whether a failed piece of this output waits to be made from an input holding it alone. */
|
|
49
|
+
retrying(address) {
|
|
50
|
+
for (const entry of this.#failures.get(address)?.values() ?? []) if (entry.retryAlone) return true;
|
|
51
|
+
return false;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* The last number a run starting at `from` may reach so that a piece waiting
|
|
56
|
+
* to be made alone is not taken again inside a longer input: the number
|
|
57
|
+
* before it, or `from` itself when the run starts there.
|
|
58
|
+
*
|
|
59
|
+
* @param {string} address
|
|
60
|
+
* @param {number} from
|
|
61
|
+
* @param {number} to
|
|
62
|
+
* @returns {number}
|
|
63
|
+
*/
|
|
64
|
+
boundFor(address, from, to) {
|
|
65
|
+
let bound = to;
|
|
66
|
+
for (const [index, entry] of this.#failures.get(address) ?? []) {
|
|
67
|
+
if (!entry.retryAlone || index < from || index > bound) continue;
|
|
68
|
+
bound = index === from ? from : index - 1;
|
|
69
|
+
}
|
|
70
|
+
return bound;
|
|
71
|
+
}
|
|
32
72
|
|
|
33
|
-
|
|
73
|
+
/**
|
|
74
|
+
* @param {string} address
|
|
75
|
+
* @param {number} index - The piece that failed.
|
|
76
|
+
* @param {string} key
|
|
77
|
+
* @param {string} reason
|
|
78
|
+
* @param {{ pieces?: number }} [held] - How many pieces the failed input held.
|
|
79
|
+
*/
|
|
80
|
+
note(address, index, key, reason, { pieces = 1 } = {}) {
|
|
34
81
|
const entries = this.#failures.get(address) ?? new Map();
|
|
35
|
-
entries.set(index, { key, reason });
|
|
82
|
+
entries.set(index, { key, reason, retryAlone: pieces > 1 });
|
|
36
83
|
this.#failures.set(address, entries);
|
|
37
84
|
}
|
|
38
85
|
|
|
@@ -25,8 +25,13 @@ export async function admitOriginalInput({ sources, reserve, readRanges, now = D
|
|
|
25
25
|
!Buffer.isBuffer(buffer) || buffer.length !== source.input.ranges[index][1] - source.input.ranges[index][0] + 1)) {
|
|
26
26
|
throw new Error("Original input reader returned incomplete ranges.");
|
|
27
27
|
}
|
|
28
|
+
// THE INPUT IS NAMED BY WHAT IT IS, NOT BY ITS BYTES. Every source is a
|
|
29
|
+
// torrent whose pieces are verified before they are stored, and a read
|
|
30
|
+
// gives every byte or none, so the same source, file, ranges and
|
|
31
|
+
// selections are the same bytes. Hashing the bytes as well took 13.45 s
|
|
32
|
+
// of 180 s of the main thread on a Matroska film whose picture and sound
|
|
33
|
+
// copy the same ~30 MB a piece (torrent-tv/meta#178).
|
|
28
34
|
hash.update(JSON.stringify([source.sourceKey, source.fileIndex, source.input.ranges, source.input.selections]));
|
|
29
|
-
for (const buffer of buffers) hash.update(buffer);
|
|
30
35
|
held.set(source.fileIndex, { source, buffers });
|
|
31
36
|
}
|
|
32
37
|
let released = false;
|
|
@@ -70,6 +70,29 @@ export function judgePiece(format, ranges, cutSeconds, interval) {
|
|
|
70
70
|
return { whole: true, throughSeconds };
|
|
71
71
|
}
|
|
72
72
|
|
|
73
|
+
/**
|
|
74
|
+
* Put a piece on the clock it was cut by, for {@link judgePiece}.
|
|
75
|
+
*
|
|
76
|
+
* The run's cut shift ({@link cutShiftOf}) says where the segment MUXER counts
|
|
77
|
+
* its cuts from, so it belongs only to a piece the muxer cut. A copied picture
|
|
78
|
+
* read from the original file is cut again by the proxy, by presentation time
|
|
79
|
+
* at the published grid (`presentationSegment`), and that piece is judged on
|
|
80
|
+
* the grid as published. Field 2026-10-11, torrent-tv/meta#178: an open-GOP
|
|
81
|
+
* HEVC piece cut again at 2511.676..2521.686 holds the next keyframe, which its
|
|
82
|
+
* leading pictures need, and so ends one frame (672 of 16 000 ticks) past the
|
|
83
|
+
* cut, as it must; moved by the muxer's shift of −0.001 s the allowance ended
|
|
84
|
+
* one millisecond before that frame, the piece was refused and playback ended.
|
|
85
|
+
*
|
|
86
|
+
* @param {object | null | undefined} ranges - The piece's media intervals.
|
|
87
|
+
* @param {number | null} shift - The run's cut shift, or null when not known.
|
|
88
|
+
* @param {boolean} cutByProxy - Whether the proxy cut the piece again.
|
|
89
|
+
* @returns {object | null | undefined} The same intervals.
|
|
90
|
+
*/
|
|
91
|
+
export function onItsCutClock(ranges, shift, cutByProxy) {
|
|
92
|
+
if (ranges && !cutByProxy && Number.isFinite(shift)) ranges.cutShiftSeconds = shift;
|
|
93
|
+
return ranges;
|
|
94
|
+
}
|
|
95
|
+
|
|
73
96
|
/**
|
|
74
97
|
* How far after its requested start a run's segment muxer counts its cuts
|
|
75
98
|
* from, read from the run's FIRST piece, or `null` when that piece does not
|
|
@@ -13,6 +13,7 @@ import { VideoTrack } from "../tracks/VideoTrack.js";
|
|
|
13
13
|
import { AudioTrack } from "../tracks/AudioTrack.js";
|
|
14
14
|
import { ContainerTrack } from "../tracks/ContainerTrack.js";
|
|
15
15
|
import { PacketIndex } from "./PacketIndex.js";
|
|
16
|
+
import { IndexMemoryUnavailable } from "./memory-unavailable.js";
|
|
16
17
|
import { readAviIndex } from "./avi-index.js";
|
|
17
18
|
import { isUnavailable } from "./unavailable.js";
|
|
18
19
|
import { Mpeg4PictureTiming } from "./mpeg4-picture-timing.js";
|
|
@@ -34,6 +35,8 @@ export class AviContainer extends Container {
|
|
|
34
35
|
#packets = null;
|
|
35
36
|
/** The sound packets of an indexed file while they are being stated, so two asks build them once. */
|
|
36
37
|
#soundPacketsReading = null;
|
|
38
|
+
/** The bytes the sound packet table was refused, while memory still could not hold them. */
|
|
39
|
+
#soundPacketsRefused = null;
|
|
37
40
|
#scan = null;
|
|
38
41
|
#declarations = null;
|
|
39
42
|
#workTags = null;
|
|
@@ -330,9 +333,25 @@ export class AviContainer extends Container {
|
|
|
330
333
|
// copied 2.3 GB for 0.19 GB of its own (torrent-tv/meta#166).
|
|
331
334
|
const indexed = await this.#readIndex();
|
|
332
335
|
if (indexed) {
|
|
336
|
+
// A refusal of memory for the table is kept for as long as the memory's
|
|
337
|
+
// owner says the same reservation still would not fit. Not kept, the
|
|
338
|
+
// next reader walked the index, counted the packets and was refused
|
|
339
|
+
// again — once per soundtrack segment, 1943 times in 8.3 s for one
|
|
340
|
+
// two-hour film (torrent-tv/meta#178). Asked at each use, the refusal
|
|
341
|
+
// cannot outlive the state of memory it was given in.
|
|
342
|
+
if (this.#soundPacketsRefused !== null) {
|
|
343
|
+
if (this.packetMemory?.fits?.(this.#soundPacketsRefused) === false) {
|
|
344
|
+
throw new IndexMemoryUnavailable(this.#soundPacketsRefused);
|
|
345
|
+
}
|
|
346
|
+
this.#soundPacketsRefused = null;
|
|
347
|
+
}
|
|
333
348
|
this.#soundPacketsReading ??= this.#soundPacketsOf(indexed).then(
|
|
334
349
|
packets => (this.#packets = packets),
|
|
335
|
-
error => {
|
|
350
|
+
error => {
|
|
351
|
+
this.#soundPacketsReading = null;
|
|
352
|
+
if (error instanceof IndexMemoryUnavailable) this.#soundPacketsRefused = error.bytes;
|
|
353
|
+
throw error;
|
|
354
|
+
});
|
|
336
355
|
return this.#soundPacketsReading;
|
|
337
356
|
}
|
|
338
357
|
const tracks = await this.readTracks();
|
|
@@ -41,32 +41,39 @@ export class IndexMemory {
|
|
|
41
41
|
let file = this.#files.get(key);
|
|
42
42
|
if (file) return file.memory;
|
|
43
43
|
file = { sourceKey, held: 0, needed: 0, records: new Set(), closed: false };
|
|
44
|
-
file.memory = {
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
44
|
+
file.memory = {
|
|
45
|
+
// Whether a reservation of this many bytes would succeed now. Asked by a
|
|
46
|
+
// reader that keeps a refusal, so the refusal never outlives the state
|
|
47
|
+
// of memory it was given in — an event comes only when the allowance
|
|
48
|
+
// changes, and bytes released under the same allowance raise none.
|
|
49
|
+
fits: bytes => !file.closed && this.#held + bytes <= this.#allowed,
|
|
50
|
+
forRecord: (record, kind = "packets") => {
|
|
51
|
+
file.records.add(record);
|
|
52
|
+
let disposed = false;
|
|
53
|
+
return {
|
|
54
|
+
reserve: bytes => {
|
|
55
|
+
if (file.closed || disposed) return false;
|
|
56
|
+
if (this.#held + bytes > this.#allowed) {
|
|
57
|
+
const needed = file.held + bytes;
|
|
58
|
+
if (needed > file.needed) { file.needed = needed; this.#requestRevision(); }
|
|
59
|
+
return false;
|
|
60
|
+
}
|
|
61
|
+
file.held += bytes;
|
|
62
|
+
this.#held += bytes;
|
|
63
|
+
if (kind === "packets") this.#packetBytes += bytes;
|
|
64
|
+
if (file.held >= file.needed) file.needed = 0;
|
|
65
|
+
return true;
|
|
66
|
+
},
|
|
67
|
+
release: bytes => {
|
|
68
|
+
file.held -= bytes;
|
|
69
|
+
this.#held -= bytes;
|
|
70
|
+
if (kind === "packets") this.#packetBytes -= bytes;
|
|
71
|
+
this.#requestRevision();
|
|
72
|
+
},
|
|
73
|
+
dispose: () => { disposed = true; file.records.delete(record); }
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
};
|
|
70
77
|
this.#files.set(key, file);
|
|
71
78
|
return file.memory;
|
|
72
79
|
}
|
|
@@ -136,15 +136,42 @@ export class DownloadMaps {
|
|
|
136
136
|
urgent = current.urgent;
|
|
137
137
|
deadlineAt = Number.isFinite(current.deadlineAt) ? current.deadlineAt - leadSeconds * 1000 : current.deadlineAt;
|
|
138
138
|
}
|
|
139
|
-
|
|
140
|
-
|
|
139
|
+
const entry = { ranges: result.ranges, requestId: result.requestId, priority, urgent, deadlineAt, interval, leadSeconds, scope, order };
|
|
140
|
+
const previous = file.metadata.get(statement);
|
|
141
|
+
if (previous && sameMetadata(previous, entry) && this.#standing(file)) return this.#unchanged(file);
|
|
142
|
+
file.metadata.set(statement, entry);
|
|
143
|
+
} else if (result.kind === "needs-memory") {
|
|
141
144
|
// Allocation refusal says nothing about whether previously missing
|
|
142
|
-
// source bytes have arrived. Keep their demand until the read resolves
|
|
143
|
-
|
|
145
|
+
// source bytes have arrived. Keep their demand until the read resolves —
|
|
146
|
+
// and since it changes nothing in the map, nothing is built for it: the
|
|
147
|
+
// sound packet table of a two-hour AVI was refused once per soundtrack
|
|
148
|
+
// segment, 1943 times in 8.3 s, each a build and sort of every zone of
|
|
149
|
+
// the file (torrent-tv/meta#178).
|
|
150
|
+
if (this.#standing(file)) return this.#unchanged(file);
|
|
151
|
+
} else if (!file.metadata.delete(statement) && this.#standing(file)) {
|
|
152
|
+
return this.#unchanged(file);
|
|
144
153
|
}
|
|
145
154
|
return this.#emit(file);
|
|
146
155
|
}
|
|
147
156
|
|
|
157
|
+
/**
|
|
158
|
+
* Whether the map last built for this file still stands, so a statement that
|
|
159
|
+
* changes nothing needs no new one: nothing `#emit` reads has changed since
|
|
160
|
+
* it ran (`dirty`), and the publication it made was not refused — `#emit`
|
|
161
|
+
* forgets the shape when the torrent thread did not apply the map or the
|
|
162
|
+
* publication failed, and that is what makes the next statement, changed or
|
|
163
|
+
* not, send it again.
|
|
164
|
+
*/
|
|
165
|
+
#standing(file) {
|
|
166
|
+
return !file.dirty && Boolean(file.shape);
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
#unchanged(file) {
|
|
170
|
+
this.#statsOf(file).skipped += 1;
|
|
171
|
+
this.#report(file);
|
|
172
|
+
return Promise.resolve();
|
|
173
|
+
}
|
|
174
|
+
|
|
148
175
|
withdrawMetadata(sourceKey, fileIndex, statement) {
|
|
149
176
|
const file = this.#files.get(`${sourceKey}:${fileIndex}`);
|
|
150
177
|
return file?.metadata.delete(statement) ? this.#emit(file) : Promise.resolve();
|
|
@@ -188,7 +215,7 @@ export class DownloadMaps {
|
|
|
188
215
|
const key = `${sourceKey}:${fileIndex}`;
|
|
189
216
|
let file = this.#files.get(key);
|
|
190
217
|
if (!file) {
|
|
191
|
-
file = { sourceKey, fileIndex, durationSeconds: 0, zones: [], inputs: new Map(), nativeZones: [], sourceZones: [], revision: 0, epoch: ++this.#nextEpoch, metadata: new Map(), pending: Promise.resolve() };
|
|
218
|
+
file = { sourceKey, fileIndex, durationSeconds: 0, zones: [], inputs: new Map(), nativeZones: [], sourceZones: [], revision: 0, epoch: ++this.#nextEpoch, metadata: new Map(), pending: Promise.resolve(), dirty: true };
|
|
192
219
|
this.#files.set(key, file);
|
|
193
220
|
}
|
|
194
221
|
return file;
|
|
@@ -196,6 +223,8 @@ export class DownloadMaps {
|
|
|
196
223
|
|
|
197
224
|
#setZones(file, zones) {
|
|
198
225
|
file.zones = zones;
|
|
226
|
+
// What `#emit` reads has changed, whether or not it is published now.
|
|
227
|
+
file.dirty = true;
|
|
199
228
|
file.inputs = new Map();
|
|
200
229
|
for (const zone of zones) {
|
|
201
230
|
if (typeof zone.outputKey !== "string" || !Number.isSafeInteger(zone.index)) continue;
|
|
@@ -208,6 +237,7 @@ export class DownloadMaps {
|
|
|
208
237
|
}
|
|
209
238
|
|
|
210
239
|
#emit(file) {
|
|
240
|
+
file.dirty = false;
|
|
211
241
|
const builtFrom = this.#clock();
|
|
212
242
|
const zones = [...file.zones, ...file.nativeZones];
|
|
213
243
|
for (const { ranges, requestId, priority, urgent, deadlineAt, order } of file.metadata.values()) {
|
|
@@ -297,7 +327,8 @@ export class DownloadMaps {
|
|
|
297
327
|
`(${total(stats.appliedMs)}ms in all); ` +
|
|
298
328
|
`stages of a sent map — built ${stage(stats.sentBuildMs)}, waited behind the previous ${stage(stats.waitedMs)}, ` +
|
|
299
329
|
`posted ${stage(stats.postMs)}, post to the torrent thread's handler ${stage(stats.toHandlerMs)}, ` +
|
|
300
|
-
`reply back ${stage(stats.replyMs)}; maps built and not sent took ${total([stats.unsentBuildMs])}ms here`
|
|
330
|
+
`reply back ${stage(stats.replyMs)}; maps built and not sent took ${total([stats.unsentBuildMs])}ms here; ` +
|
|
331
|
+
`${stats.skipped} statement(s) changed nothing and built no map`);
|
|
301
332
|
file.stats = emptyStats(now);
|
|
302
333
|
}
|
|
303
334
|
}
|
|
@@ -305,7 +336,7 @@ export class DownloadMaps {
|
|
|
305
336
|
/** @param {number} since */
|
|
306
337
|
function emptyStats(since) {
|
|
307
338
|
return { since, sent: 0, unchanged: 0, zones: [], deliveredMs: [], appliedMs: [],
|
|
308
|
-
sentBuildMs: [], waitedMs: [], postMs: [], toHandlerMs: [], replyMs: [], unsentBuildMs: 0 };
|
|
339
|
+
sentBuildMs: [], waitedMs: [], postMs: [], toHandlerMs: [], replyMs: [], unsentBuildMs: 0, skipped: 0 };
|
|
309
340
|
}
|
|
310
341
|
|
|
311
342
|
/** One stage's median, largest and sum, in milliseconds with tenths. @param {number[]} values */
|
|
@@ -316,6 +347,22 @@ function stage(values) {
|
|
|
316
347
|
return `median ${sorted[Math.floor(sorted.length / 2)].toFixed(1)}ms max ${sorted.at(-1).toFixed(1)}ms sum ${Math.round(sum)}ms`;
|
|
317
348
|
}
|
|
318
349
|
|
|
350
|
+
/**
|
|
351
|
+
* Whether two metadata demands are the same demand: every field, the byte
|
|
352
|
+
* ranges and the interval by value. The fields that do not reach the
|
|
353
|
+
* publication — the interval, the preparation time, the scope — decide how the
|
|
354
|
+
* demand is repriced and withdrawn later, so a change of them is a change.
|
|
355
|
+
*/
|
|
356
|
+
function sameMetadata(left, right) {
|
|
357
|
+
return left.requestId === right.requestId && Object.is(left.priority, right.priority) &&
|
|
358
|
+
left.urgent === right.urgent && Object.is(left.deadlineAt, right.deadlineAt) &&
|
|
359
|
+
Object.is(left.leadSeconds, right.leadSeconds) && left.scope === right.scope && left.order === right.order &&
|
|
360
|
+
(left.interval?.from ?? null) === (right.interval?.from ?? null) &&
|
|
361
|
+
(left.interval?.to ?? null) === (right.interval?.to ?? null) &&
|
|
362
|
+
left.ranges.length === right.ranges.length &&
|
|
363
|
+
left.ranges.every(([start, end], index) => start === right.ranges[index][0] && end === right.ranges[index][1]);
|
|
364
|
+
}
|
|
365
|
+
|
|
319
366
|
/** How many numbers describe one zone's part in what the torrent does. */
|
|
320
367
|
const ZONE_FIELDS = 7;
|
|
321
368
|
|