@torrent-tv/proxy 2.107.56 → 2.107.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/cli.js +24 -15
- package/package.json +1 -1
- package/server.js +195 -1139
- package/services/encode/EncodeInputResolution.js +157 -0
- package/services/encode/LocalObservations.js +20 -2
- package/services/encode/measure-host.js +147 -0
- package/services/identity/installation-id.js +17 -6
- package/services/media/SegmentSourceRanges.js +95 -0
- package/services/media/SubtitleWalk.js +172 -0
- package/services/media/WorkTagsReport.js +30 -0
- package/services/media/segment-reading.js +36 -0
- package/services/server/MediaReadParams.js +134 -0
- package/services/server/PreparationReads.js +175 -0
- package/services/server/forwarded-response-log.js +55 -0
- package/services/server/start-playback.js +34 -0
- package/services/server/torrent-pool-ports.js +105 -0
- package/services/server/wire-outputs.js +62 -49
- package/services/storage/files/CompletedFiles.js +6 -1
- package/services/storage/memory-report.js +5 -5
- package/services/storage/processor-share.js +24 -0
- package/services/storage/segment-store/SegmentStore.js +2 -1
- package/services/torrent/PlaybackResolution.js +397 -0
- package/services/torrent/worker/worker.js +1 -6
- package/services/viewer/watched-addresses.js +51 -0
- package/utils/atomic-json.js +46 -0
|
@@ -0,0 +1,397 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file The download map's pass: what the torrent is told about the file a
|
|
3
|
+
* viewer plays, worked out from the viewers' map (torrent-tv/meta#270, #273).
|
|
4
|
+
*
|
|
5
|
+
* `DownloadMaps` hands it a file's map; it answers the windows of source bytes
|
|
6
|
+
* the swarm is to fetch. It owns what that pass keeps from map to map: the
|
|
7
|
+
* projection of each played file (`PlaybackProjection`), what each segment of
|
|
8
|
+
* each output reads and the bytes it was worked out to read, the cost of the
|
|
9
|
+
* passes. Everything it reads of the world comes in by a port: the file's
|
|
10
|
+
* parameters, the resolver of a segment's source ranges, the rule of what a
|
|
11
|
+
* segment reads, the outputs somebody watches, the piece length, the table of
|
|
12
|
+
* what each segment reads, the wait-trace and the log.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { performance } from "node:perf_hooks";
|
|
16
|
+
import { takeTurns } from "../../utils/take-turns.js";
|
|
17
|
+
import { joinedWithinPieces } from "./demand/pieces.js";
|
|
18
|
+
import { PlaybackProjection } from "./PlaybackProjection.js";
|
|
19
|
+
|
|
20
|
+
export class PlaybackResolution {
|
|
21
|
+
#paramsFor;
|
|
22
|
+
#sourceRanges;
|
|
23
|
+
#segmentReading;
|
|
24
|
+
#playedOutputsOf;
|
|
25
|
+
#pieceLengthOf;
|
|
26
|
+
#noteSegmentInput;
|
|
27
|
+
#refresh;
|
|
28
|
+
#observeWait;
|
|
29
|
+
#log;
|
|
30
|
+
#warn;
|
|
31
|
+
#now;
|
|
32
|
+
#clock;
|
|
33
|
+
#turns;
|
|
34
|
+
/** What the torrent is told of each played file, kept from map to map (`PlaybackProjection`); forgotten with the source. */
|
|
35
|
+
#projections = new Map();
|
|
36
|
+
/**
|
|
37
|
+
* WHAT ONE SEGMENT OF AN OUTPUT IS, kept from pass to pass: where it lies in
|
|
38
|
+
* the source, what it reads — its tracks and the key its bytes are kept under
|
|
39
|
+
* — and, once known, those bytes joined at the precision the torrent fetches,
|
|
40
|
+
* by output and segment. Worked out again only when the output's timeline,
|
|
41
|
+
* the tracks, the file's start time or the output's choices change; forgotten
|
|
42
|
+
* with the source. None of it depends on where a viewer stands, and a pass is
|
|
43
|
+
* made every time one moves: looked up, made and joined again for each of a
|
|
44
|
+
* film's 3500 segments on every pass, they were 6 s of the 40 s the passes
|
|
45
|
+
* cost the main thread in fifteen minutes on Home Assistant (r30,
|
|
46
|
+
* torrent-tv/meta#178).
|
|
47
|
+
*/
|
|
48
|
+
#segmentReadings = new Map();
|
|
49
|
+
/** The choices of one output's tracks as a key, worked out once for the list of them a pass shares. */
|
|
50
|
+
#choicesKeys = new WeakMap();
|
|
51
|
+
/** An output's cut grid, checked once for each timeline. */
|
|
52
|
+
#checkedGrids = new WeakMap();
|
|
53
|
+
/**
|
|
54
|
+
* How many bytes a second of an output's film takes, from what it has made,
|
|
55
|
+
* read again only when what it has made changes (`SegmentStore.revisionOf`):
|
|
56
|
+
* summed over every made segment on every pass, it was a walk of the whole
|
|
57
|
+
* film a second (torrent-tv/meta#178).
|
|
58
|
+
*/
|
|
59
|
+
#madeRates = new Map();
|
|
60
|
+
// WHAT THE PASSES COST, said once a minute: how many maps were taken, how
|
|
61
|
+
// many segments a map made the projection work out again and how many of
|
|
62
|
+
// those maps it worked out whole, how many segments had their ranges worked
|
|
63
|
+
// out, the windows told, and the time from a pass's start to its result —
|
|
64
|
+
// which includes waits for reads, so it bounds the main thread's share
|
|
65
|
+
// rather than measuring it (torrent-tv/meta#178).
|
|
66
|
+
#passCosts;
|
|
67
|
+
// A container whose packet table a pass did not wait for builds the map
|
|
68
|
+
// again once, when the table is built or refused (`directSegmentInput`).
|
|
69
|
+
#refreshAfterTable = new WeakSet();
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* @param {object} ports
|
|
73
|
+
* @param {(request: { sourceKey: string, fileIndex: number, requestId?: string, packetInterval?: object, demand?: object }) => Promise<object | null>} ports.paramsFor
|
|
74
|
+
* @param {object} ports.sourceRanges - `SegmentSourceRanges`.
|
|
75
|
+
* @param {(trackList: object[], choices: object[], sourceInterval: object) => object | null} ports.segmentReading
|
|
76
|
+
* @param {(sourceKey: string, fileIndex: number) => object[]} ports.playedOutputsOf
|
|
77
|
+
* @param {(sourceKey: string) => number} ports.pieceLengthOf - NaN when the torrent is not known.
|
|
78
|
+
* @param {(sourceKey: string, fileIndex: number, outputKey: string, index: number, joined: Array<[number, number]>) => void} ports.noteSegmentInput
|
|
79
|
+
* @param {(sourceKey: string, fileIndex: number) => Promise<unknown>} ports.refresh
|
|
80
|
+
* @param {(sourceKey: string, event: string, fields: object) => void} ports.observeWait
|
|
81
|
+
* @param {(line: string) => void} ports.log
|
|
82
|
+
* @param {(line: string) => void} ports.warn
|
|
83
|
+
* @param {() => number} [ports.now] - The clock of the once-a-minute reading.
|
|
84
|
+
* @param {() => number} [ports.clock] - Pass and P4.1 timing.
|
|
85
|
+
* @param {() => (() => Promise<unknown>)} [ports.turns]
|
|
86
|
+
*/
|
|
87
|
+
constructor({ paramsFor, sourceRanges, segmentReading, playedOutputsOf, pieceLengthOf, noteSegmentInput, refresh,
|
|
88
|
+
observeWait, log, warn, now = Date.now, clock = () => performance.now(), turns = takeTurns }) {
|
|
89
|
+
this.#paramsFor = paramsFor;
|
|
90
|
+
this.#sourceRanges = sourceRanges;
|
|
91
|
+
this.#segmentReading = segmentReading;
|
|
92
|
+
this.#playedOutputsOf = playedOutputsOf;
|
|
93
|
+
this.#pieceLengthOf = pieceLengthOf;
|
|
94
|
+
this.#noteSegmentInput = noteSegmentInput;
|
|
95
|
+
this.#refresh = refresh;
|
|
96
|
+
this.#observeWait = observeWait;
|
|
97
|
+
this.#log = log;
|
|
98
|
+
this.#warn = warn;
|
|
99
|
+
this.#now = now;
|
|
100
|
+
this.#clock = clock;
|
|
101
|
+
this.#turns = turns;
|
|
102
|
+
this.#passCosts = { since: now(), passes: 0, whole: 0, reworked: 0, resolved: 0, windows: 0, ms: 0, maxMs: 0 };
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** The file has no playback zones: what is told of it is emptied. */
|
|
106
|
+
clear(sourceKey, fileIndex) {
|
|
107
|
+
this.#projections.get(`${sourceKey}:${fileIndex}`)?.clear();
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** The source is forgotten: its projections, records and made rates go. */
|
|
111
|
+
forget(sourceKey) {
|
|
112
|
+
for (const key of this.#segmentReadings.keys()) if (key.startsWith(`${sourceKey}:`)) this.#segmentReadings.delete(key);
|
|
113
|
+
for (const key of this.#projections.keys()) if (key.startsWith(`${sourceKey}:`)) this.#projections.delete(key);
|
|
114
|
+
for (const key of this.#madeRates.keys()) if (key.startsWith(`${sourceKey}:`)) this.#madeRates.delete(key);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/** One output's segments, by index. */
|
|
118
|
+
#segmentRecordsOf(map, output) {
|
|
119
|
+
const key = `${map.sourceKey}:${map.fileIndex}:${output.outputKey}`;
|
|
120
|
+
let records = this.#segmentReadings.get(key);
|
|
121
|
+
if (!records) this.#segmentReadings.set(key, records = new Map());
|
|
122
|
+
return records;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
#choicesKeyOf(choices) {
|
|
126
|
+
let key = this.#choicesKeys.get(choices);
|
|
127
|
+
if (key === undefined) this.#choicesKeys.set(choices, key = choices.map(choice => `${choice.type}/${choice.index}/${choice.mode}`).join(","));
|
|
128
|
+
return key;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
#segmentRecordOf(records, demand, { timeline, trackList, shift, map }) {
|
|
132
|
+
const choicesKey = this.#choicesKeyOf(demand.tracks);
|
|
133
|
+
const kept = records.get(demand.index);
|
|
134
|
+
if (kept && kept.timeline === timeline && kept.trackList === trackList && kept.shift === shift && kept.choicesKey === choicesKey) return kept;
|
|
135
|
+
const sourceInterval = timeline.sourceInterval(demand.index, demand.index, shift);
|
|
136
|
+
const found = this.#segmentReading(trackList, demand.tracks, sourceInterval);
|
|
137
|
+
const reading = found && { ...found, rangesKey: this.#sourceRanges.keyOf(map.sourceKey, map.fileIndex, found.interval) };
|
|
138
|
+
// `joined` is the segment's ranges as the torrent is told them, made from
|
|
139
|
+
// the list `ranges` at `pieceLength`; `registered` is the list the input
|
|
140
|
+
// table was last given.
|
|
141
|
+
const record = { timeline, trackList, shift, choicesKey, sourceInterval, reading,
|
|
142
|
+
ranges: null, pieceLength: Number.NaN, joined: null, registered: null };
|
|
143
|
+
records.set(demand.index, record);
|
|
144
|
+
return record;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
#gridOf(timeline) {
|
|
148
|
+
if (!timeline || typeof timeline !== "object") return null;
|
|
149
|
+
if (this.#checkedGrids.has(timeline)) return this.#checkedGrids.get(timeline);
|
|
150
|
+
const grid = timeline.published ?? timeline.boundaries;
|
|
151
|
+
if (!Array.isArray(grid)) {
|
|
152
|
+
this.#checkedGrids.set(timeline, null);
|
|
153
|
+
return null;
|
|
154
|
+
}
|
|
155
|
+
for (let index = 0; index + 1 < grid.length; index++) {
|
|
156
|
+
if (!Number.isFinite(grid[index]) || !Number.isFinite(grid[index + 1]) || !(grid[index + 1] > grid[index])) {
|
|
157
|
+
throw new Error("An output segment needs a valid interval.");
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
this.#checkedGrids.set(timeline, grid);
|
|
161
|
+
return grid;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
#bytesPerSecondOf(entry) {
|
|
165
|
+
const kept = this.#madeRates.get(entry.outputKey);
|
|
166
|
+
if (kept && kept.revision === entry.revision && kept.timeline === entry.timeline) return kept.bytesPerSecond;
|
|
167
|
+
let bytes = 0, seconds = 0;
|
|
168
|
+
for (const [index, size] of entry.sizes()) {
|
|
169
|
+
const span = entry.timeline.publishedStartOf(index + 1) - entry.timeline.publishedStartOf(index);
|
|
170
|
+
if (span > 0) { bytes += size; seconds += span; }
|
|
171
|
+
}
|
|
172
|
+
const bytesPerSecond = seconds > 0 ? bytes / seconds : null;
|
|
173
|
+
this.#madeRates.set(entry.outputKey, { revision: entry.revision, timeline: entry.timeline, bytesPerSecond });
|
|
174
|
+
return bytesPerSecond;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
#notePass({ whole = false, reworked = 0, resolved = 0, windows = 0, ms = 0 }) {
|
|
178
|
+
const passCosts = this.#passCosts;
|
|
179
|
+
passCosts.passes += 1;
|
|
180
|
+
if (whole) passCosts.whole += 1;
|
|
181
|
+
passCosts.reworked += reworked; passCosts.resolved += resolved; passCosts.windows += windows;
|
|
182
|
+
passCosts.ms += ms; passCosts.maxMs = Math.max(passCosts.maxMs, ms);
|
|
183
|
+
const now = this.#now();
|
|
184
|
+
if (now - passCosts.since < 60_000) return;
|
|
185
|
+
this.#log(`download map passes in ${((now - passCosts.since) / 1000).toFixed(1)}s: ${passCosts.passes} map(s), ` +
|
|
186
|
+
`${passCosts.whole} worked out whole, ${passCosts.reworked} segment(s) worked out again, ` +
|
|
187
|
+
`${passCosts.resolved} segment(s) had their ranges worked out, ${passCosts.windows} window(s) told; ` +
|
|
188
|
+
`${Math.round(passCosts.ms)}ms from start to result in all, longest ${Math.round(passCosts.maxMs)}ms`);
|
|
189
|
+
Object.assign(passCosts, { since: now, passes: 0, whole: 0, reworked: 0, resolved: 0, windows: 0, ms: 0, maxMs: 0 });
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
#refreshWhenBuilt(container, built, { sourceKey, fileIndex }) {
|
|
193
|
+
if (this.#refreshAfterTable.has(container)) return;
|
|
194
|
+
this.#refreshAfterTable.add(container);
|
|
195
|
+
void built.then(() => {
|
|
196
|
+
this.#refreshAfterTable.delete(container);
|
|
197
|
+
return this.#refresh(sourceKey, fileIndex);
|
|
198
|
+
}).catch(error => this.#warn(`download map after a packet table: ${error?.message ?? error}`));
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* One pass over a file's map.
|
|
203
|
+
*
|
|
204
|
+
* @param {{ sourceKey: string, fileIndex: number, zones: object[], isCurrent: () => boolean,
|
|
205
|
+
* workingOut?: (priority: number, zonesOf: () => object[]) => void }} map
|
|
206
|
+
* @returns {Promise<object[]>} The windows to tell the torrent; empty when the map was replaced meanwhile.
|
|
207
|
+
*/
|
|
208
|
+
async resolve(map) {
|
|
209
|
+
const fileKey = `${map.sourceKey}:${map.fileIndex}`;
|
|
210
|
+
if (map.zones.length === 0) {
|
|
211
|
+
this.#projections.get(fileKey)?.clear();
|
|
212
|
+
return [];
|
|
213
|
+
}
|
|
214
|
+
const params = await this.#paramsFor({ sourceKey: map.sourceKey, fileIndex: map.fileIndex });
|
|
215
|
+
if (!params || !map.isCurrent()) return [];
|
|
216
|
+
const tracks = await this.#sourceRanges.tracksOf(params, () => map.isCurrent());
|
|
217
|
+
if (tracks.kind !== "result") return [];
|
|
218
|
+
const container = tracks.container;
|
|
219
|
+
const media = await this.#sourceRanges.shiftOf(params, () => map.isCurrent());
|
|
220
|
+
if (media.kind !== "result") return [];
|
|
221
|
+
const shift = media.shift;
|
|
222
|
+
// A zone's ranges are stated to the torrent at the precision it fetches:
|
|
223
|
+
// ranges closer than a piece hold the same pieces as one.
|
|
224
|
+
const pieceLength = this.#pieceLengthOf(map.sourceKey);
|
|
225
|
+
const passStartedAt = this.#clock();
|
|
226
|
+
// THE PROJECTION IS KEPT while what it was made from stands: the file's
|
|
227
|
+
// tracks, where its clock starts and how the torrent fetches. Any of them
|
|
228
|
+
// changed, every segment's ranges are other ranges.
|
|
229
|
+
let projection = this.#projections.get(fileKey);
|
|
230
|
+
if (!projection || projection.trackList !== tracks.trackList || projection.shift !== shift ||
|
|
231
|
+
!Object.is(projection.pieceLength, pieceLength)) {
|
|
232
|
+
projection?.clear();
|
|
233
|
+
projection = new PlaybackProjection();
|
|
234
|
+
Object.assign(projection, { trackList: tracks.trackList, shift, pieceLength });
|
|
235
|
+
this.#projections.set(fileKey, projection);
|
|
236
|
+
}
|
|
237
|
+
// Every output of this file somebody is watching, with what decides how
|
|
238
|
+
// long its segments take to make and deliver, what it has made, and what
|
|
239
|
+
// its next run would copy.
|
|
240
|
+
const played = [];
|
|
241
|
+
/** By output: what its segments read and how they are looked up. */
|
|
242
|
+
const readers = new Map();
|
|
243
|
+
for (const entry of this.#playedOutputsOf(map.sourceKey, map.fileIndex)) {
|
|
244
|
+
const grid = this.#gridOf(entry.timeline);
|
|
245
|
+
if (!grid) continue;
|
|
246
|
+
// Every piece answers for its exact ending boundary, as the encode
|
|
247
|
+
// plan reads it: the player may request the preceding piece there.
|
|
248
|
+
played.push({ outputKey: entry.outputKey, grid, encodeSpeed: entry.encodeSpeed,
|
|
249
|
+
bytesPerSecond: this.#bytesPerSecondOf(entry), linkMbps: entry.linkMbps,
|
|
250
|
+
readyRevision: entry.revision, ready: entry.sizes, target: entry.target });
|
|
251
|
+
readers.set(entry.outputKey, { records: this.#segmentRecordsOf(map, entry), choices: entry.choices,
|
|
252
|
+
facts: { timeline: entry.timeline, trackList: tracks.trackList, shift, map } });
|
|
253
|
+
}
|
|
254
|
+
// WHAT THE MAP CHANGED, and only that, is worked out again here.
|
|
255
|
+
projection.update({ zones: map.zones, outputs: played });
|
|
256
|
+
const reworked = projection.lastUpdate;
|
|
257
|
+
let resolved = 0;
|
|
258
|
+
// A container that names the bytes of an interval from an index it holds
|
|
259
|
+
// answers every segment by arithmetic: once that index is read
|
|
260
|
+
// ("source-navigation"), the ranges are computed here directly instead of
|
|
261
|
+
// through the file's queue of reads, three steps per segment. On Home
|
|
262
|
+
// Assistant those steps made a pass over a two-hour AVI take minutes while
|
|
263
|
+
// the arithmetic takes milliseconds (torrent-tv/meta#151).
|
|
264
|
+
const navigated = await this.#sourceRanges.navigable(params, () => map.isCurrent());
|
|
265
|
+
if (navigated.kind === "stale") return [];
|
|
266
|
+
const direct = navigated.kind === "result" && navigated.value === true &&
|
|
267
|
+
typeof container?.readSourceRanges === "function" ? container : null;
|
|
268
|
+
// An output whose file-level reading is not there in this pass — the
|
|
269
|
+
// index or Cues that say where a segment is, not yet downloaded, or the
|
|
270
|
+
// table its segments read, refused memory — answers its other segments
|
|
271
|
+
// the same way. One unanswered request is kept to be retried when the
|
|
272
|
+
// bytes or the memory change, and the map is built again when it
|
|
273
|
+
// answers; asking every segment held the main thread for 3.3 s on a cold
|
|
274
|
+
// two-hour AVI (torrent-tv/meta#178).
|
|
275
|
+
const waiting = new Set();
|
|
276
|
+
const turn = this.#turns();
|
|
277
|
+
// ONLY THE SEGMENTS WHOSE RANGES ARE NOT KNOWN are worked out, in the
|
|
278
|
+
// order the swarm is to fetch them. Range resolution can use either a
|
|
279
|
+
// direct source index or the fallback reads below; yield between zones
|
|
280
|
+
// in both cases so a large map cannot hold the main thread across its
|
|
281
|
+
// full projection.
|
|
282
|
+
for (const zone of projection.unresolved()) {
|
|
283
|
+
if (!map.isCurrent()) return [];
|
|
284
|
+
await turn();
|
|
285
|
+
if (!map.isCurrent()) return [];
|
|
286
|
+
const resolutionStartedAt = this.#clock();
|
|
287
|
+
const resolution = { outcome: "unresolved", method: null, trackCount: null, rangeCount: null, rangeBytes: null };
|
|
288
|
+
const resolutionId = `${map.fileIndex}:${zone.outputKey}:${zone.index}:${passStartedAt.toFixed(3)}`;
|
|
289
|
+
this.#observeWait(map.sourceKey, "process", { process: "P4.1", phase: "begin", resolutionId,
|
|
290
|
+
sourceId: `${map.sourceKey}:${map.fileIndex}`, fileIndex: map.fileIndex, outputKey: zone.outputKey,
|
|
291
|
+
segment: zone.index, priority: zone.priority, deadlineAt: zone.deadlineAt });
|
|
292
|
+
try {
|
|
293
|
+
const reader = readers.get(zone.outputKey);
|
|
294
|
+
if (!reader) {
|
|
295
|
+
resolution.outcome = "reader-missing";
|
|
296
|
+
continue;
|
|
297
|
+
}
|
|
298
|
+
// WHAT A SEGMENT READS IS WORKED OUT ONCE (`#segmentRecordOf`). Rebuilt
|
|
299
|
+
// for every segment of the film on every map, this pass held the main
|
|
300
|
+
// thread up to 1.5 s at a time on Home Assistant (torrent-tv/meta#178).
|
|
301
|
+
const record = this.#segmentRecordOf(reader.records, { index: zone.index, tracks: reader.choices }, reader.facts);
|
|
302
|
+
const reading = record.reading;
|
|
303
|
+
if (!reading) {
|
|
304
|
+
resolution.outcome = "segment-reading-unavailable";
|
|
305
|
+
projection.setUnreadable(zone.outputKey, zone.index);
|
|
306
|
+
continue;
|
|
307
|
+
}
|
|
308
|
+
const { wanted, modes, interval, rangesKey } = reading;
|
|
309
|
+
resolution.trackCount = wanted.length;
|
|
310
|
+
resolution.trackIds = wanted.map(track => track.trackNumber);
|
|
311
|
+
resolution.sourceInterval = { from: interval.from, to: interval.to };
|
|
312
|
+
const convert = (ranges, method) => {
|
|
313
|
+
if (record.ranges !== ranges || record.pieceLength !== pieceLength) {
|
|
314
|
+
record.ranges = ranges;
|
|
315
|
+
record.pieceLength = pieceLength;
|
|
316
|
+
record.joined = joinedWithinPieces(ranges, pieceLength);
|
|
317
|
+
}
|
|
318
|
+
const joined = record.joined;
|
|
319
|
+
resolution.method = method;
|
|
320
|
+
resolution.rangeCount = joined.length;
|
|
321
|
+
resolution.rangeBytes = joined.reduce((total, [start, end]) => total + end - start + 1, 0);
|
|
322
|
+
// What the segment reads is kept for the forecast in a table of its
|
|
323
|
+
// own, whatever form the torrent is told it in.
|
|
324
|
+
if (record.registered !== joined) {
|
|
325
|
+
this.#noteSegmentInput(map.sourceKey, map.fileIndex, zone.outputKey, zone.index, joined);
|
|
326
|
+
record.registered = joined;
|
|
327
|
+
}
|
|
328
|
+
projection.setRanges(zone.outputKey, zone.index, joined);
|
|
329
|
+
resolution.outcome = "resolved";
|
|
330
|
+
resolved += 1;
|
|
331
|
+
};
|
|
332
|
+
// The bytes one segment needs are a fact of the file and the output, so
|
|
333
|
+
// they are worked out once. A whole film is thousands of segments and
|
|
334
|
+
// the map changes every few seconds; worked out afresh on every change,
|
|
335
|
+
// a pass over a two-hour AVI never finished and the next segment's bytes
|
|
336
|
+
// were never asked for (Home Assistant 2026-10-08, torrent-tv/meta#151).
|
|
337
|
+
const known = record.ranges ? { ranges: record.ranges, method: "segment-record-cache" } : this.#sourceRanges.cached(rangesKey);
|
|
338
|
+
if (known) { convert(known.ranges, known.method); continue; }
|
|
339
|
+
if (waiting.has(zone.outputKey)) {
|
|
340
|
+
resolution.outcome = "waiting-for-output-input";
|
|
341
|
+
continue;
|
|
342
|
+
}
|
|
343
|
+
// What is resolved so far is stated before a lower zone is worked out
|
|
344
|
+
// (DownloadMaps, torrent-tv/meta#178).
|
|
345
|
+
map.workingOut?.(zone.priority, () => projection.windows());
|
|
346
|
+
const found = await this.#sourceRanges.resolve({ rangesKey, direct, interval, tracks: wanted, modes,
|
|
347
|
+
isCurrent: () => map.isCurrent(),
|
|
348
|
+
whenBuilt: built => this.#refreshWhenBuilt(direct, built, map),
|
|
349
|
+
intervalParams: () => {
|
|
350
|
+
resolution.method = "container-index-fallback";
|
|
351
|
+
return this.#paramsFor({ sourceKey: map.sourceKey, fileIndex: map.fileIndex, packetInterval: interval,
|
|
352
|
+
requestId: `download:${map.sourceKey}:${map.fileIndex}:${zone.outputKey}:${zone.index}`,
|
|
353
|
+
demand: { ...zone.owner, leadSeconds: zone.leadSeconds } });
|
|
354
|
+
} });
|
|
355
|
+
if (found.kind === "result") { convert(found.ranges, found.method); continue; }
|
|
356
|
+
resolution.method = found.method;
|
|
357
|
+
if (found.kind === "stale") {
|
|
358
|
+
resolution.outcome = "map-replaced";
|
|
359
|
+
return [];
|
|
360
|
+
}
|
|
361
|
+
if (found.kind === "pending") {
|
|
362
|
+
// A table still being built holds this output's segments, and only
|
|
363
|
+
// this output's: the map is built again when it is.
|
|
364
|
+
resolution.outcome = "source-index-pending";
|
|
365
|
+
waiting.add(zone.outputKey);
|
|
366
|
+
continue;
|
|
367
|
+
}
|
|
368
|
+
if (found.kind === "needs-source") {
|
|
369
|
+
resolution.outcome = "source-file-unavailable";
|
|
370
|
+
continue;
|
|
371
|
+
}
|
|
372
|
+
if (found.step === "navigation") {
|
|
373
|
+
resolution.outcome = `source-navigation-${found.kind}`;
|
|
374
|
+
if (found.kind === "needs-ranges" || found.kind === "needs-memory") waiting.add(zone.outputKey);
|
|
375
|
+
continue;
|
|
376
|
+
}
|
|
377
|
+
if (found.kind === "needs-memory") waiting.add(zone.outputKey);
|
|
378
|
+
resolution.outcome = found.step === "packets" ? `source-packets-${found.kind}` : `segment-input-${found.kind}`;
|
|
379
|
+
} catch (error) {
|
|
380
|
+
resolution.outcome = "error";
|
|
381
|
+
resolution.error = error?.message ?? String(error);
|
|
382
|
+
throw error;
|
|
383
|
+
} finally {
|
|
384
|
+
const resolutionEndedAt = this.#clock();
|
|
385
|
+
this.#observeWait(map.sourceKey, "process", { process: "P4.1", phase: "end", resolutionId,
|
|
386
|
+
sourceId: `${map.sourceKey}:${map.fileIndex}`, fileIndex: map.fileIndex, outputKey: zone.outputKey,
|
|
387
|
+
segment: zone.index, priority: zone.priority,
|
|
388
|
+
startedAt: resolutionStartedAt, endedAt: resolutionEndedAt, durationMs: resolutionEndedAt - resolutionStartedAt,
|
|
389
|
+
...resolution });
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
const result = projection.windows();
|
|
393
|
+
this.#notePass({ whole: reworked.whole, reworked: reworked.segments, resolved, windows: result.length,
|
|
394
|
+
ms: this.#clock() - passStartedAt });
|
|
395
|
+
return result;
|
|
396
|
+
}
|
|
397
|
+
}
|
|
@@ -793,20 +793,15 @@ startMemoryReport({
|
|
|
793
793
|
scope: "thread",
|
|
794
794
|
label: "torrent worker",
|
|
795
795
|
intervalMs: WORKER_MEMORY_SAMPLE_MS,
|
|
796
|
-
quietMs: 60_000,
|
|
797
|
-
changeBytes: 25 * 1024 * 1024,
|
|
798
796
|
snapshotDir: workerData?.stateDir || undefined,
|
|
799
797
|
heapSnapshots: workerData?.heapSnapshots === true,
|
|
800
|
-
snapshotFloorBytes: 400 * 1024 * 1024,
|
|
801
|
-
snapshotGrowthBytes: 400 * 1024 * 1024,
|
|
802
798
|
// CHOSEN, not derived: no healthy session has recorded this figure yet, so
|
|
803
799
|
// there is no measured normal to stand above. It is set well below the
|
|
804
800
|
// 2.1-4.3 GB the worker held in the minutes before the kills of 2026-10-01,
|
|
805
801
|
// while the machine still has room to write a snapshot. The line prints what the
|
|
806
802
|
// stores hold beside it, which is what a derived floor will come from.
|
|
807
803
|
bufferSnapshotFloorBytes: 500 * 1024 * 1024,
|
|
808
|
-
bufferSnapshotGrowthBytes: 500 * 1024 * 1024
|
|
809
|
-
keepSnapshots: 3
|
|
804
|
+
bufferSnapshotGrowthBytes: 500 * 1024 * 1024
|
|
810
805
|
});
|
|
811
806
|
|
|
812
807
|
const STORE_REPORT_INTERVAL_MS = 60_000;
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file Which output addresses are being prepared, and which are watched, for
|
|
3
|
+
* the encoder admission. Moved out of `wire-outputs.js` verbatim
|
|
4
|
+
* (torrent-tv/meta#280).
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { outputsBeingPrepared } from "./choices.js";
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* The output keys of the outputs a viewer is being prepared for.
|
|
11
|
+
*
|
|
12
|
+
* @param {object} viewers
|
|
13
|
+
* @param {{ get: (id: string) => { outputKey?: string } | undefined }} outputs
|
|
14
|
+
* @returns {Set<string>}
|
|
15
|
+
*/
|
|
16
|
+
export function preparedAddresses(viewers, outputs) {
|
|
17
|
+
const addresses = new Set();
|
|
18
|
+
for (const id of outputsBeingPrepared(viewers)) {
|
|
19
|
+
const key = outputs.get(id)?.outputKey;
|
|
20
|
+
if (key) {
|
|
21
|
+
addresses.add(key);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
return addresses;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Every output a present viewer receives that still has something left to
|
|
29
|
+
* make, by address.
|
|
30
|
+
*
|
|
31
|
+
* @param {object} viewers
|
|
32
|
+
* @param {{ values: () => Iterable<object>, supersededBy: (session: object, variantId: string | null) => boolean }} outputs
|
|
33
|
+
* @param {(output: object) => boolean} finished
|
|
34
|
+
* @returns {Set<string>}
|
|
35
|
+
*/
|
|
36
|
+
export function watchedAddresses(viewers, outputs, finished) {
|
|
37
|
+
const addresses = new Set();
|
|
38
|
+
for (const session of outputs.values()) {
|
|
39
|
+
const address = session.outputKey;
|
|
40
|
+
if (!address || addresses.has(address) || finished(session)) {
|
|
41
|
+
continue;
|
|
42
|
+
}
|
|
43
|
+
for (const viewer of viewers.forOutput(session).values()) {
|
|
44
|
+
if (viewer.isPresent() && !outputs.supersededBy(session, viewer.activeVariantId ?? null)) {
|
|
45
|
+
addresses.add(address);
|
|
46
|
+
break;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
return addresses;
|
|
51
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import nodeFs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
|
|
4
|
+
let writes = 0;
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Write a file so that a reader (or a restart after a crash) sees either the
|
|
8
|
+
* previous content or the new content, never a half-written file: the text goes
|
|
9
|
+
* to a temporary file beside the target, then is renamed over it. Synchronous;
|
|
10
|
+
* throws on failure and removes its temporary file when it can. It never writes
|
|
11
|
+
* `filePath` itself.
|
|
12
|
+
*
|
|
13
|
+
* @param {string} filePath
|
|
14
|
+
* @param {string | Uint8Array} data
|
|
15
|
+
* @param {{ fs?: Partial<typeof nodeFs>, mode?: number }} [options] - `fs`
|
|
16
|
+
* overrides individual `node:fs` functions (a check stops the write between
|
|
17
|
+
* the two steps with it).
|
|
18
|
+
* @returns {void}
|
|
19
|
+
*/
|
|
20
|
+
export function writeFileAtomic(filePath, data, { fs = {}, mode } = {}) {
|
|
21
|
+
const io = { ...nodeFs, ...fs };
|
|
22
|
+
const temporary = path.join(path.dirname(filePath), `${path.basename(filePath)}.${process.pid}.${++writes}.tmp`);
|
|
23
|
+
try {
|
|
24
|
+
io.writeFileSync(temporary, data, mode === undefined ? undefined : { mode });
|
|
25
|
+
io.renameSync(temporary, filePath);
|
|
26
|
+
} catch (error) {
|
|
27
|
+
try {
|
|
28
|
+
nodeFs.rmSync(temporary, { force: true });
|
|
29
|
+
} catch {
|
|
30
|
+
// The write already failed; a temporary file that cannot be removed is the lesser fault.
|
|
31
|
+
}
|
|
32
|
+
throw error;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Serialise `value` as JSON and write it with {@link writeFileAtomic}.
|
|
38
|
+
*
|
|
39
|
+
* @param {string} filePath
|
|
40
|
+
* @param {unknown} value
|
|
41
|
+
* @param {{ fs?: Partial<typeof nodeFs>, mode?: number }} [options]
|
|
42
|
+
* @returns {void}
|
|
43
|
+
*/
|
|
44
|
+
export function writeJsonAtomic(filePath, value, options = {}) {
|
|
45
|
+
writeFileAtomic(filePath, `${JSON.stringify(value)}\n`, options);
|
|
46
|
+
}
|