@torrent-tv/proxy 2.91.0 → 2.92.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/cli.js +3 -2
- package/package.json +18 -2
- package/routes/api/playback-plan/post.js +21 -4
- package/routes/api/sources/files/get.js +20 -28
- package/routes/api/sources/post.js +4 -2
- package/routes/api/sources/stats/get.js +4 -11
- package/routes/api/sources/viewer/post.js +16 -0
- package/routes/api/sources/warm/post.js +6 -215
- package/routes/api/subtitles/get.js +22 -125
- package/routes/api/subtitles/post.js +26 -0
- package/routes/api/transcode-sessions/post.js +24 -2
- package/routes/media/get.js +80 -0
- package/routes/stream/get.js +26 -94
- package/routes/transcode/audio-warm/get.js +9 -21
- package/routes/transcode/variant-warm/get.js +8 -14
- package/server.js +410 -46
- package/services/encode/AdmittedInput.js +136 -0
- package/services/encode/CushionReport.js +1 -76
- package/services/encode/EncodeInputs.js +142 -0
- package/services/encode/EncodeOrchestrator.js +4 -1
- package/services/encode/EncodeRun.js +13 -4
- package/services/encode/EncodeRuns.js +132 -52
- package/services/encode/InputFailures.js +37 -0
- package/services/encode/MatroskaInput.js +158 -0
- package/services/encode/OutputOpening.js +16 -64
- package/services/encode/Renditions.js +1 -17
- package/services/encode/admitted-command.js +60 -0
- package/services/encode/annex-b-input.js +37 -0
- package/services/encode/bit-payload.js +29 -0
- package/services/encode/output/OutputCatalog.js +1 -1
- package/services/encode/output/Timeline.js +11 -0
- package/services/encode/pcm-normalization.js +30 -0
- package/services/encode/piece-completeness.js +54 -1
- package/services/encode/quality/HostLoad.js +19 -73
- package/services/encode/quality/QualityOffer.js +1 -1
- package/services/encode/run-command.js +1 -589
- package/services/encode/segment-formats/mp4-boxes.js +3 -1
- package/services/encode/segment-formats/presentation-segment.js +114 -0
- package/services/media/ContainerOrchestrator.js +118 -48
- package/services/media/KeyframeTables.js +16 -115
- package/services/media/MediaReadRequests.js +120 -0
- package/services/media/ProbeRequests.js +135 -0
- package/services/media/SegmentInputs.js +48 -0
- package/services/media/SourceFile.js +6 -0
- package/services/media/SubtitleOrchestrator.js +190 -4
- package/services/media/await-plan.js +47 -0
- package/services/media/container/AsfContainer.js +217 -0
- package/services/media/container/AviContainer.js +343 -34
- package/services/media/container/CadencedPictureOrder.js +45 -0
- package/services/media/container/Container.js +7 -3
- package/services/media/container/ContainerFactory.js +10 -3
- package/services/media/container/FfprobeContainer.js +222 -0
- package/services/media/container/MatroskaContainer.js +72 -46
- package/services/media/container/Mp4Container.js +472 -139
- package/services/media/container/MpegPsContainer.js +281 -0
- package/services/media/container/MpegTsContainer.js +277 -0
- package/services/media/container/PacketIndex.js +270 -0
- package/services/media/container/PacketRecords.js +237 -0
- package/services/media/container/RetainedBytes.js +39 -0
- package/services/media/container/RetainedReads.js +51 -0
- package/services/media/container/ac3-frame.js +46 -0
- package/services/media/container/asf-packets.js +153 -0
- package/services/media/container/avc-picture-order.js +50 -0
- package/services/media/container/avi-open-dml.js +47 -0
- package/services/media/container/dts-frame.js +51 -0
- package/services/media/container/ebml-stream.js +12 -1
- package/services/media/container/ffprobe-record.js +45 -0
- package/services/media/container/flv-packet-ranges.js +27 -0
- package/services/media/container/following-packet-time.js +37 -0
- package/services/media/container/h264-configuration.js +183 -0
- package/services/media/container/h264-elementary-index.js +245 -0
- package/services/media/container/hevc-configuration.js +165 -0
- package/services/media/container/hevc-elementary-index.js +132 -0
- package/services/media/container/hevc-picture-order.js +66 -0
- package/services/media/container/latm-frame.js +99 -0
- package/services/media/container/matroska-packets.js +207 -0
- package/services/media/container/memory-unavailable.js +8 -0
- package/services/media/container/mp3-packet-dependencies.js +41 -0
- package/services/media/container/mp4-fragments.js +148 -0
- package/services/media/container/mpeg-audio-frame.js +19 -0
- package/services/media/container/mpeg-audio-reservoir.js +27 -0
- package/services/media/container/mpeg-elementary-index.js +196 -0
- package/services/media/container/mpeg-sequence-reader.js +69 -0
- package/services/media/container/mpeg-ts-packets.js +93 -0
- package/services/media/container/mpeg-video-configuration.js +50 -0
- package/services/media/container/mpeg4-picture-timing.js +108 -0
- package/services/media/container/pes-header.js +34 -0
- package/services/media/playback-declarations.js +57 -0
- package/services/media/playback-planner.js +72 -230
- package/services/media/probe-packets.js +66 -0
- package/services/media/read-playback-declarations.js +18 -0
- package/services/media/tracks/AudioTrack.js +9 -0
- package/services/media/tracks/TextSubtitleTrack.js +4 -1
- package/services/server/OutputLifecycle.js +2 -0
- package/services/server/SegmentServing.js +40 -33
- package/services/server/ViewerRequests.js +22 -41
- package/services/server/controllers/PlaybackController.js +4 -0
- package/services/server/controllers/SubtitleController.js +8 -29
- package/services/server/transcode-session-files.js +58 -99
- package/services/server/wire-outputs.js +58 -15
- package/services/storage/IndexMemory.js +85 -0
- package/services/storage/MachineBudget.js +27 -3
- package/services/storage/allowance.js +8 -1
- package/services/storage/piece-store/piece-disk-store.js +18 -3
- package/services/storage/piece-store/piece-lru.js +8 -4
- package/services/storage/piece-store/shared-piece-store.js +26 -11
- package/services/storage/segment-store/SegmentStore.js +49 -18
- package/services/storage/wire.js +7 -0
- package/services/torrent/Contents.js +52 -20
- package/services/torrent/demand/DemandRegister.js +13 -4
- package/services/torrent/demand/Window.js +13 -2
- package/services/torrent/demand-progress.js +25 -0
- package/services/torrent/download/DeadlineScheduler.js +167 -0
- package/services/torrent/download/FutureDownload.js +166 -0
- package/services/torrent/download/SwarmSelection.js +30 -59
- package/services/torrent/download/registry.js +45 -5
- package/services/torrent/download/withdraw-claim.js +2 -0
- package/services/torrent/torrent-pool.js +39 -600
- package/services/torrent/worker/channel.js +8 -1
- package/services/torrent/worker/client.js +74 -72
- package/services/torrent/worker/files-in-use.js +9 -0
- package/services/torrent/worker/handle-state.js +12 -0
- package/services/torrent/worker/held-bytes.js +82 -99
- package/services/torrent/worker/held-file-bytes.js +17 -0
- package/services/torrent/worker/piece-reader.js +67 -683
- package/services/torrent/worker/pool-adapter.js +37 -123
- package/services/torrent/worker/protocol.js +6 -17
- package/services/torrent/worker/whole-file-stats.js +2 -0
- package/services/torrent/worker/worker.js +86 -156
- package/services/transport/data-channel-handler.js +56 -13
- package/services/viewer/DownloadMaps.js +168 -0
- package/services/viewer/NativeSourceMap.js +33 -0
- package/services/viewer/PriorityMap.js +49 -40
- package/services/viewer/PriorityOrchestrator.js +42 -13
- package/services/viewer/SourcePreparation.js +278 -0
- package/services/viewer/Viewer.js +11 -0
- package/services/viewer/Viewers.js +109 -2
- package/services/viewer/playback-readiness.js +50 -63
- package/services/viewer/segment-demands.js +37 -0
- package/services/viewer/urgent-output-ready.js +26 -0
- package/services/encode/run-inputs.js +0 -53
- package/services/media/keyframe-probe.js +0 -157
- package/services/media/media-info-probe.js +0 -93
- package/services/torrent/worker/background-fill.js +0 -213
- package/services/torrent/worker/resume-warm.js +0 -107
|
@@ -20,21 +20,19 @@
|
|
|
20
20
|
// before WebTorrent can reach the native one. Two isolates using
|
|
21
21
|
// node-datachannel at once abort the process, and the torrent's wss trackers
|
|
22
22
|
// create peer connections of their own.
|
|
23
|
-
import { isUsableTorrentHandle } from "./handle-state.js";
|
|
23
|
+
import { isUsableTorrentHandle, requireCurrentTorrent } from "./handle-state.js";
|
|
24
24
|
import "./install-webrtc-shim.js";
|
|
25
25
|
import { parentPort, workerData } from "node:worker_threads";
|
|
26
|
+
import { Readable } from "node:stream";
|
|
27
|
+
import { heldFileBytes } from "./held-file-bytes.js";
|
|
26
28
|
import { createSendStream } from "./channel.js";
|
|
27
29
|
import { readFragments, supplyFiguresFor } from "./piece-reader.js";
|
|
28
|
-
import {
|
|
29
|
-
|
|
30
|
-
} from "./resume-warm.js";
|
|
31
|
-
import { fillFileInBackground } from "./background-fill.js";
|
|
32
|
-
import { demandFor } from "../download/registry.js";
|
|
33
|
-
import { heldRangesOf, ownsItsMemory, readHeldBytes } from "./held-bytes.js";
|
|
30
|
+
import { demandFor, forecastDownloads } from "../download/registry.js";
|
|
31
|
+
import { heldRangesOf, ownsItsMemory, readHeldBytes, readHeldRanges } from "./held-bytes.js";
|
|
34
32
|
import { CompletedFiles, completedFilesRoot } from "../../storage/files/CompletedFiles.js";
|
|
35
33
|
import { pieceFromWholeFiles, pieceIsInWholeFiles } from "../../storage/files/piece-from-whole-file.js";
|
|
36
34
|
import { Command, Event } from "./protocol.js";
|
|
37
|
-
import { filesInUse } from "./files-in-use.js";
|
|
35
|
+
import { filesInUse, readWhileInUse } from "./files-in-use.js";
|
|
38
36
|
import { createWholeSources } from "./whole-sources.js";
|
|
39
37
|
import { wholeFileStats } from "./whole-file-stats.js";
|
|
40
38
|
import { describeHeldObjects, readUtpOpenCount } from "./held-objects.js";
|
|
@@ -88,15 +86,9 @@ const pool = new TorrentPool({
|
|
|
88
86
|
|
|
89
87
|
/** Torrents by sourceKey — the main thread names them, this thread owns them. */
|
|
90
88
|
const torrentsByKey = new Map();
|
|
89
|
+
/** Resolved handles for reads that must never wait for or revive a source. */
|
|
90
|
+
const resolvedTorrents = new Map();
|
|
91
91
|
|
|
92
|
-
/**
|
|
93
|
-
* How each source was named when it was added, so a torrent that has since been
|
|
94
|
-
* destroyed can be added again. Kept separately from {@link torrentsByKey}
|
|
95
|
-
* because that map holds the promise, not the recipe.
|
|
96
|
-
*
|
|
97
|
-
* @type {Map<string, { sourceType: string, source: string }>}
|
|
98
|
-
*/
|
|
99
|
-
const sourceRecipes = new Map();
|
|
100
92
|
/** In-flight reads, so a cancel can stop one mid-body. */
|
|
101
93
|
const readsById = new Map();
|
|
102
94
|
/**
|
|
@@ -154,57 +146,19 @@ async function knownTorrent(sourceKey) {
|
|
|
154
146
|
return null;
|
|
155
147
|
}
|
|
156
148
|
const torrent = await pending.catch(() => null);
|
|
157
|
-
return isUsableTorrentHandle(torrent) ? torrent : null;
|
|
149
|
+
return torrentsByKey.get(sourceKey) === pending && isUsableTorrentHandle(torrent) ? torrent : null;
|
|
158
150
|
}
|
|
159
151
|
|
|
160
152
|
/**
|
|
161
|
-
*
|
|
162
|
-
*
|
|
163
|
-
* For the questions that steer or measure a download: a file held whole has no
|
|
164
|
-
* download, and rebuilding its torrent to be told so is what made the torrent
|
|
165
|
-
* come back after every removal (see `whole-sources.js`). The torrent is still
|
|
166
|
-
* used while it exists.
|
|
153
|
+
* Resolve a live torrent handle and retain its synchronous lookup.
|
|
167
154
|
*
|
|
168
155
|
* @param {string} sourceKey
|
|
169
|
-
* @param {number} fileIndex
|
|
170
156
|
* @returns {Promise<import("webtorrent").Torrent | null>}
|
|
171
157
|
*/
|
|
172
|
-
async function torrentUnlessWhole(sourceKey, fileIndex) {
|
|
173
|
-
return wholeSources.fileOf(sourceKey, fileIndex) ? knownTorrent(sourceKey) : requireTorrent(sourceKey);
|
|
174
|
-
}
|
|
175
|
-
|
|
176
158
|
async function requireTorrent(sourceKey) {
|
|
177
|
-
const
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
}
|
|
181
|
-
const torrent = await pending;
|
|
182
|
-
if (isUsableTorrentHandle(torrent)) {
|
|
183
|
-
return torrent;
|
|
184
|
-
}
|
|
185
|
-
// The pool destroys a torrent that has gone unread for a quarter of an hour,
|
|
186
|
-
// and under disk pressure. It clears its OWN map when it does; this one it
|
|
187
|
-
// knows nothing about, so the promise here went on resolving to a corpse: a
|
|
188
|
-
// destroyed torrent keeps its object but loses its files. Every later session
|
|
189
|
-
// for that source then failed the same way — the plan and the codec probe
|
|
190
|
-
// answered from cache in milliseconds, nothing waited for metadata because
|
|
191
|
-
// everything believed the torrent was known, and ffmpeg's first read died on
|
|
192
|
-
// `File N not found` 130 ms in, after which the session answered 500 for
|
|
193
|
-
// ever. Measured 2026-08-06 on two sessions in a row, both from a phone,
|
|
194
|
-
// which is what made it look like a mobile problem.
|
|
195
|
-
const recipe = sourceRecipes.get(sourceKey);
|
|
196
|
-
if (!recipe) {
|
|
197
|
-
torrentsByKey.delete(sourceKey);
|
|
198
|
-
throw new Error(`Source ${sourceKey} is gone and cannot be re-added.`);
|
|
199
|
-
}
|
|
200
|
-
const revived = pool.getTorrent(recipe.sourceType, recipe.source);
|
|
201
|
-
torrentsByKey.set(sourceKey, revived);
|
|
202
|
-
revived.catch(() => {
|
|
203
|
-
if (torrentsByKey.get(sourceKey) === revived) {
|
|
204
|
-
torrentsByKey.delete(sourceKey);
|
|
205
|
-
}
|
|
206
|
-
});
|
|
207
|
-
return revived;
|
|
159
|
+
const torrent = await requireCurrentTorrent(torrentsByKey, sourceKey);
|
|
160
|
+
resolvedTorrents.set(sourceKey, torrent);
|
|
161
|
+
return torrent;
|
|
208
162
|
}
|
|
209
163
|
|
|
210
164
|
|
|
@@ -277,7 +231,7 @@ function sendFragment(id, fragment) {
|
|
|
277
231
|
* @param {number | null} params.end - Inclusive.
|
|
278
232
|
* @returns {Promise<void>}
|
|
279
233
|
*/
|
|
280
|
-
async function streamRange({ id, sourceKey, fileIndex, start, end
|
|
234
|
+
async function streamRange({ id, sourceKey, fileIndex, start, end }) {
|
|
281
235
|
const torrent = await requireTorrent(sourceKey);
|
|
282
236
|
const file = torrent.files?.[fileIndex];
|
|
283
237
|
if (!file) {
|
|
@@ -301,8 +255,7 @@ async function streamRange({ id, sourceKey, fileIndex, start, end, windowBytes }
|
|
|
301
255
|
fileIndex,
|
|
302
256
|
start: rangeStart,
|
|
303
257
|
end: rangeEnd,
|
|
304
|
-
cancellation: sender
|
|
305
|
-
windowBytes
|
|
258
|
+
cancellation: sender
|
|
306
259
|
})) {
|
|
307
260
|
if (sender.isCancelled()) {
|
|
308
261
|
fragment.release();
|
|
@@ -349,14 +302,15 @@ async function streamRange({ id, sourceKey, fileIndex, start, end, windowBytes }
|
|
|
349
302
|
async function runCommand(command, params, id) {
|
|
350
303
|
switch (command) {
|
|
351
304
|
case Command.ADD_SOURCE: {
|
|
305
|
+
const retained = wholeSources.describe(params.sourceKey);
|
|
306
|
+
if (retained) {
|
|
307
|
+
announceWholeFiles(params.sourceKey, retained);
|
|
308
|
+
return retained;
|
|
309
|
+
}
|
|
352
310
|
// Registered before it resolves, so anything naming this source while it
|
|
353
311
|
// is being added waits for it instead of being told it does not exist.
|
|
354
312
|
// Reusing the same promise for a repeated add also collapses two callers
|
|
355
313
|
// racing to open the same torrent into one.
|
|
356
|
-
sourceRecipes.set(params.sourceKey, {
|
|
357
|
-
sourceType: params.sourceType,
|
|
358
|
-
source: params.source
|
|
359
|
-
});
|
|
360
314
|
let pending = torrentsByKey.get(params.sourceKey);
|
|
361
315
|
if (!pending) {
|
|
362
316
|
pending = pool.getTorrent(params.sourceType, params.source);
|
|
@@ -379,9 +333,14 @@ async function runCommand(command, params, id) {
|
|
|
379
333
|
const added = await pending;
|
|
380
334
|
const described = isUsableTorrentHandle(added) ? null : wholeSources.describe(params.sourceKey);
|
|
381
335
|
if (described) {
|
|
336
|
+
announceWholeFiles(params.sourceKey, described);
|
|
382
337
|
return described;
|
|
383
338
|
}
|
|
384
339
|
const torrent = isUsableTorrentHandle(added) ? added : await requireTorrent(params.sourceKey);
|
|
340
|
+
resolvedTorrents.set(params.sourceKey, torrent);
|
|
341
|
+
ensureArrivalsWired(params.sourceKey, torrent);
|
|
342
|
+
announceWholeFiles(params.sourceKey, { infoHash: torrent.infoHash,
|
|
343
|
+
files: (torrent.files ?? []).map((file, index) => ({ index, name: file.name, length: file.length })) });
|
|
385
344
|
return {
|
|
386
345
|
infoHash: torrent.infoHash,
|
|
387
346
|
name: torrent.name,
|
|
@@ -453,51 +412,12 @@ async function runCommand(command, params, id) {
|
|
|
453
412
|
}
|
|
454
413
|
|
|
455
414
|
|
|
456
|
-
case Command.FILL_FILE: {
|
|
457
|
-
const torrent = await torrentUnlessWhole(params.sourceKey, params.fileIndex);
|
|
458
|
-
return {
|
|
459
|
-
started: torrent ? fillFileInBackground(torrent, params.fileIndex, params.sourceKey) : false
|
|
460
|
-
};
|
|
461
|
-
}
|
|
462
|
-
|
|
463
|
-
case Command.FILL_TORRENT: {
|
|
464
|
-
const torrent = wholeSources.isWhole(params.sourceKey)
|
|
465
|
-
? await knownTorrent(params.sourceKey)
|
|
466
|
-
: await requireTorrent(params.sourceKey);
|
|
467
|
-
return { started: torrent ? pool.fillTorrentAsCapacityAllows(torrent) : false };
|
|
468
|
-
}
|
|
469
|
-
|
|
470
|
-
case Command.WARM_POSITION: {
|
|
471
|
-
const torrent = await torrentUnlessWhole(params.sourceKey, params.fileIndex);
|
|
472
|
-
if (!torrent) {
|
|
473
|
-
return { started: false };
|
|
474
|
-
}
|
|
475
|
-
return {
|
|
476
|
-
started: await warmResumePosition(
|
|
477
|
-
torrent,
|
|
478
|
-
params.fileIndex,
|
|
479
|
-
params.sourceKey,
|
|
480
|
-
params.positionSeconds,
|
|
481
|
-
{
|
|
482
|
-
// Told by the main thread, which is where what a file states about
|
|
483
|
-
// itself is read.
|
|
484
|
-
durationSeconds: params.durationSeconds,
|
|
485
|
-
fetchRegion: (start, bytes) =>
|
|
486
|
-
pool.prefetchFileRegion(torrent, params.fileIndex, start, bytes)
|
|
487
|
-
}
|
|
488
|
-
)
|
|
489
|
-
};
|
|
490
|
-
}
|
|
491
|
-
|
|
492
|
-
|
|
493
415
|
case Command.FILE_STATS: {
|
|
494
|
-
const torrent = await
|
|
416
|
+
const torrent = await knownTorrent(params.sourceKey);
|
|
495
417
|
if (!torrent) {
|
|
496
|
-
|
|
418
|
+
const whole = wholeSources.fileOf(params.sourceKey, params.fileIndex);
|
|
419
|
+
return whole ? wholeFileStats(whole.length) : { pending: true };
|
|
497
420
|
}
|
|
498
|
-
const stats = pool.getFileStats(torrent, params.fileIndex, {
|
|
499
|
-
resumeAnchorByteStart: params.resumeAnchorByteStart ?? null
|
|
500
|
-
});
|
|
501
421
|
// What this file's own interruptions demand, measured by the reader in
|
|
502
422
|
// this thread. It travels with the stats because the caller asking for
|
|
503
423
|
// them is the one that has to decide with them — the browser's smallest
|
|
@@ -505,44 +425,37 @@ async function runCommand(command, params, id) {
|
|
|
505
425
|
// second interruption has been seen: one wait shows no interval, and an
|
|
506
426
|
// interval invented from one point is exactly what this work removes.
|
|
507
427
|
const file = Array.isArray(torrent?.files) ? torrent.files[params.fileIndex] : null;
|
|
428
|
+
const forecast = file && params.forecast === true ? (await forecastDownloads()).get(torrent) : null;
|
|
429
|
+
const stats = pool.getFileStats(torrent, params.fileIndex);
|
|
430
|
+
const ranges = [];
|
|
431
|
+
if (forecast) for (const [piece, availableAt] of forecast.arrivals) {
|
|
432
|
+
const start = piece * torrent.pieceLength;
|
|
433
|
+
const end = Math.min(torrent.length, (piece + 1) * torrent.pieceLength);
|
|
434
|
+
if (end > start) ranges.push({ start, end, availableAt });
|
|
435
|
+
}
|
|
508
436
|
return {
|
|
509
437
|
...stats,
|
|
438
|
+
...(params.forecast === true ? { downloadForecast: {
|
|
439
|
+
measuredAt: forecast?.measuredAt ?? Date.now(), global: true, ranges
|
|
440
|
+
} } : {}),
|
|
510
441
|
supply: supplyFiguresFor(torrent?.infoHash, file?.name, params.segmentSeconds ?? 4)
|
|
511
442
|
};
|
|
512
443
|
}
|
|
513
444
|
|
|
514
445
|
case Command.PRIORITY_MAP: {
|
|
515
|
-
//
|
|
516
|
-
//
|
|
517
|
-
// torrent may be on its way out — and `requireTorrent` rebuilds a dead
|
|
518
|
-
// handle from its recipe, so asking that way would re-add a torrent in
|
|
519
|
-
// order to be told that nothing is wanted of it.
|
|
446
|
+
// Source preparation owns adding the source. A delayed publication may
|
|
447
|
+
// only update an existing source, never revive one after its viewer left.
|
|
520
448
|
const zones = Array.isArray(params.zones) ? params.zones : [];
|
|
521
449
|
// Nor may one for a file held whole, which has nothing left to fetch.
|
|
522
|
-
const torrent =
|
|
523
|
-
? await knownTorrent(params.sourceKey)
|
|
524
|
-
: await torrentUnlessWhole(params.sourceKey, params.fileIndex);
|
|
450
|
+
const torrent = await knownTorrent(params.sourceKey);
|
|
525
451
|
if (torrent) {
|
|
452
|
+
ensureArrivalsWired(params.sourceKey, torrent);
|
|
526
453
|
pool.applyPriorityMap(torrent, params.fileIndex, zones, params.durationSeconds);
|
|
454
|
+
torrent.emit("priority-map-changed", params.fileIndex);
|
|
527
455
|
}
|
|
528
456
|
return true;
|
|
529
457
|
}
|
|
530
458
|
|
|
531
|
-
case Command.PRIORITIZE: {
|
|
532
|
-
const torrent = await torrentUnlessWhole(params.sourceKey, params.fileIndex);
|
|
533
|
-
if (torrent) {
|
|
534
|
-
pool.prioritizeByteRange(torrent, params.fileIndex, params.byteStart, params.windowBytes, {
|
|
535
|
-
wholeFileRead: params.wholeFileRead === true
|
|
536
|
-
});
|
|
537
|
-
}
|
|
538
|
-
return true;
|
|
539
|
-
}
|
|
540
|
-
|
|
541
|
-
case Command.PREFETCH_EDGES: {
|
|
542
|
-
const torrent = await torrentUnlessWhole(params.sourceKey, params.fileIndex);
|
|
543
|
-
return torrent ? pool.prefetchFileEdges(torrent, params.fileIndex, params.options ?? {}) : undefined;
|
|
544
|
-
}
|
|
545
|
-
|
|
546
459
|
case Command.READ_RANGE: {
|
|
547
460
|
// Streams its own reply; the caller's promise resolves once the body has
|
|
548
461
|
// been fully sent, which is what lets the client await completion.
|
|
@@ -552,7 +465,6 @@ async function runCommand(command, params, id) {
|
|
|
552
465
|
fileIndex: params.fileIndex,
|
|
553
466
|
start: params.start ?? null,
|
|
554
467
|
end: params.end ?? null,
|
|
555
|
-
windowBytes: params.windowBytes
|
|
556
468
|
});
|
|
557
469
|
return true;
|
|
558
470
|
}
|
|
@@ -566,7 +478,8 @@ async function runCommand(command, params, id) {
|
|
|
566
478
|
}
|
|
567
479
|
|
|
568
480
|
case Command.HELD_RANGES: {
|
|
569
|
-
const
|
|
481
|
+
const known = resolvedTorrents.get(params.sourceKey);
|
|
482
|
+
const torrent = isUsableTorrentHandle(known) ? known : null;
|
|
570
483
|
if (!torrent) {
|
|
571
484
|
const length = wholeSources.fileOf(params.sourceKey, params.fileIndex)?.length ?? 0;
|
|
572
485
|
return { ranges: length > 0 ? [[0, length - 1]] : [] };
|
|
@@ -575,11 +488,21 @@ async function runCommand(command, params, id) {
|
|
|
575
488
|
}
|
|
576
489
|
|
|
577
490
|
case Command.READ_HELD: {
|
|
578
|
-
const
|
|
579
|
-
const
|
|
491
|
+
const known = resolvedTorrents.get(params.sourceKey);
|
|
492
|
+
const torrent = isUsableTorrentHandle(known) ? known : null;
|
|
493
|
+
const bytes = await readWhileInUse(openReads, id, torrent, params.fileIndex,
|
|
494
|
+
() => readHeldBytes(torrent, params.fileIndex, params.start, params.end, logger));
|
|
580
495
|
return { bytes };
|
|
581
496
|
}
|
|
582
497
|
|
|
498
|
+
case Command.READ_HELD_RANGES: {
|
|
499
|
+
const known = resolvedTorrents.get(params.sourceKey);
|
|
500
|
+
const torrent = isUsableTorrentHandle(known) ? known : null;
|
|
501
|
+
const buffers = await readWhileInUse(openReads, id, torrent, params.fileIndex,
|
|
502
|
+
() => readHeldRanges(torrent, params.fileIndex, params.ranges, params.maxBytes, logger));
|
|
503
|
+
return { buffers };
|
|
504
|
+
}
|
|
505
|
+
|
|
583
506
|
case Command.SPILL_ALLOWANCE: {
|
|
584
507
|
// The disk has one owner and it is on the main thread, where the segments
|
|
585
508
|
// are. What arrives is this thread’s whole share; the stores divide it
|
|
@@ -604,7 +527,7 @@ async function runCommand(command, params, id) {
|
|
|
604
527
|
|
|
605
528
|
case Command.DESTROY_ALL: {
|
|
606
529
|
torrentsByKey.clear();
|
|
607
|
-
|
|
530
|
+
resolvedTorrents.clear();
|
|
608
531
|
await pool.destroyAll();
|
|
609
532
|
return true;
|
|
610
533
|
}
|
|
@@ -635,10 +558,11 @@ parentPort.on("message", async (message) => {
|
|
|
635
558
|
// A held read's bytes are handed over rather than copied, and only when
|
|
636
559
|
// the buffer is the whole of its own memory (`ownsItsMemory`): anything
|
|
637
560
|
// else would take memory the store or another read still uses with it.
|
|
638
|
-
const transfer = command === Command.READ_HELD && ownsItsMemory(result?.bytes) ? [result.bytes.buffer]
|
|
561
|
+
const transfer = command === Command.READ_HELD && ownsItsMemory(result?.bytes) ? [result.bytes.buffer]
|
|
562
|
+
: command === Command.READ_HELD_RANGES && result?.buffers?.every(ownsItsMemory) ? result.buffers.map(bytes => bytes.buffer) : [];
|
|
639
563
|
parentPort.postMessage({ type: Event.RESULT, id, result }, transfer);
|
|
640
564
|
} catch (error) {
|
|
641
|
-
parentPort.postMessage({ type: Event.ERROR, id, error: error?.message ?? String(error) });
|
|
565
|
+
parentPort.postMessage({ type: Event.ERROR, id, error: error?.message ?? String(error), code: error?.code });
|
|
642
566
|
}
|
|
643
567
|
});
|
|
644
568
|
|
|
@@ -886,12 +810,12 @@ function describePieceBuffers() {
|
|
|
886
810
|
* @param {object} torrent
|
|
887
811
|
* @returns {void}
|
|
888
812
|
*/
|
|
889
|
-
function announceArrivals(sourceKey, torrent) {
|
|
813
|
+
function announceArrivals(sourceKey, torrent, reason = "verified") {
|
|
890
814
|
const fileIndexes = [...demandFor(torrent).register.files()];
|
|
891
815
|
if (fileIndexes.length === 0) {
|
|
892
816
|
return;
|
|
893
817
|
}
|
|
894
|
-
parentPort.postMessage({ type: Event.
|
|
818
|
+
parentPort.postMessage({ type: Event.PIECES_CHANGED, sourceKey, fileIndexes, reason });
|
|
895
819
|
}
|
|
896
820
|
|
|
897
821
|
|
|
@@ -922,23 +846,18 @@ function ensureArrivalsWired(sourceKey, torrent) {
|
|
|
922
846
|
}
|
|
923
847
|
arrivalsWired.add(torrent);
|
|
924
848
|
torrent.on("verified", () => announceArrivals(sourceKey, torrent));
|
|
849
|
+
torrent.on("piece-withdrawn", () => announceArrivals(sourceKey, torrent, "withdrawn"));
|
|
850
|
+
torrent.once("close", () => {
|
|
851
|
+
if (resolvedTorrents.get(sourceKey) !== torrent) return;
|
|
852
|
+
resolvedTorrents.delete(sourceKey);
|
|
853
|
+
torrentsByKey.delete(sourceKey);
|
|
854
|
+
if (wholeSources.isWhole(sourceKey)) {
|
|
855
|
+
const retained = wholeSources.describe(sourceKey);
|
|
856
|
+
announceWholeFiles(sourceKey, retained);
|
|
857
|
+
} else parentPort.postMessage({ type: Event.SOURCE_FORGOTTEN, sourceKey });
|
|
858
|
+
});
|
|
925
859
|
}
|
|
926
860
|
|
|
927
|
-
/**
|
|
928
|
-
* How often arrivals are announced regardless, as a fallback beside the
|
|
929
|
-
* per-piece `verified` listener above — it catches a listener attached after
|
|
930
|
-
* pieces had already verified, and anything the event path might otherwise
|
|
931
|
-
* miss. Cheap: the walk it wakes skips clusters it has already read.
|
|
932
|
-
*/
|
|
933
|
-
const ARRIVAL_ANNOUNCE_INTERVAL_MS = 3_000;
|
|
934
|
-
|
|
935
|
-
setInterval(() => {
|
|
936
|
-
for (const [sourceKey, torrent] of pool.torrents) {
|
|
937
|
-
ensureArrivalsWired(sourceKey, torrent);
|
|
938
|
-
announceArrivals(sourceKey, torrent);
|
|
939
|
-
}
|
|
940
|
-
}, ARRIVAL_ANNOUNCE_INTERVAL_MS).unref();
|
|
941
|
-
|
|
942
861
|
/**
|
|
943
862
|
* Files this proxy has downloaded whole. One directory, two readers of it: this
|
|
944
863
|
* thread writes them, the main thread serves them without asking anybody.
|
|
@@ -946,6 +865,16 @@ setInterval(() => {
|
|
|
946
865
|
const completedFiles = new CompletedFiles({ root: completedFilesRoot() });
|
|
947
866
|
const wholeSources = createWholeSources({ find: (infoHash, fileIndex) => completedFiles.find(infoHash, fileIndex) });
|
|
948
867
|
|
|
868
|
+
/** Adopted files and newly completed files state the same availability event. */
|
|
869
|
+
function announceWholeFiles(sourceKey, description) {
|
|
870
|
+
for (const file of description?.files ?? []) {
|
|
871
|
+
const held = completedFiles.find(description.infoHash, file.index);
|
|
872
|
+
if (!held || held.length !== file.length) continue;
|
|
873
|
+
parentPort.postMessage({ type: Event.FILE_COMPLETE, sourceKey, infoHash: description.infoHash,
|
|
874
|
+
fileIndex: file.index, path: held.path, length: held.length, name: file.name });
|
|
875
|
+
}
|
|
876
|
+
}
|
|
877
|
+
|
|
949
878
|
/**
|
|
950
879
|
* Which torrent a set of files belongs to.
|
|
951
880
|
*
|
|
@@ -1049,7 +978,7 @@ async function keepWholeFiles() {
|
|
|
1049
978
|
fileIndex,
|
|
1050
979
|
length: file.length,
|
|
1051
980
|
name: file.name,
|
|
1052
|
-
open: () =>
|
|
981
|
+
open: () => Readable.from(heldFileBytes(torrent, fileIndex))
|
|
1053
982
|
});
|
|
1054
983
|
if (kept) {
|
|
1055
984
|
logger.info(
|
|
@@ -1069,6 +998,7 @@ async function keepWholeFiles() {
|
|
|
1069
998
|
}
|
|
1070
999
|
parentPort.postMessage({
|
|
1071
1000
|
type: Event.FILE_COMPLETE,
|
|
1001
|
+
sourceKey,
|
|
1072
1002
|
infoHash,
|
|
1073
1003
|
fileIndex,
|
|
1074
1004
|
path: kept.path,
|
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
* Browser → Proxy
|
|
12
12
|
* ```
|
|
13
13
|
* { type: "request", requestId, method, path, query, headers, body }
|
|
14
|
+
* { type: "request-cancel", requestId }
|
|
14
15
|
* { type: "ping", id }
|
|
15
16
|
* { type: "probe-echo", seen: { <label>: seq }, report }
|
|
16
17
|
* ```
|
|
@@ -46,6 +47,10 @@
|
|
|
46
47
|
/** @import { DataChannel } from 'node-datachannel' */
|
|
47
48
|
|
|
48
49
|
import { createDeliveryProbe, PROBE_INTERVAL_MS } from "./delivery-probe.js";
|
|
50
|
+
import { Agent } from "undici";
|
|
51
|
+
|
|
52
|
+
// Route availability and caller cancellation determine media request lifetime.
|
|
53
|
+
const loopbackDispatcher = new Agent({ headersTimeout: 0, bodyTimeout: 0 });
|
|
49
54
|
|
|
50
55
|
/**
|
|
51
56
|
* Configuration for the data channel handler.
|
|
@@ -854,6 +859,8 @@ export function createDataChannelHandler({
|
|
|
854
859
|
|
|
855
860
|
/** Request id → its ASCII bytes; see {@link requestIdBytes}. */
|
|
856
861
|
const requestIdCache = new Map();
|
|
862
|
+
/** Channel-scoped requests cannot be cancelled by a different connection. */
|
|
863
|
+
const activeRequests = new WeakMap();
|
|
857
864
|
|
|
858
865
|
const { watchSendQueue, readDelivery, noteBrowserReport } = makeSendQueueWatcher({
|
|
859
866
|
log: (message) => log(message),
|
|
@@ -912,6 +919,8 @@ export function createDataChannelHandler({
|
|
|
912
919
|
// assembled request through the same path as a single-message request.
|
|
913
920
|
/** @type {Map<string, { meta: object, chunks: Buffer[], receivedBytes: number, bodyBytes: number, timer: ReturnType<typeof setTimeout> }>} */
|
|
914
921
|
const partials = new Map();
|
|
922
|
+
const requests = new Map();
|
|
923
|
+
activeRequests.set(channel, requests);
|
|
915
924
|
|
|
916
925
|
const dropPartial = (requestId) => {
|
|
917
926
|
const entry = partials.get(requestId);
|
|
@@ -1032,6 +1041,12 @@ export function createDataChannelHandler({
|
|
|
1032
1041
|
return;
|
|
1033
1042
|
}
|
|
1034
1043
|
|
|
1044
|
+
if (message.type === "request-cancel") {
|
|
1045
|
+
dropPartial(message.requestId);
|
|
1046
|
+
requests.get(message.requestId)?.abort();
|
|
1047
|
+
return;
|
|
1048
|
+
}
|
|
1049
|
+
|
|
1035
1050
|
if (message.type === "ping") {
|
|
1036
1051
|
send(channel, { type: "pong", id: message.id });
|
|
1037
1052
|
return;
|
|
@@ -1089,6 +1104,8 @@ export function createDataChannelHandler({
|
|
|
1089
1104
|
clearTimeout(entry.timer);
|
|
1090
1105
|
}
|
|
1091
1106
|
partials.clear();
|
|
1107
|
+
for (const controller of requests.values()) controller.abort();
|
|
1108
|
+
requests.clear();
|
|
1092
1109
|
forgetChannel(channel);
|
|
1093
1110
|
log(`[dc] Session ${tag}: channel closed`);
|
|
1094
1111
|
// One closed data channel does not mean the peer connection is gone:
|
|
@@ -1113,6 +1130,22 @@ export function createDataChannelHandler({
|
|
|
1113
1130
|
* @returns {Promise<void>}
|
|
1114
1131
|
*/
|
|
1115
1132
|
async function handleRequest(channel, req, viaChunks = false) {
|
|
1133
|
+
const requests = activeRequests.get(channel);
|
|
1134
|
+
if (typeof req.requestId !== "string" || !req.requestId || !requests) return;
|
|
1135
|
+
if (requests.has(req.requestId)) {
|
|
1136
|
+
send(channel, { type: "response-error", requestId: req.requestId, error: "Duplicate active request." });
|
|
1137
|
+
return;
|
|
1138
|
+
}
|
|
1139
|
+
const controller = new AbortController();
|
|
1140
|
+
requests.set(req.requestId, controller);
|
|
1141
|
+
try {
|
|
1142
|
+
await forwardRequest(channel, req, viaChunks, controller.signal);
|
|
1143
|
+
} finally {
|
|
1144
|
+
if (requests.get(req.requestId) === controller) requests.delete(req.requestId);
|
|
1145
|
+
}
|
|
1146
|
+
}
|
|
1147
|
+
|
|
1148
|
+
async function forwardRequest(channel, req, viaChunks, signal) {
|
|
1116
1149
|
const { requestId, method, path, query, headers: forwardedHeaders, body } = req;
|
|
1117
1150
|
|
|
1118
1151
|
// Reject paths that are not absolute, contain traversal sequences, or
|
|
@@ -1144,9 +1177,12 @@ export function createDataChannelHandler({
|
|
|
1144
1177
|
method,
|
|
1145
1178
|
headers: requestHeaders,
|
|
1146
1179
|
body: body != null ? body : undefined,
|
|
1147
|
-
redirect: "manual"
|
|
1180
|
+
redirect: "manual",
|
|
1181
|
+
dispatcher: loopbackDispatcher,
|
|
1182
|
+
signal
|
|
1148
1183
|
});
|
|
1149
1184
|
} catch (fetchError) {
|
|
1185
|
+
if (signal.aborted) return;
|
|
1150
1186
|
log(`[dc] ${method} ${path}${queryInfo} → error: ${fetchError?.message ?? String(fetchError)}`);
|
|
1151
1187
|
send(channel, { type: "response-error", requestId, error: fetchError?.message ?? String(fetchError) });
|
|
1152
1188
|
return;
|
|
@@ -1169,8 +1205,8 @@ export function createDataChannelHandler({
|
|
|
1169
1205
|
return;
|
|
1170
1206
|
}
|
|
1171
1207
|
|
|
1208
|
+
const reader = response.body.getReader();
|
|
1172
1209
|
try {
|
|
1173
|
-
const reader = response.body.getReader();
|
|
1174
1210
|
// [net-debug] TEMPORARY: measure transfer size/time and channel buffering.
|
|
1175
1211
|
// fetchMs = time waiting for the route (incl. ffmpeg segment finalization).
|
|
1176
1212
|
// ttfbMs = time from body-read start to the first chunk with data (loopback).
|
|
@@ -1191,8 +1227,10 @@ export function createDataChannelHandler({
|
|
|
1191
1227
|
const body = bodySender((bytes) => sendChunk(channel, requestId, bytes, false), sendChunkBytes);
|
|
1192
1228
|
resetEventLoopDelay();
|
|
1193
1229
|
while (true) {
|
|
1230
|
+
if (signal.aborted) return;
|
|
1194
1231
|
const readStartedAt = performance.now();
|
|
1195
1232
|
const { done, value } = await reader.read();
|
|
1233
|
+
if (signal.aborted) return;
|
|
1196
1234
|
readMs += performance.now() - readStartedAt;
|
|
1197
1235
|
if (done) {
|
|
1198
1236
|
chunks += body.flush();
|
|
@@ -1233,11 +1271,16 @@ export function createDataChannelHandler({
|
|
|
1233
1271
|
// buffer is large — wait for it to drain. Prevents the SCTP send buffer
|
|
1234
1272
|
// from ballooning, which stalls throughput.
|
|
1235
1273
|
const drainStepAt = performance.now();
|
|
1236
|
-
await waitForBufferDrain(channel);
|
|
1274
|
+
await waitForBufferDrain(channel, signal);
|
|
1237
1275
|
drainMs += performance.now() - drainStepAt;
|
|
1238
1276
|
}
|
|
1239
|
-
} catch {
|
|
1240
|
-
|
|
1277
|
+
} catch (error) {
|
|
1278
|
+
if (!signal.aborted) {
|
|
1279
|
+
send(channel, { type: "response-error", requestId, error: error?.message ?? String(error) });
|
|
1280
|
+
}
|
|
1281
|
+
} finally {
|
|
1282
|
+
if (signal.aborted) await reader.cancel().catch(() => {});
|
|
1283
|
+
reader.releaseLock();
|
|
1241
1284
|
}
|
|
1242
1285
|
}
|
|
1243
1286
|
|
|
@@ -1296,10 +1339,10 @@ export function createDataChannelHandler({
|
|
|
1296
1339
|
* @param {DataChannel} channel
|
|
1297
1340
|
* @returns {Promise<void>}
|
|
1298
1341
|
*/
|
|
1299
|
-
function waitForBufferDrain(channel) {
|
|
1342
|
+
function waitForBufferDrain(channel, signal) {
|
|
1300
1343
|
return new Promise((resolve) => {
|
|
1301
1344
|
try {
|
|
1302
|
-
if (typeof channel.bufferedAmount !== "function" || channel.bufferedAmount() <= DC_BUFFER_HIGH_WATER) {
|
|
1345
|
+
if (signal.aborted || typeof channel.bufferedAmount !== "function" || channel.bufferedAmount() <= DC_BUFFER_HIGH_WATER) {
|
|
1303
1346
|
resolve();
|
|
1304
1347
|
return;
|
|
1305
1348
|
}
|
|
@@ -1309,14 +1352,13 @@ export function createDataChannelHandler({
|
|
|
1309
1352
|
if (settled) return;
|
|
1310
1353
|
settled = true;
|
|
1311
1354
|
if (poll) clearInterval(poll);
|
|
1312
|
-
|
|
1355
|
+
signal.removeEventListener("abort", done);
|
|
1313
1356
|
resolve();
|
|
1314
1357
|
};
|
|
1315
|
-
|
|
1316
|
-
channel.
|
|
1317
|
-
//
|
|
1318
|
-
//
|
|
1319
|
-
// defect. Polling waits until the condition is actually met.
|
|
1358
|
+
signal.addEventListener("abort", done, { once: true });
|
|
1359
|
+
// The native channel exposes one callback for all requests. Sampling
|
|
1360
|
+
// avoids one request replacing another's callback. Cancellation ends
|
|
1361
|
+
// the wait immediately; elapsed time alone never resumes sending.
|
|
1320
1362
|
poll = setInterval(() => {
|
|
1321
1363
|
try {
|
|
1322
1364
|
if (channel.bufferedAmount() <= DC_BUFFER_LOW_WATER) {
|
|
@@ -1332,6 +1374,7 @@ export function createDataChannelHandler({
|
|
|
1332
1374
|
if (channel.bufferedAmount() <= DC_BUFFER_LOW_WATER) {
|
|
1333
1375
|
done();
|
|
1334
1376
|
}
|
|
1377
|
+
if (signal.aborted) done();
|
|
1335
1378
|
} catch {
|
|
1336
1379
|
resolve();
|
|
1337
1380
|
}
|