@torrent-tv/proxy 2.91.0 → 2.92.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/cli.js +3 -2
- package/package.json +18 -2
- package/routes/api/playback-plan/post.js +21 -4
- package/routes/api/sources/files/get.js +20 -28
- package/routes/api/sources/post.js +4 -2
- package/routes/api/sources/stats/get.js +4 -11
- package/routes/api/sources/viewer/post.js +16 -0
- package/routes/api/sources/warm/post.js +6 -215
- package/routes/api/subtitles/get.js +22 -125
- package/routes/api/subtitles/post.js +26 -0
- package/routes/api/transcode-sessions/post.js +24 -2
- package/routes/media/get.js +80 -0
- package/routes/stream/get.js +26 -94
- package/routes/transcode/audio-warm/get.js +9 -21
- package/routes/transcode/variant-warm/get.js +8 -14
- package/server.js +410 -46
- package/services/encode/AdmittedInput.js +136 -0
- package/services/encode/CushionReport.js +1 -76
- package/services/encode/EncodeInputs.js +142 -0
- package/services/encode/EncodeOrchestrator.js +4 -1
- package/services/encode/EncodeRun.js +13 -4
- package/services/encode/EncodeRuns.js +132 -52
- package/services/encode/InputFailures.js +37 -0
- package/services/encode/MatroskaInput.js +158 -0
- package/services/encode/OutputOpening.js +16 -64
- package/services/encode/Renditions.js +1 -17
- package/services/encode/admitted-command.js +60 -0
- package/services/encode/annex-b-input.js +37 -0
- package/services/encode/bit-payload.js +29 -0
- package/services/encode/output/OutputCatalog.js +1 -1
- package/services/encode/output/Timeline.js +11 -0
- package/services/encode/pcm-normalization.js +30 -0
- package/services/encode/piece-completeness.js +54 -1
- package/services/encode/quality/HostLoad.js +19 -73
- package/services/encode/quality/QualityOffer.js +1 -1
- package/services/encode/run-command.js +1 -589
- package/services/encode/segment-formats/mp4-boxes.js +3 -1
- package/services/encode/segment-formats/presentation-segment.js +114 -0
- package/services/media/ContainerOrchestrator.js +118 -48
- package/services/media/KeyframeTables.js +16 -115
- package/services/media/MediaReadRequests.js +120 -0
- package/services/media/ProbeRequests.js +135 -0
- package/services/media/SegmentInputs.js +48 -0
- package/services/media/SourceFile.js +6 -0
- package/services/media/SubtitleOrchestrator.js +190 -4
- package/services/media/await-plan.js +47 -0
- package/services/media/container/AsfContainer.js +217 -0
- package/services/media/container/AviContainer.js +343 -34
- package/services/media/container/CadencedPictureOrder.js +45 -0
- package/services/media/container/Container.js +7 -3
- package/services/media/container/ContainerFactory.js +10 -3
- package/services/media/container/FfprobeContainer.js +222 -0
- package/services/media/container/MatroskaContainer.js +72 -46
- package/services/media/container/Mp4Container.js +472 -139
- package/services/media/container/MpegPsContainer.js +281 -0
- package/services/media/container/MpegTsContainer.js +277 -0
- package/services/media/container/PacketIndex.js +270 -0
- package/services/media/container/PacketRecords.js +237 -0
- package/services/media/container/RetainedBytes.js +39 -0
- package/services/media/container/RetainedReads.js +51 -0
- package/services/media/container/ac3-frame.js +46 -0
- package/services/media/container/asf-packets.js +153 -0
- package/services/media/container/avc-picture-order.js +50 -0
- package/services/media/container/avi-open-dml.js +47 -0
- package/services/media/container/dts-frame.js +51 -0
- package/services/media/container/ebml-stream.js +12 -1
- package/services/media/container/ffprobe-record.js +45 -0
- package/services/media/container/flv-packet-ranges.js +27 -0
- package/services/media/container/following-packet-time.js +37 -0
- package/services/media/container/h264-configuration.js +183 -0
- package/services/media/container/h264-elementary-index.js +245 -0
- package/services/media/container/hevc-configuration.js +165 -0
- package/services/media/container/hevc-elementary-index.js +132 -0
- package/services/media/container/hevc-picture-order.js +66 -0
- package/services/media/container/latm-frame.js +99 -0
- package/services/media/container/matroska-packets.js +207 -0
- package/services/media/container/memory-unavailable.js +8 -0
- package/services/media/container/mp3-packet-dependencies.js +41 -0
- package/services/media/container/mp4-fragments.js +148 -0
- package/services/media/container/mpeg-audio-frame.js +19 -0
- package/services/media/container/mpeg-audio-reservoir.js +27 -0
- package/services/media/container/mpeg-elementary-index.js +196 -0
- package/services/media/container/mpeg-sequence-reader.js +69 -0
- package/services/media/container/mpeg-ts-packets.js +93 -0
- package/services/media/container/mpeg-video-configuration.js +50 -0
- package/services/media/container/mpeg4-picture-timing.js +108 -0
- package/services/media/container/pes-header.js +34 -0
- package/services/media/playback-declarations.js +57 -0
- package/services/media/playback-planner.js +72 -230
- package/services/media/probe-packets.js +66 -0
- package/services/media/read-playback-declarations.js +18 -0
- package/services/media/tracks/AudioTrack.js +9 -0
- package/services/media/tracks/TextSubtitleTrack.js +4 -1
- package/services/server/OutputLifecycle.js +2 -0
- package/services/server/SegmentServing.js +40 -33
- package/services/server/ViewerRequests.js +22 -41
- package/services/server/controllers/PlaybackController.js +4 -0
- package/services/server/controllers/SubtitleController.js +8 -29
- package/services/server/transcode-session-files.js +58 -99
- package/services/server/wire-outputs.js +58 -15
- package/services/storage/IndexMemory.js +85 -0
- package/services/storage/MachineBudget.js +27 -3
- package/services/storage/allowance.js +8 -1
- package/services/storage/piece-store/piece-disk-store.js +18 -3
- package/services/storage/piece-store/piece-lru.js +8 -4
- package/services/storage/piece-store/shared-piece-store.js +26 -11
- package/services/storage/segment-store/SegmentStore.js +49 -18
- package/services/storage/wire.js +7 -0
- package/services/torrent/Contents.js +52 -20
- package/services/torrent/demand/DemandRegister.js +13 -4
- package/services/torrent/demand/Window.js +13 -2
- package/services/torrent/demand-progress.js +25 -0
- package/services/torrent/download/DeadlineScheduler.js +167 -0
- package/services/torrent/download/FutureDownload.js +166 -0
- package/services/torrent/download/SwarmSelection.js +30 -59
- package/services/torrent/download/registry.js +45 -5
- package/services/torrent/download/withdraw-claim.js +2 -0
- package/services/torrent/torrent-pool.js +39 -600
- package/services/torrent/worker/channel.js +8 -1
- package/services/torrent/worker/client.js +74 -72
- package/services/torrent/worker/files-in-use.js +9 -0
- package/services/torrent/worker/handle-state.js +12 -0
- package/services/torrent/worker/held-bytes.js +82 -99
- package/services/torrent/worker/held-file-bytes.js +17 -0
- package/services/torrent/worker/piece-reader.js +67 -683
- package/services/torrent/worker/pool-adapter.js +37 -123
- package/services/torrent/worker/protocol.js +6 -17
- package/services/torrent/worker/whole-file-stats.js +2 -0
- package/services/torrent/worker/worker.js +86 -156
- package/services/transport/data-channel-handler.js +56 -13
- package/services/viewer/DownloadMaps.js +168 -0
- package/services/viewer/NativeSourceMap.js +33 -0
- package/services/viewer/PriorityMap.js +49 -40
- package/services/viewer/PriorityOrchestrator.js +42 -13
- package/services/viewer/SourcePreparation.js +278 -0
- package/services/viewer/Viewer.js +11 -0
- package/services/viewer/Viewers.js +109 -2
- package/services/viewer/playback-readiness.js +50 -63
- package/services/viewer/segment-demands.js +37 -0
- package/services/viewer/urgent-output-ready.js +26 -0
- package/services/encode/run-inputs.js +0 -53
- package/services/media/keyframe-probe.js +0 -157
- package/services/media/media-info-probe.js +0 -93
- package/services/torrent/worker/background-fill.js +0 -213
- package/services/torrent/worker/resume-warm.js +0 -107
|
@@ -12,6 +12,13 @@
|
|
|
12
12
|
|
|
13
13
|
import { Event, STREAM_HIGH_WATER_CHUNKS } from "./protocol.js";
|
|
14
14
|
|
|
15
|
+
/** Preserve terminal source state when a failure crosses the thread boundary. */
|
|
16
|
+
export function workerReplyError(message, fallback = "Torrent worker request failed.") {
|
|
17
|
+
const error = new Error(message.error ?? fallback);
|
|
18
|
+
if (typeof message.code === "string") error.code = message.code;
|
|
19
|
+
return error;
|
|
20
|
+
}
|
|
21
|
+
|
|
15
22
|
/**
|
|
16
23
|
* Issue request ids that stay unique for the life of a thread.
|
|
17
24
|
*
|
|
@@ -77,7 +84,7 @@ export function createCaller(port) {
|
|
|
77
84
|
}
|
|
78
85
|
if (message.type === Event.ERROR) {
|
|
79
86
|
pending.delete(message.id);
|
|
80
|
-
entry.reject(
|
|
87
|
+
entry.reject(workerReplyError(message));
|
|
81
88
|
return true;
|
|
82
89
|
}
|
|
83
90
|
return false;
|
|
@@ -19,7 +19,7 @@ import { Readable } from "node:stream";
|
|
|
19
19
|
import { open } from "node:fs/promises";
|
|
20
20
|
import { fileURLToPath } from "node:url";
|
|
21
21
|
import { logger, writeAlreadyDecided } from "../../../utils/logger.js";
|
|
22
|
-
import { createCaller, createReceiveStream } from "./channel.js";
|
|
22
|
+
import { createCaller, createReceiveStream, workerReplyError } from "./channel.js";
|
|
23
23
|
import { Command, Event } from "./protocol.js";
|
|
24
24
|
|
|
25
25
|
const WORKER_URL = new URL("./worker.js", import.meta.url);
|
|
@@ -110,11 +110,12 @@ export class TorrentWorkerClient {
|
|
|
110
110
|
/** @type {(event: { sourceKey: string, fileIndex: number, trackIndex: number, cues: object[], language: string }) => void} */
|
|
111
111
|
/** @type {(event: { sourceKey: string, fileIndexes: number[] }) => void} */
|
|
112
112
|
#onPiecesArrived;
|
|
113
|
+
#onSourceForgotten;
|
|
113
114
|
|
|
114
115
|
/**
|
|
115
116
|
* @param {{ memoryBytes?: number, stateDir?: string, onPiecesArrived?: (event: object) => void }} [options]
|
|
116
117
|
*/
|
|
117
|
-
constructor({ memoryBytes, stateDir, onPiecesArrived } = {}) {
|
|
118
|
+
constructor({ memoryBytes, stateDir, onPiecesArrived, onSourceForgotten } = {}) {
|
|
118
119
|
this.#worker = new Worker(fileURLToPath(WORKER_URL), {
|
|
119
120
|
// `stateDir` travels because the worker writes heap snapshots of its own
|
|
120
121
|
// isolate there. It cannot choose a directory any other way: a worker may
|
|
@@ -124,6 +125,7 @@ export class TorrentWorkerClient {
|
|
|
124
125
|
});
|
|
125
126
|
this.#caller = createCaller(this.#worker);
|
|
126
127
|
this.#onPiecesArrived = onPiecesArrived ?? (() => undefined);
|
|
128
|
+
this.#onSourceForgotten = onSourceForgotten ?? (() => undefined);
|
|
127
129
|
|
|
128
130
|
this.#worker.on("message", (message) => {
|
|
129
131
|
// A failed read must fail its stream. This is checked BEFORE the caller
|
|
@@ -135,13 +137,13 @@ export class TorrentWorkerClient {
|
|
|
135
137
|
if (message?.type === Event.ERROR && this.#reads.has(message.id)) {
|
|
136
138
|
const read = this.#reads.get(message.id);
|
|
137
139
|
this.#reads.delete(message.id);
|
|
138
|
-
read.fail(
|
|
140
|
+
read.fail(workerReplyError(message, "Torrent worker read failed."));
|
|
139
141
|
return;
|
|
140
142
|
}
|
|
141
143
|
if (message?.type === Event.ERROR && this.#fragmentReaders.has(message.id)) {
|
|
142
144
|
const reader = this.#fragmentReaders.get(message.id);
|
|
143
145
|
this.#fragmentReaders.delete(message.id);
|
|
144
|
-
reader.fail(
|
|
146
|
+
reader.fail(workerReplyError(message, "Torrent worker read failed."));
|
|
145
147
|
return;
|
|
146
148
|
}
|
|
147
149
|
if (this.#caller.handleReply(message)) {
|
|
@@ -233,13 +235,20 @@ export class TorrentWorkerClient {
|
|
|
233
235
|
length: message.length,
|
|
234
236
|
name: message.name
|
|
235
237
|
});
|
|
238
|
+
this.#onPiecesArrived({ sourceKey: message.sourceKey ?? `torrent:${message.infoHash}`,
|
|
239
|
+
fileIndexes: [message.fileIndex], reason: "file-complete" });
|
|
240
|
+
break;
|
|
241
|
+
case Event.SOURCE_FORGOTTEN:
|
|
242
|
+
this.#onSourceForgotten({ sourceKey: message.sourceKey });
|
|
236
243
|
break;
|
|
237
244
|
case Event.PIECES_ARRIVED:
|
|
245
|
+
case Event.PIECES_CHANGED:
|
|
238
246
|
// ANNOUNCED by the thread that owns the swarm, acted on here: the
|
|
239
247
|
// subtitle walk is this thread's and needs to know a piece arrived.
|
|
240
248
|
this.#onPiecesArrived({
|
|
241
249
|
sourceKey: message.sourceKey,
|
|
242
|
-
|
|
250
|
+
fileIndexes: Array.isArray(message.fileIndexes) ? message.fileIndexes : [],
|
|
251
|
+
...(typeof message.reason === "string" ? { reason: message.reason } : {})
|
|
243
252
|
});
|
|
244
253
|
break;
|
|
245
254
|
default:
|
|
@@ -323,11 +332,11 @@ export class TorrentWorkerClient {
|
|
|
323
332
|
/**
|
|
324
333
|
* Live download figures for the progress display.
|
|
325
334
|
*
|
|
326
|
-
* @param {{ sourceKey: string, fileIndex: number
|
|
335
|
+
* @param {{ sourceKey: string, fileIndex: number }} params
|
|
327
336
|
* @returns {Promise<object>}
|
|
328
337
|
*/
|
|
329
|
-
async getFileStats({ sourceKey, fileIndex,
|
|
330
|
-
return this.#caller.call(Command.FILE_STATS, { sourceKey, fileIndex,
|
|
338
|
+
async getFileStats({ sourceKey, fileIndex, forecast = false }) {
|
|
339
|
+
return this.#caller.call(Command.FILE_STATS, { sourceKey, fileIndex, forecast });
|
|
331
340
|
}
|
|
332
341
|
|
|
333
342
|
/**
|
|
@@ -340,22 +349,6 @@ export class TorrentWorkerClient {
|
|
|
340
349
|
|
|
341
350
|
|
|
342
351
|
|
|
343
|
-
/**
|
|
344
|
-
* Start fetching the region a viewer is about to resume at.
|
|
345
|
-
*
|
|
346
|
-
* @param {{ sourceKey: string, fileIndex: number, positionSeconds: number }} params
|
|
347
|
-
* @returns {Promise<{ started: boolean }>}
|
|
348
|
-
*/
|
|
349
|
-
/**
|
|
350
|
-
* Fetch one whole file in the room the viewer's own reading leaves.
|
|
351
|
-
*
|
|
352
|
-
* @param {{ sourceKey: string, fileIndex: number }} params
|
|
353
|
-
* @returns {Promise<{ started: boolean }>}
|
|
354
|
-
*/
|
|
355
|
-
async fillFile({ sourceKey, fileIndex }) {
|
|
356
|
-
return this.#caller.call(Command.FILL_FILE, { sourceKey, fileIndex });
|
|
357
|
-
}
|
|
358
|
-
|
|
359
352
|
/**
|
|
360
353
|
* The byte ranges of one file the torrent holds WHOLE.
|
|
361
354
|
*
|
|
@@ -401,6 +394,22 @@ export class TorrentWorkerClient {
|
|
|
401
394
|
return bytes ? Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength) : null;
|
|
402
395
|
}
|
|
403
396
|
|
|
397
|
+
async readHeldRanges({ sourceKey, fileIndex, ranges, maxBytes }) {
|
|
398
|
+
const whole = this.#wholeFileOf(sourceKey, fileIndex);
|
|
399
|
+
if (whole) {
|
|
400
|
+
try {
|
|
401
|
+
return await readWholeFileRanges(whole, ranges, maxBytes);
|
|
402
|
+
} catch (error) {
|
|
403
|
+
const key = `${String(sourceKey).slice("torrent:".length).toLowerCase()}/${fileIndex}`;
|
|
404
|
+
if (this.wholeFiles.get(key) === whole) this.wholeFiles.delete(key);
|
|
405
|
+
logger.info(`whole-file input ${sourceKey}:${fileIndex} is unavailable: ${error?.code ?? error?.message ?? error}`);
|
|
406
|
+
return null;
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
const answer = await this.#caller.call(Command.READ_HELD_RANGES, { sourceKey, fileIndex, ranges, maxBytes });
|
|
410
|
+
return answer?.buffers ? answer.buffers.map(bytes => Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength)) : null;
|
|
411
|
+
}
|
|
412
|
+
|
|
404
413
|
/**
|
|
405
414
|
* The file kept whole on disk for this source and index, if there is one.
|
|
406
415
|
*
|
|
@@ -423,11 +432,6 @@ export class TorrentWorkerClient {
|
|
|
423
432
|
return this.wholeFiles.get(`${infoHash}/${fileIndex}`) ?? null;
|
|
424
433
|
}
|
|
425
434
|
|
|
426
|
-
async warmResumePosition({ sourceKey, fileIndex, positionSeconds, durationSeconds }) {
|
|
427
|
-
return this.#caller.call(Command.WARM_POSITION, { sourceKey, fileIndex, positionSeconds, durationSeconds });
|
|
428
|
-
}
|
|
429
|
-
|
|
430
|
-
|
|
431
435
|
/**
|
|
432
436
|
* Bytes every torrent on the worker has moved, downloaded and uploaded apart.
|
|
433
437
|
*
|
|
@@ -454,27 +458,6 @@ export class TorrentWorkerClient {
|
|
|
454
458
|
return this.#caller.call(Command.HELD_TORRENTS, {});
|
|
455
459
|
}
|
|
456
460
|
|
|
457
|
-
/** State conditional demand for every file in a selected torrent. */
|
|
458
|
-
async fillTorrent(sourceKey) {
|
|
459
|
-
return this.#caller.call(Command.FILL_TORRENT, { sourceKey });
|
|
460
|
-
}
|
|
461
|
-
|
|
462
|
-
/**
|
|
463
|
-
* Reorder piece selection around a read position (seek prioritisation).
|
|
464
|
-
*
|
|
465
|
-
* @param {{ sourceKey: string, fileIndex: number, byteStart: number, windowBytes?: number, wholeFileRead?: boolean }} params
|
|
466
|
-
* @returns {Promise<void>}
|
|
467
|
-
*/
|
|
468
|
-
async prioritizeByteRange({ sourceKey, fileIndex, byteStart, windowBytes, wholeFileRead }) {
|
|
469
|
-
await this.#caller.call(Command.PRIORITIZE, {
|
|
470
|
-
sourceKey,
|
|
471
|
-
fileIndex,
|
|
472
|
-
byteStart,
|
|
473
|
-
windowBytes,
|
|
474
|
-
wholeFileRead
|
|
475
|
-
});
|
|
476
|
-
}
|
|
477
|
-
|
|
478
461
|
/**
|
|
479
462
|
* Hand the download the priority map for one file.
|
|
480
463
|
*
|
|
@@ -495,16 +478,6 @@ export class TorrentWorkerClient {
|
|
|
495
478
|
});
|
|
496
479
|
}
|
|
497
480
|
|
|
498
|
-
/**
|
|
499
|
-
* Pre-fetch the head and tail the codec probe needs.
|
|
500
|
-
*
|
|
501
|
-
* @param {{ sourceKey: string, fileIndex: number, options?: { headBytes?: number, tailBytes?: number, timeoutMs?: number, awaited?: boolean } }} params
|
|
502
|
-
* @returns {Promise<unknown>}
|
|
503
|
-
*/
|
|
504
|
-
async prefetchFileEdges({ sourceKey, fileIndex, options = {} }) {
|
|
505
|
-
return this.#caller.call(Command.PREFETCH_EDGES, { sourceKey, fileIndex, options });
|
|
506
|
-
}
|
|
507
|
-
|
|
508
481
|
/**
|
|
509
482
|
* Read a byte range as a stream.
|
|
510
483
|
*
|
|
@@ -512,10 +485,10 @@ export class TorrentWorkerClient {
|
|
|
512
485
|
* (viewer gone, seek superseded) stops the worker reading, so pieces are not
|
|
513
486
|
* fetched for a stream nobody will drain.
|
|
514
487
|
*
|
|
515
|
-
* @param {{ sourceKey: string, fileIndex: number, start?: number | null, end?: number | null
|
|
488
|
+
* @param {{ sourceKey: string, fileIndex: number, start?: number | null, end?: number | null }} params
|
|
516
489
|
* @returns {ReadableStream<Uint8Array>}
|
|
517
490
|
*/
|
|
518
|
-
createReadStream({ sourceKey, fileIndex, start = null, end = null
|
|
491
|
+
createReadStream({ sourceKey, fileIndex, start = null, end = null }) {
|
|
519
492
|
// Same id sequence as commands — see `nextId` in `channel.js`.
|
|
520
493
|
const readId = this.#caller.nextId();
|
|
521
494
|
const receive = createReceiveStream({
|
|
@@ -534,7 +507,7 @@ export class TorrentWorkerClient {
|
|
|
534
507
|
this.#worker.postMessage({
|
|
535
508
|
command: Command.READ_RANGE,
|
|
536
509
|
id: readId,
|
|
537
|
-
params: { sourceKey, fileIndex, start, end
|
|
510
|
+
params: { sourceKey, fileIndex, start, end }
|
|
538
511
|
});
|
|
539
512
|
|
|
540
513
|
return receive.stream;
|
|
@@ -554,10 +527,10 @@ export class TorrentWorkerClient {
|
|
|
554
527
|
* Returns `null` when this source has no shared pool, so the caller can fall
|
|
555
528
|
* back to {@link createReadStream}.
|
|
556
529
|
*
|
|
557
|
-
* @param {{ sourceKey: string, fileIndex: number, start?: number | null, end?: number | null
|
|
530
|
+
* @param {{ sourceKey: string, fileIndex: number, start?: number | null, end?: number | null }} params
|
|
558
531
|
* @returns {{ [Symbol.asyncIterator]: () => AsyncGenerator<{ bytes: Uint8Array, release: () => void }>, cancel: () => void } | null}
|
|
559
532
|
*/
|
|
560
|
-
createFragmentReader({ sourceKey, fileIndex, start = null, end = null
|
|
533
|
+
createFragmentReader({ sourceKey, fileIndex, start = null, end = null }) {
|
|
561
534
|
const readId = this.#caller.nextId();
|
|
562
535
|
/** @type {{ bytes: Uint8Array, release: () => void }[]} */
|
|
563
536
|
const queue = [];
|
|
@@ -598,7 +571,7 @@ export class TorrentWorkerClient {
|
|
|
598
571
|
this.#worker.postMessage({
|
|
599
572
|
command: Command.READ_RANGE,
|
|
600
573
|
id: readId,
|
|
601
|
-
params: { sourceKey, fileIndex, start, end
|
|
574
|
+
params: { sourceKey, fileIndex, start, end }
|
|
602
575
|
});
|
|
603
576
|
|
|
604
577
|
return {
|
|
@@ -661,7 +634,7 @@ export class TorrentWorkerClient {
|
|
|
661
634
|
files: info.files.map((file) => ({
|
|
662
635
|
...file,
|
|
663
636
|
/**
|
|
664
|
-
* @param {{ start?: number, end?: number
|
|
637
|
+
* @param {{ start?: number, end?: number }} [options]
|
|
665
638
|
* @returns {ReadableStream<Uint8Array>}
|
|
666
639
|
*/
|
|
667
640
|
createReadStream(options = {}) {
|
|
@@ -674,8 +647,7 @@ export class TorrentWorkerClient {
|
|
|
674
647
|
sourceKey,
|
|
675
648
|
fileIndex: file.index,
|
|
676
649
|
start: options.start ?? null,
|
|
677
|
-
end: options.end ?? null
|
|
678
|
-
windowBytes: options.windowBytes
|
|
650
|
+
end: options.end ?? null
|
|
679
651
|
})
|
|
680
652
|
);
|
|
681
653
|
},
|
|
@@ -684,7 +656,7 @@ export class TorrentWorkerClient {
|
|
|
684
656
|
* Fragments of shared memory, for a caller that can say when it has
|
|
685
657
|
* finished with each one. `null` when this source has no shared pool.
|
|
686
658
|
*
|
|
687
|
-
* @param {{ start?: number, end?: number
|
|
659
|
+
* @param {{ start?: number, end?: number }} [options]
|
|
688
660
|
* @returns {ReturnType<TorrentWorkerClient["createFragmentReader"]>}
|
|
689
661
|
*/
|
|
690
662
|
createFragmentReader(options = {}) {
|
|
@@ -692,8 +664,7 @@ export class TorrentWorkerClient {
|
|
|
692
664
|
sourceKey,
|
|
693
665
|
fileIndex: file.index,
|
|
694
666
|
start: options.start ?? null,
|
|
695
|
-
end: options.end ?? null
|
|
696
|
-
windowBytes: options.windowBytes
|
|
667
|
+
end: options.end ?? null
|
|
697
668
|
});
|
|
698
669
|
}
|
|
699
670
|
}))
|
|
@@ -834,3 +805,34 @@ export async function readWholeFile(whole, start, end) {
|
|
|
834
805
|
await handle.close();
|
|
835
806
|
}
|
|
836
807
|
}
|
|
808
|
+
|
|
809
|
+
/** One open descriptor holds the complete file throughout segment acquisition. */
|
|
810
|
+
export async function readWholeFileRanges(whole, ranges, maxBytes) {
|
|
811
|
+
if (!Array.isArray(ranges) || ranges.length === 0 || !Number.isSafeInteger(maxBytes) || maxBytes <= 0) return null;
|
|
812
|
+
let wanted = 0;
|
|
813
|
+
for (const range of ranges) {
|
|
814
|
+
if (!Array.isArray(range) || range.length !== 2) return null;
|
|
815
|
+
const [start, end] = range;
|
|
816
|
+
if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start < 0 || end < start || end >= whole.length) return null;
|
|
817
|
+
wanted += end - start + 1;
|
|
818
|
+
if (!Number.isSafeInteger(wanted) || wanted > maxBytes) return null;
|
|
819
|
+
}
|
|
820
|
+
const handle = await open(whole.path, "r");
|
|
821
|
+
try {
|
|
822
|
+
if ((await handle.stat()).size !== whole.length) return null;
|
|
823
|
+
const result = [];
|
|
824
|
+
for (const [start, end] of ranges) {
|
|
825
|
+
const bytes = Buffer.allocUnsafeSlow(end - start + 1);
|
|
826
|
+
let offset = 0;
|
|
827
|
+
while (offset < bytes.length) {
|
|
828
|
+
const { bytesRead } = await handle.read(bytes, offset, bytes.length - offset, start + offset);
|
|
829
|
+
if (bytesRead === 0) return null;
|
|
830
|
+
offset += bytesRead;
|
|
831
|
+
}
|
|
832
|
+
result.push(bytes);
|
|
833
|
+
}
|
|
834
|
+
return result;
|
|
835
|
+
} finally {
|
|
836
|
+
await handle.close();
|
|
837
|
+
}
|
|
838
|
+
}
|
|
@@ -33,3 +33,12 @@ export function filesInUse({ torrent, windows, openReads }) {
|
|
|
33
33
|
}
|
|
34
34
|
return used;
|
|
35
35
|
}
|
|
36
|
+
|
|
37
|
+
/** Range admission owns its read before any asynchronous storage operation. */
|
|
38
|
+
export async function readWhileInUse(openReads, id, torrent, fileIndex, read) {
|
|
39
|
+
if (!torrent) return read();
|
|
40
|
+
const entry = { torrent, fileIndex };
|
|
41
|
+
openReads.set(id, entry);
|
|
42
|
+
try { return await read(); }
|
|
43
|
+
finally { if (openReads.get(id) === entry) openReads.delete(id); }
|
|
44
|
+
}
|
|
@@ -21,3 +21,15 @@
|
|
|
21
21
|
export function isUsableTorrentHandle(torrent) {
|
|
22
22
|
return Boolean(torrent) && torrent.destroyed !== true && (torrent.files?.length ?? 0) > 0;
|
|
23
23
|
}
|
|
24
|
+
|
|
25
|
+
/** Resolve only the source lifetime that the caller originally observed. */
|
|
26
|
+
export async function requireCurrentTorrent(handles, sourceKey, pending = handles.get(sourceKey)) {
|
|
27
|
+
const torrent = pending ? await pending : null;
|
|
28
|
+
if (!pending || handles.get(sourceKey) !== pending || !isUsableTorrentHandle(torrent)) {
|
|
29
|
+
if (pending && handles.get(sourceKey) === pending) handles.delete(sourceKey);
|
|
30
|
+
const error = new Error(`Source ${sourceKey} is no longer available.`);
|
|
31
|
+
error.code = "SOURCE_FORGOTTEN";
|
|
32
|
+
throw error;
|
|
33
|
+
}
|
|
34
|
+
return torrent;
|
|
35
|
+
}
|
|
@@ -1,9 +1,12 @@
|
|
|
1
|
+
import { pieceStoreOf } from "../piece-store-of.js";
|
|
2
|
+
|
|
1
3
|
/**
|
|
2
4
|
* @file What of a file the torrent already HOLDS, and reading it without
|
|
3
5
|
* asking the swarm for anything.
|
|
4
6
|
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
+
* Storage answers both questions: which pieces remain available, and how to
|
|
8
|
+
* read them. The library bitfield is not a second availability authority.
|
|
9
|
+
* They are here because
|
|
7
10
|
* of that and for no other reason — everything else that used to keep them
|
|
8
11
|
* company, the subtitle plan and the cue walk and the cursor a browser follows,
|
|
9
12
|
* is what a FILE says about itself and moved to the thread the sessions are on
|
|
@@ -27,8 +30,8 @@
|
|
|
27
30
|
/**
|
|
28
31
|
* The byte ranges of one file that are downloaded WHOLE.
|
|
29
32
|
*
|
|
30
|
-
* Whole because a piece is the unit
|
|
31
|
-
*
|
|
33
|
+
* Whole because a piece is the unit storage retains: a range that is half a
|
|
34
|
+
* piece short cannot be read, so
|
|
32
35
|
* reporting it would be reporting bytes that are not there.
|
|
33
36
|
*
|
|
34
37
|
* @param {object} torrent
|
|
@@ -40,7 +43,8 @@
|
|
|
40
43
|
export function heldRangesOf(torrent, fileIndex) {
|
|
41
44
|
const file = torrent?.files?.[fileIndex];
|
|
42
45
|
const pieceLength = Number(torrent?.pieceLength);
|
|
43
|
-
|
|
46
|
+
const store = pieceStoreOf(torrent);
|
|
47
|
+
if (!file || !Number.isSafeInteger(pieceLength) || pieceLength <= 0 || typeof store?.locationOf !== "function") {
|
|
44
48
|
return [];
|
|
45
49
|
}
|
|
46
50
|
const offset = Number(file.offset) || 0;
|
|
@@ -54,7 +58,7 @@ export function heldRangesOf(torrent, fileIndex) {
|
|
|
54
58
|
const ranges = [];
|
|
55
59
|
let runStart = -1;
|
|
56
60
|
for (let index = first; index <= last; index += 1) {
|
|
57
|
-
if (
|
|
61
|
+
if (store.locationOf(index) !== "missing") {
|
|
58
62
|
if (runStart < 0) {
|
|
59
63
|
runStart = index;
|
|
60
64
|
}
|
|
@@ -99,14 +103,16 @@ function rangeWithin(firstPiece, lastPiece, pieceLength, offset, length) {
|
|
|
99
103
|
export function isRangeHeld(torrent, fileIndex, start, end) {
|
|
100
104
|
const file = torrent?.files?.[fileIndex];
|
|
101
105
|
const pieceLength = Number(torrent?.pieceLength);
|
|
102
|
-
|
|
106
|
+
const store = pieceStoreOf(torrent);
|
|
107
|
+
if (!file || !Number.isSafeInteger(pieceLength) || pieceLength <= 0 || typeof store?.locationOf !== "function" ||
|
|
108
|
+
!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start < 0 || end < start || end >= file.length) {
|
|
103
109
|
return false;
|
|
104
110
|
}
|
|
105
111
|
const offset = Number(file.offset) || 0;
|
|
106
112
|
const first = Math.floor((offset + start) / pieceLength);
|
|
107
113
|
const last = Math.floor((offset + end) / pieceLength);
|
|
108
114
|
for (let index = first; index <= last; index += 1) {
|
|
109
|
-
if (
|
|
115
|
+
if (store.locationOf(index) === "missing") {
|
|
110
116
|
return false;
|
|
111
117
|
}
|
|
112
118
|
}
|
|
@@ -114,105 +120,82 @@ export function isRangeHeld(torrent, fileIndex, start, end) {
|
|
|
114
120
|
}
|
|
115
121
|
|
|
116
122
|
/**
|
|
117
|
-
*
|
|
118
|
-
*
|
|
119
|
-
*
|
|
120
|
-
*
|
|
121
|
-
* held before the read starts, so a read that does not finish is a store that
|
|
122
|
-
* failed to serve what it has, and the line it writes says so. Until 2026-10-01
|
|
123
|
-
* this read was started on bytes that had NOT arrived, the stream waited for
|
|
124
|
-
* them, and this was the bound that gave it up — 36 such reads gave up at once
|
|
125
|
-
* at a torrent's open, and one of them was the Cues table of the file being
|
|
126
|
-
* watched.
|
|
127
|
-
*/
|
|
128
|
-
const READ_ABANDON_MS = 30_000;
|
|
129
|
-
|
|
130
|
-
/**
|
|
131
|
-
* Read a byte range of a file straight from the store, without asking the swarm
|
|
132
|
-
* for anything.
|
|
133
|
-
*
|
|
134
|
-
* Answered at once with null where any piece under the range has not arrived:
|
|
135
|
-
* "not here" is the whole answer, and nothing is waited for or requested.
|
|
136
|
-
*
|
|
137
|
-
* The bytes come back in a buffer that owns its whole memory — not a slice of
|
|
138
|
-
* Node's shared pool and not a view of the store's own blocks — so the reply can
|
|
139
|
-
* hand that memory to the other thread instead of copying it, and nothing the
|
|
140
|
-
* store or another read still uses goes with it.
|
|
141
|
-
*
|
|
142
|
-
* @param {object} torrent
|
|
143
|
-
* @param {number} fileIndex
|
|
144
|
-
* @param {number} start
|
|
145
|
-
* @param {number} end - Inclusive.
|
|
146
|
-
* @param {{ info: Function }} [logger]
|
|
147
|
-
* @returns {Promise<Buffer | null>}
|
|
123
|
+
* Read downloaded bytes through storage only. Availability and acquisition of
|
|
124
|
+
* all input holds are one synchronous operation before the first disk read.
|
|
125
|
+
* Missing input returns null without selecting pieces or opening a torrent
|
|
126
|
+
* file stream. The returned allocation can be transferred to another thread.
|
|
148
127
|
*/
|
|
149
|
-
export function readHeldBytes(torrent, fileIndex, start, end, logger = null) {
|
|
128
|
+
export async function readHeldBytes(torrent, fileIndex, start, end, logger = null) {
|
|
150
129
|
const file = torrent?.files?.[fileIndex];
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
130
|
+
const pieceLength = Number(torrent?.pieceLength);
|
|
131
|
+
const store = pieceStoreOf(torrent);
|
|
132
|
+
if (!file || !store || !Number.isSafeInteger(start) || !Number.isSafeInteger(end) ||
|
|
133
|
+
start < 0 || end < start || start >= Number(file.length) || !(pieceLength > 0)) return null;
|
|
154
134
|
const last = Math.min(end, Number(file.length) - 1);
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
135
|
+
const absoluteStart = (Number(file.offset) || 0) + start;
|
|
136
|
+
const absoluteEnd = (Number(file.offset) || 0) + last;
|
|
137
|
+
const firstPiece = Math.floor(absoluteStart / pieceLength);
|
|
138
|
+
const lastPiece = Math.floor(absoluteEnd / pieceLength);
|
|
139
|
+
const indexes = Array.from({ length: lastPiece - firstPiece + 1 }, (_, index) => firstPiece + index);
|
|
140
|
+
const release = store.holdAvailable(indexes);
|
|
141
|
+
if (!release) return null;
|
|
142
|
+
try {
|
|
143
|
+
const owned = Buffer.allocUnsafeSlow(last - start + 1);
|
|
144
|
+
for (const index of indexes) {
|
|
145
|
+
const from = Math.max(absoluteStart, index * pieceLength);
|
|
146
|
+
const to = Math.min(absoluteEnd, (index + 1) * pieceLength - 1);
|
|
147
|
+
const wanted = to - from + 1;
|
|
148
|
+
const bytes = await new Promise((resolve, reject) => {
|
|
149
|
+
store.get(index, { offset: from - index * pieceLength, length: wanted }, (error, value) => {
|
|
150
|
+
if (error) reject(error);
|
|
151
|
+
else resolve(value);
|
|
152
|
+
});
|
|
153
|
+
});
|
|
154
|
+
if (!bytes || bytes.length !== wanted) return null;
|
|
155
|
+
owned.set(bytes, from - absoluteStart);
|
|
166
156
|
}
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
settled = true;
|
|
175
|
-
if (abandon !== null) {
|
|
176
|
-
clearTimeout(abandon);
|
|
177
|
-
}
|
|
178
|
-
if (value === null) {
|
|
179
|
-
stream.destroy?.();
|
|
180
|
-
}
|
|
181
|
-
resolve(value);
|
|
182
|
-
};
|
|
183
|
-
abandon = setTimeout(() => {
|
|
184
|
-
logger?.info(
|
|
185
|
-
`subtitles: a read of ${start}-${last} in "${String(file.name).slice(0, 40)}" ` +
|
|
186
|
-
`was held but not served in ${READ_ABANDON_MS / 1000}s and was given up — the store failed to serve bytes it has`
|
|
187
|
-
);
|
|
188
|
-
settle(null);
|
|
189
|
-
}, READ_ABANDON_MS);
|
|
190
|
-
abandon.unref?.();
|
|
191
|
-
stream.on("data", (chunk) => chunks.push(chunk));
|
|
192
|
-
stream.on("end", () => settle(ownedCopyOf(chunks)));
|
|
193
|
-
stream.on("error", () => settle(null));
|
|
194
|
-
});
|
|
157
|
+
return owned;
|
|
158
|
+
} catch (error) {
|
|
159
|
+
logger?.info?.(`held read ${fileIndex}:${start}-${last} failed: ${error?.message ?? error}`);
|
|
160
|
+
return null;
|
|
161
|
+
} finally {
|
|
162
|
+
release();
|
|
163
|
+
}
|
|
195
164
|
}
|
|
196
165
|
|
|
197
|
-
/**
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
const
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
166
|
+
/** Acquire every input piece before copying any of the segment's ranges. */
|
|
167
|
+
export async function readHeldRanges(torrent, fileIndex, ranges, maxBytes, logger = null) {
|
|
168
|
+
const file = torrent?.files?.[fileIndex];
|
|
169
|
+
const pieceLength = Number(torrent?.pieceLength);
|
|
170
|
+
const store = pieceStoreOf(torrent);
|
|
171
|
+
if (!file || !store || !Array.isArray(ranges) || ranges.length === 0 ||
|
|
172
|
+
!Number.isSafeInteger(maxBytes) || maxBytes <= 0 || !(pieceLength > 0)) return null;
|
|
173
|
+
const indexes = new Set();
|
|
174
|
+
let bytes = 0;
|
|
175
|
+
for (const range of ranges) {
|
|
176
|
+
if (!Array.isArray(range) || range.length !== 2) return null;
|
|
177
|
+
const [start, end] = range;
|
|
178
|
+
if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start < 0 || end < start || end >= file.length) return null;
|
|
179
|
+
bytes += end - start + 1;
|
|
180
|
+
if (!Number.isSafeInteger(bytes) || bytes > maxBytes) return null;
|
|
181
|
+
const first = Math.floor(((Number(file.offset) || 0) + start) / pieceLength);
|
|
182
|
+
const last = Math.floor(((Number(file.offset) || 0) + end) / pieceLength);
|
|
183
|
+
for (let index = first; index <= last; index++) indexes.add(index);
|
|
184
|
+
}
|
|
185
|
+
const release = store.holdAvailable([...indexes]);
|
|
186
|
+
if (!release) return null;
|
|
187
|
+
try {
|
|
188
|
+
const result = [];
|
|
189
|
+
for (const [start, end] of ranges) {
|
|
190
|
+
const owned = await readHeldBytes(torrent, fileIndex, start, end, logger);
|
|
191
|
+
if (!owned) return null;
|
|
192
|
+
result.push(owned);
|
|
193
|
+
}
|
|
194
|
+
return result;
|
|
195
|
+
} finally {
|
|
196
|
+
release();
|
|
212
197
|
}
|
|
213
|
-
return owned;
|
|
214
198
|
}
|
|
215
|
-
|
|
216
199
|
/**
|
|
217
200
|
* Whether a buffer may be TRANSFERRED to another thread: it must be the whole of
|
|
218
201
|
* a plain ArrayBuffer. A slice of the shared pool would take every other
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { readHeldBytes } from "./held-bytes.js";
|
|
2
|
+
|
|
3
|
+
/** Whole-file assembly reads held storage pieces without creating download demand. */
|
|
4
|
+
export async function* heldFileBytes(torrent, fileIndex) {
|
|
5
|
+
const file = torrent?.files?.[fileIndex];
|
|
6
|
+
const pieceLength = Number(torrent?.pieceLength), offset = Number(file?.offset) || 0;
|
|
7
|
+
if (!file || !Number.isSafeInteger(file.length) || file.length <= 0 || !Number.isSafeInteger(pieceLength) || pieceLength <= 0) {
|
|
8
|
+
throw new Error("Whole-file assembly requires a declared file and piece length.");
|
|
9
|
+
}
|
|
10
|
+
for (let start = 0; start < file.length;) {
|
|
11
|
+
const end = Math.min(file.length - 1, (Math.floor((offset + start) / pieceLength) + 1) * pieceLength - offset - 1);
|
|
12
|
+
const bytes = await readHeldBytes(torrent, fileIndex, start, end);
|
|
13
|
+
if (!bytes) throw new Error(`Whole-file assembly lost held bytes ${start}-${end} of file ${fileIndex}.`);
|
|
14
|
+
yield bytes;
|
|
15
|
+
start = end + 1;
|
|
16
|
+
}
|
|
17
|
+
}
|