@torrent-tv/proxy 2.90.0 → 2.92.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/cli.js +5 -2
- package/package.json +18 -2
- package/routes/api/playback-plan/post.js +21 -4
- package/routes/api/sources/files/get.js +20 -28
- package/routes/api/sources/post.js +4 -2
- package/routes/api/sources/stats/get.js +4 -11
- package/routes/api/sources/viewer/post.js +16 -0
- package/routes/api/sources/warm/post.js +6 -215
- package/routes/api/subtitles/get.js +22 -125
- package/routes/api/subtitles/post.js +26 -0
- package/routes/api/transcode-sessions/post.js +24 -2
- package/routes/media/get.js +80 -0
- package/routes/stream/get.js +26 -94
- package/routes/transcode/audio-warm/get.js +9 -21
- package/routes/transcode/variant-warm/get.js +8 -14
- package/server.js +410 -46
- package/services/encode/AdmittedInput.js +136 -0
- package/services/encode/CushionReport.js +1 -76
- package/services/encode/EncodeInputs.js +142 -0
- package/services/encode/EncodeOrchestrator.js +4 -1
- package/services/encode/EncodeRun.js +13 -4
- package/services/encode/EncodeRuns.js +132 -52
- package/services/encode/InputFailures.js +37 -0
- package/services/encode/MatroskaInput.js +158 -0
- package/services/encode/OutputOpening.js +16 -64
- package/services/encode/Renditions.js +1 -17
- package/services/encode/admitted-command.js +60 -0
- package/services/encode/annex-b-input.js +37 -0
- package/services/encode/bit-payload.js +29 -0
- package/services/encode/output/OutputCatalog.js +1 -1
- package/services/encode/output/Timeline.js +11 -0
- package/services/encode/pcm-normalization.js +30 -0
- package/services/encode/piece-completeness.js +54 -1
- package/services/encode/quality/HostLoad.js +19 -73
- package/services/encode/quality/QualityOffer.js +1 -1
- package/services/encode/run-command.js +1 -589
- package/services/encode/segment-formats/mp4-boxes.js +3 -1
- package/services/encode/segment-formats/presentation-segment.js +114 -0
- package/services/media/ContainerOrchestrator.js +118 -48
- package/services/media/KeyframeTables.js +16 -115
- package/services/media/MediaReadRequests.js +120 -0
- package/services/media/ProbeRequests.js +135 -0
- package/services/media/SegmentInputs.js +48 -0
- package/services/media/SourceFile.js +6 -0
- package/services/media/SubtitleOrchestrator.js +190 -4
- package/services/media/await-plan.js +47 -0
- package/services/media/container/AsfContainer.js +217 -0
- package/services/media/container/AviContainer.js +343 -34
- package/services/media/container/CadencedPictureOrder.js +45 -0
- package/services/media/container/Container.js +7 -3
- package/services/media/container/ContainerFactory.js +10 -3
- package/services/media/container/FfprobeContainer.js +222 -0
- package/services/media/container/MatroskaContainer.js +72 -46
- package/services/media/container/Mp4Container.js +472 -139
- package/services/media/container/MpegPsContainer.js +281 -0
- package/services/media/container/MpegTsContainer.js +277 -0
- package/services/media/container/PacketIndex.js +270 -0
- package/services/media/container/PacketRecords.js +237 -0
- package/services/media/container/RetainedBytes.js +39 -0
- package/services/media/container/RetainedReads.js +51 -0
- package/services/media/container/ac3-frame.js +46 -0
- package/services/media/container/asf-packets.js +153 -0
- package/services/media/container/avc-picture-order.js +50 -0
- package/services/media/container/avi-open-dml.js +47 -0
- package/services/media/container/dts-frame.js +51 -0
- package/services/media/container/ebml-stream.js +12 -1
- package/services/media/container/ffprobe-record.js +45 -0
- package/services/media/container/flv-packet-ranges.js +27 -0
- package/services/media/container/following-packet-time.js +37 -0
- package/services/media/container/h264-configuration.js +183 -0
- package/services/media/container/h264-elementary-index.js +245 -0
- package/services/media/container/hevc-configuration.js +165 -0
- package/services/media/container/hevc-elementary-index.js +132 -0
- package/services/media/container/hevc-picture-order.js +66 -0
- package/services/media/container/latm-frame.js +99 -0
- package/services/media/container/matroska-packets.js +207 -0
- package/services/media/container/memory-unavailable.js +8 -0
- package/services/media/container/mp3-packet-dependencies.js +41 -0
- package/services/media/container/mp4-fragments.js +148 -0
- package/services/media/container/mpeg-audio-frame.js +19 -0
- package/services/media/container/mpeg-audio-reservoir.js +27 -0
- package/services/media/container/mpeg-elementary-index.js +196 -0
- package/services/media/container/mpeg-sequence-reader.js +69 -0
- package/services/media/container/mpeg-ts-packets.js +93 -0
- package/services/media/container/mpeg-video-configuration.js +50 -0
- package/services/media/container/mpeg4-picture-timing.js +108 -0
- package/services/media/container/pes-header.js +34 -0
- package/services/media/playback-declarations.js +57 -0
- package/services/media/playback-planner.js +72 -230
- package/services/media/probe-packets.js +61 -0
- package/services/media/read-playback-declarations.js +18 -0
- package/services/media/tracks/AudioTrack.js +9 -0
- package/services/media/tracks/TextSubtitleTrack.js +4 -1
- package/services/server/OutputLifecycle.js +2 -0
- package/services/server/SegmentServing.js +40 -33
- package/services/server/ViewerRequests.js +22 -41
- package/services/server/controllers/PlaybackController.js +4 -0
- package/services/server/controllers/SubtitleController.js +8 -29
- package/services/server/transcode-session-files.js +58 -99
- package/services/server/wire-outputs.js +58 -15
- package/services/storage/IndexMemory.js +85 -0
- package/services/storage/MachineBudget.js +27 -3
- package/services/storage/allowance.js +8 -1
- package/services/storage/piece-store/piece-disk-store.js +18 -3
- package/services/storage/piece-store/piece-lru.js +8 -4
- package/services/storage/piece-store/shared-piece-store.js +26 -11
- package/services/storage/segment-store/SegmentStore.js +49 -18
- package/services/storage/wire.js +7 -0
- package/services/torrent/Contents.js +52 -20
- package/services/torrent/demand/DemandRegister.js +13 -4
- package/services/torrent/demand/Window.js +13 -2
- package/services/torrent/demand-progress.js +25 -0
- package/services/torrent/download/DeadlineScheduler.js +167 -0
- package/services/torrent/download/FutureDownload.js +166 -0
- package/services/torrent/download/SwarmSelection.js +30 -59
- package/services/torrent/download/registry.js +45 -5
- package/services/torrent/download/withdraw-claim.js +2 -0
- package/services/torrent/torrent-pool.js +39 -600
- package/services/torrent/worker/channel.js +8 -1
- package/services/torrent/worker/client.js +74 -72
- package/services/torrent/worker/files-in-use.js +9 -0
- package/services/torrent/worker/handle-state.js +12 -0
- package/services/torrent/worker/held-bytes.js +82 -99
- package/services/torrent/worker/held-file-bytes.js +17 -0
- package/services/torrent/worker/piece-reader.js +67 -683
- package/services/torrent/worker/pool-adapter.js +37 -123
- package/services/torrent/worker/protocol.js +6 -17
- package/services/torrent/worker/whole-file-stats.js +2 -0
- package/services/torrent/worker/worker.js +86 -156
- package/services/transport/data-channel-handler.js +56 -13
- package/services/transport/tunnel-client.js +111 -20
- package/services/viewer/DownloadMaps.js +168 -0
- package/services/viewer/NativeSourceMap.js +33 -0
- package/services/viewer/PriorityMap.js +49 -40
- package/services/viewer/PriorityOrchestrator.js +42 -13
- package/services/viewer/SourcePreparation.js +278 -0
- package/services/viewer/Viewer.js +11 -0
- package/services/viewer/Viewers.js +109 -2
- package/services/viewer/playback-readiness.js +50 -63
- package/services/viewer/segment-demands.js +37 -0
- package/services/viewer/urgent-output-ready.js +26 -0
- package/services/encode/run-inputs.js +0 -53
- package/services/media/keyframe-probe.js +0 -157
- package/services/media/media-info-probe.js +0 -93
- package/services/torrent/worker/background-fill.js +0 -213
- package/services/torrent/worker/resume-warm.js +0 -107
|
@@ -108,6 +108,23 @@ export class SegmentStore {
|
|
|
108
108
|
|
|
109
109
|
/** Segment publication waiters by output key and segment number. @type {Map<string, Map<number, Set<Function>>>} */
|
|
110
110
|
#waiters = new Map();
|
|
111
|
+
#changeListeners = new Map();
|
|
112
|
+
|
|
113
|
+
/** Register before inspecting files; changes include playlists, init and segments. */
|
|
114
|
+
subscribeChanges(key, listener) {
|
|
115
|
+
const listeners = this.#changeListeners.get(key) ?? new Set();
|
|
116
|
+
this.#changeListeners.set(key, listeners);
|
|
117
|
+
listeners.add(listener);
|
|
118
|
+
this.#watch(key);
|
|
119
|
+
return () => {
|
|
120
|
+
listeners.delete(listener);
|
|
121
|
+
if (listeners.size === 0 && this.#changeListeners.get(key) === listeners) this.#changeListeners.delete(key);
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
#changed(key) {
|
|
126
|
+
for (const listener of [...(this.#changeListeners.get(key) ?? [])]) listener();
|
|
127
|
+
}
|
|
111
128
|
|
|
112
129
|
/** Output key → the watch on its directory. @type {Map<string, import("node:fs").FSWatcher>} */
|
|
113
130
|
#watchers = new Map();
|
|
@@ -499,11 +516,11 @@ export class SegmentStore {
|
|
|
499
516
|
* @param {string} key
|
|
500
517
|
* @param {string} makingName - What the encoder called it while writing.
|
|
501
518
|
* @param {{ servedNameOf?: (name: string) => string | null, segmentIndexFromName?: (name: string) => number }} format
|
|
502
|
-
* @param {{ mediaRanges?: object | null }} [read] - The piece's media
|
|
519
|
+
* @param {{ mediaRanges?: object | null, bytes?: Buffer | null }} [read] - The piece's media
|
|
503
520
|
* intervals as its format read them from the closed file, kept with it.
|
|
504
521
|
* @returns {string | null} The served name, or null where nothing was renamed.
|
|
505
522
|
*/
|
|
506
|
-
publish(key, makingName, format, { mediaRanges = null } = {}) {
|
|
523
|
+
publish(key, makingName, format, { mediaRanges = null, bytes = null } = {}) {
|
|
507
524
|
const served = format?.servedNameOf?.(makingName) ?? null;
|
|
508
525
|
if (!served) {
|
|
509
526
|
return null;
|
|
@@ -511,6 +528,10 @@ export class SegmentStore {
|
|
|
511
528
|
const dir = path.join(this.#root, directoryNameFor(key));
|
|
512
529
|
const index = format?.segmentIndexFromName?.(served) ?? -1;
|
|
513
530
|
try {
|
|
531
|
+
if (bytes !== null) {
|
|
532
|
+
if (!Buffer.isBuffer(bytes) || bytes.length === 0) throw new TypeError("Published segment bytes must be complete and nonempty.");
|
|
533
|
+
writeFileSync(path.join(dir, makingName), bytes);
|
|
534
|
+
}
|
|
514
535
|
renameSync(path.join(dir, makingName), path.join(dir, served));
|
|
515
536
|
} catch (error) {
|
|
516
537
|
// The file may already be gone — a process killed between closing the
|
|
@@ -685,10 +706,12 @@ export class SegmentStore {
|
|
|
685
706
|
// the directory is asked about only where it can have gone.
|
|
686
707
|
if (event === "rename" && !existsSync(dir)) {
|
|
687
708
|
this.#held.delete(key);
|
|
709
|
+
this.#changed(key);
|
|
688
710
|
end();
|
|
689
711
|
return;
|
|
690
712
|
}
|
|
691
713
|
this.#noticed(key, typeof name === "string" ? name : null);
|
|
714
|
+
this.#changed(key);
|
|
692
715
|
});
|
|
693
716
|
// A directory that goes away takes its watch with it rather than leaving
|
|
694
717
|
// an error nobody reads.
|
|
@@ -803,12 +826,14 @@ export class SegmentStore {
|
|
|
803
826
|
* @param {number} index
|
|
804
827
|
*/
|
|
805
828
|
announce(key, index) {
|
|
829
|
+
this.#changed(key);
|
|
806
830
|
const byIndex = this.#waiters.get(key);
|
|
807
831
|
const waiting = byIndex?.get(index);
|
|
808
832
|
if (!waiting) return;
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
833
|
+
// An event only wakes readers. The store's current bytes decide readiness,
|
|
834
|
+
// including when a publication was followed by removal before this wake.
|
|
835
|
+
if (!this.pathOf(key, index)) return;
|
|
836
|
+
for (const resolve of [...waiting]) resolve(true);
|
|
812
837
|
}
|
|
813
838
|
|
|
814
839
|
/**
|
|
@@ -822,7 +847,7 @@ export class SegmentStore {
|
|
|
822
847
|
* @returns {number}
|
|
823
848
|
*/
|
|
824
849
|
waitingFor(key) {
|
|
825
|
-
let waiting = 0;
|
|
850
|
+
let waiting = this.#changeListeners.get(key)?.size ?? 0;
|
|
826
851
|
for (const holders of this.#waiters.get(key)?.values() ?? []) {
|
|
827
852
|
waiting += holders.size;
|
|
828
853
|
}
|
|
@@ -842,16 +867,13 @@ export class SegmentStore {
|
|
|
842
867
|
* @returns {Promise<boolean>}
|
|
843
868
|
*/
|
|
844
869
|
waitFor(key, index, timeoutMs, cancelled = null) {
|
|
845
|
-
if (this.pathOf(key, index)) return Promise.resolve(true);
|
|
846
|
-
// An output adopted from a previous process has a directory nobody made in
|
|
847
|
-
// this one, so the wait itself is the first moment it is watched.
|
|
848
|
-
this.#watch(key);
|
|
849
870
|
return new Promise((resolve) => {
|
|
850
871
|
const byIndex = this.#waiters.get(key) ?? new Map();
|
|
851
872
|
const waiting = byIndex.get(index) ?? new Set();
|
|
852
873
|
this.#waiters.set(key, byIndex);
|
|
853
874
|
byIndex.set(index, waiting);
|
|
854
875
|
let settled = false;
|
|
876
|
+
let timer = null;
|
|
855
877
|
const finish = (published) => {
|
|
856
878
|
if (settled) return;
|
|
857
879
|
settled = true;
|
|
@@ -865,15 +887,16 @@ export class SegmentStore {
|
|
|
865
887
|
resolve(published);
|
|
866
888
|
};
|
|
867
889
|
waiting.add(finish);
|
|
890
|
+
// Subscribe before inspecting availability or starting the directory
|
|
891
|
+
// watch. A synchronous publication during either operation is visible.
|
|
892
|
+
this.#watch(key);
|
|
893
|
+
if (this.pathOf(key, index)) finish(true);
|
|
894
|
+
if (settled) return;
|
|
868
895
|
// No deadline is a wait that ends only on the publication: a timer given
|
|
869
896
|
// an infinite delay fires at once, which would turn the wait into a poll.
|
|
870
|
-
|
|
897
|
+
timer = Number.isFinite(timeoutMs) ? setTimeout(() => finish(false), Math.max(0, timeoutMs)) : null;
|
|
871
898
|
timer?.unref?.();
|
|
872
|
-
cancelled?.then(() => finish(false));
|
|
873
|
-
// Publication may have landed after the first path check and before this
|
|
874
|
-
// waiter was registered. Recheck once after registration so that window
|
|
875
|
-
// cannot turn a present segment into a full-deadline wait.
|
|
876
|
-
if (this.pathOf(key, index)) finish(true);
|
|
899
|
+
cancelled?.then(() => finish(false), () => finish(false));
|
|
877
900
|
});
|
|
878
901
|
}
|
|
879
902
|
|
|
@@ -892,7 +915,7 @@ export class SegmentStore {
|
|
|
892
915
|
* name in a dead run's stretch is a piece it closed.
|
|
893
916
|
*
|
|
894
917
|
* @param {string} key
|
|
895
|
-
* @param {number} startedAt - The run's
|
|
918
|
+
* @param {number | string} startedAt - The run's unique working-name tag.
|
|
896
919
|
* @returns {number} How many unfinished pieces were removed.
|
|
897
920
|
*/
|
|
898
921
|
clearUpAfter(key, startedAt) {
|
|
@@ -904,7 +927,8 @@ export class SegmentStore {
|
|
|
904
927
|
key,
|
|
905
928
|
this.directoryFor(key),
|
|
906
929
|
format,
|
|
907
|
-
|
|
930
|
+
typeof startedAt === "string" && /^[0-9a-z]+$/.test(startedAt)
|
|
931
|
+
? startedAt : String(Number.isInteger(startedAt) && startedAt > 0 ? startedAt : 0)
|
|
908
932
|
);
|
|
909
933
|
if (removed > 0) {
|
|
910
934
|
this.#logger?.info?.(
|
|
@@ -984,6 +1008,7 @@ export class SegmentStore {
|
|
|
984
1008
|
* @param {string} because
|
|
985
1009
|
*/
|
|
986
1010
|
drop(key, because) {
|
|
1011
|
+
this.#changed(key);
|
|
987
1012
|
const dir = path.join(this.#root, directoryNameFor(key));
|
|
988
1013
|
this.#unwatch(key);
|
|
989
1014
|
try {
|
|
@@ -996,6 +1021,12 @@ export class SegmentStore {
|
|
|
996
1021
|
this.#mediaRanges.delete(key);
|
|
997
1022
|
this.#touched.delete(key);
|
|
998
1023
|
this.#inits.delete(key);
|
|
1024
|
+
// A removed output cannot publish to the existing waiters. Let callers
|
|
1025
|
+
// re-evaluate its lifecycle instead of retaining an unbounded wait.
|
|
1026
|
+
const byIndex = this.#waiters.get(key);
|
|
1027
|
+
for (const waiting of byIndex?.values() ?? []) {
|
|
1028
|
+
for (const finish of [...waiting]) finish(false);
|
|
1029
|
+
}
|
|
999
1030
|
this.#logger.info(`segment-store dropped ${directoryNameFor(key)} (${because})`);
|
|
1000
1031
|
}
|
|
1001
1032
|
|
package/services/storage/wire.js
CHANGED
|
@@ -61,6 +61,8 @@ export function wireMachineBudget({
|
|
|
61
61
|
segmentStore,
|
|
62
62
|
spill,
|
|
63
63
|
memory,
|
|
64
|
+
encodeInputs,
|
|
65
|
+
indexMemory,
|
|
64
66
|
wholeFiles,
|
|
65
67
|
diagnostics,
|
|
66
68
|
diagnosticsRoot = "",
|
|
@@ -87,6 +89,10 @@ export function wireMachineBudget({
|
|
|
87
89
|
};
|
|
88
90
|
|
|
89
91
|
budget.defineResource({ name: "memory", readFree: () => availableMemoryBytes() });
|
|
92
|
+
if (encodeInputs) budget.register({ name: "admitted encode inputs", resource: "memory",
|
|
93
|
+
held: () => encodeInputs.held(), wanted: () => encodeInputs.wanted(), required: () => encodeInputs.held(), allow: bytes => encodeInputs.allow(bytes) });
|
|
94
|
+
if (indexMemory) budget.register({ name: "media metadata", resource: "memory",
|
|
95
|
+
held: () => indexMemory.held(), wanted: () => indexMemory.wanted(), required: () => indexMemory.required(), allow: bytes => indexMemory.allow(bytes) });
|
|
90
96
|
|
|
91
97
|
const segmentsOn = deviceOf(segmentStore.root);
|
|
92
98
|
defineDisk(segmentsOn, segmentStore.root);
|
|
@@ -162,6 +168,7 @@ export function wireMachineBudget({
|
|
|
162
168
|
}
|
|
163
169
|
|
|
164
170
|
return {
|
|
171
|
+
capacityOf: name => budget.capacityOf(name),
|
|
165
172
|
revise: async () => {
|
|
166
173
|
// Re-read the evidence before dividing: it is the one claimant whose
|
|
167
174
|
// bytes are written by something that never reports them.
|
|
@@ -27,19 +27,12 @@
|
|
|
27
27
|
* into the torrent's own list — and whoever takes them holds nothing of this
|
|
28
28
|
* object.
|
|
29
29
|
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
* torrent runs continuously from 0.00 to 1.00 with no gap anywhere, because a
|
|
37
|
-
* collection of short clips is made of short clips. So there is nothing here to
|
|
38
|
-
* derive a rule from, and a rule invented anyway would decide the order in
|
|
39
|
-
* which a stranger's bandwidth is spent. Every picture is an item.
|
|
40
|
-
*
|
|
41
|
-
* Nothing here reads bytes, waits on the swarm or knows about ffmpeg. Like the
|
|
42
|
-
* functions it is built on, it is a function of the list of names.
|
|
30
|
+
* Exact `sample`/`samples` words in a file or folder exclude an example clip.
|
|
31
|
+
* A video named exactly after an identified release group is its promo. Other
|
|
32
|
+
* words such as `making`, file sizes and guessed group names exclude nothing.
|
|
33
|
+
* Extension classifications are provisional until a completed media read
|
|
34
|
+
* reports whether the file carries video. This object owns every revision.
|
|
35
|
+
* Nothing here reads bytes, waits on the swarm or knows about ffmpeg.
|
|
43
36
|
*/
|
|
44
37
|
|
|
45
38
|
import { readEpisodeMarker } from "./episode-naming.js";
|
|
@@ -132,6 +125,10 @@ function showKey(text) {
|
|
|
132
125
|
}
|
|
133
126
|
|
|
134
127
|
export class TorrentContents {
|
|
128
|
+
#files;
|
|
129
|
+
#name;
|
|
130
|
+
#releaseGroup;
|
|
131
|
+
#videoFacts = new Map();
|
|
135
132
|
/** @type {TorrentItem[]} */
|
|
136
133
|
#items = [];
|
|
137
134
|
|
|
@@ -154,24 +151,56 @@ export class TorrentContents {
|
|
|
154
151
|
* @param {string} [params.name] - The torrent's name, which WebTorrent
|
|
155
152
|
* prefixes to every path in a multi-file torrent.
|
|
156
153
|
*/
|
|
157
|
-
constructor({ files, name = "" }) {
|
|
158
|
-
|
|
154
|
+
constructor({ files, name = "", releaseGroup = "" }) {
|
|
155
|
+
this.#files = Array.isArray(files) ? files : [];
|
|
156
|
+
this.#name = name;
|
|
157
|
+
this.#releaseGroup = releaseGroup;
|
|
158
|
+
this.#rebuild();
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/** A completed track read replaces the provisional extension classification. */
|
|
162
|
+
noteVideo(fileIndex, hasVideo) {
|
|
163
|
+
if (!Number.isSafeInteger(fileIndex) || !this.#files[fileIndex] || typeof hasVideo !== "boolean") return false;
|
|
164
|
+
if (this.#videoFacts.get(fileIndex) === hasVideo) return false;
|
|
165
|
+
this.#videoFacts.set(fileIndex, hasVideo);
|
|
166
|
+
this.#rebuild();
|
|
167
|
+
return true;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/** Only an explicitly identified release group may exclude its own promo. */
|
|
171
|
+
noteReleaseGroup(group) {
|
|
172
|
+
if (typeof group !== "string" || this.#releaseGroup === group) return false;
|
|
173
|
+
this.#releaseGroup = group;
|
|
174
|
+
this.#rebuild();
|
|
175
|
+
return true;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
#rebuild() {
|
|
179
|
+
const list = this.#files;
|
|
180
|
+
const name = this.#name;
|
|
181
|
+
this.#items = [];
|
|
182
|
+
this.#itemByFile.clear();
|
|
159
183
|
const described = list.map((file, fileIndex) => {
|
|
160
184
|
const { folders, name: fileName } = splitTorrentPath(file?.path ?? file?.name ?? "", name);
|
|
185
|
+
const stem = fileName.slice(0, fileName.length - extensionOf(fileName).length);
|
|
186
|
+
const sample = [...folders, stem].some((part) => /(?:^|[^\p{L}\p{N}])samples?(?:$|[^\p{L}\p{N}])/iu.test(part));
|
|
187
|
+
const promo = Boolean(this.#releaseGroup) && stem.toLowerCase() === this.#releaseGroup.toLowerCase();
|
|
161
188
|
return {
|
|
162
189
|
fileIndex,
|
|
163
190
|
name: fileName,
|
|
164
191
|
folders,
|
|
165
192
|
relativePath: [...folders, fileName].join("/"),
|
|
166
193
|
length: Number.isFinite(file?.length) ? file.length : 0,
|
|
167
|
-
isVideo: VIDEO_FILE_EXTENSIONS.has(extensionOf(fileName))
|
|
194
|
+
isVideo: this.#videoFacts.get(fileIndex) ?? VIDEO_FILE_EXTENSIONS.has(extensionOf(fileName)),
|
|
195
|
+
provisional: !this.#videoFacts.has(fileIndex),
|
|
196
|
+
excludedReason: sample ? "sample" : promo ? "release-group-promo" : null
|
|
168
197
|
};
|
|
169
198
|
});
|
|
170
199
|
this.#described = described;
|
|
171
|
-
this.#videoCount = described.filter((file) => file.isVideo).length;
|
|
200
|
+
this.#videoCount = described.filter((file) => file.isVideo && !file.excludedReason).length;
|
|
172
201
|
|
|
173
202
|
const claimed = new Set();
|
|
174
|
-
for (const file of described.filter((one) => one.isVideo).sort(inReadingOrder)) {
|
|
203
|
+
for (const file of described.filter((one) => one.isVideo && !one.excludedReason).sort(inReadingOrder)) {
|
|
175
204
|
const { audio, subtitles, images } = matchSidecarFiles({
|
|
176
205
|
files: list,
|
|
177
206
|
videoIndex: file.fileIndex,
|
|
@@ -318,7 +347,9 @@ export class TorrentContents {
|
|
|
318
347
|
name: file.name,
|
|
319
348
|
relativePath: file.relativePath,
|
|
320
349
|
length: file.length,
|
|
321
|
-
kind: this.kindOf(file.fileIndex)
|
|
350
|
+
kind: this.kindOf(file.fileIndex),
|
|
351
|
+
provisional: file.provisional,
|
|
352
|
+
excludedReason: file.isVideo ? file.excludedReason : null
|
|
322
353
|
}));
|
|
323
354
|
}
|
|
324
355
|
|
|
@@ -375,7 +406,8 @@ export function contentsOf(torrent) {
|
|
|
375
406
|
}
|
|
376
407
|
const contents = new TorrentContents({
|
|
377
408
|
files,
|
|
378
|
-
name: typeof torrent?.name === "string" ? torrent.name : ""
|
|
409
|
+
name: typeof torrent?.name === "string" ? torrent.name : "",
|
|
410
|
+
releaseGroup: typeof torrent?.releaseGroup === "string" ? torrent.releaseGroup : ""
|
|
379
411
|
});
|
|
380
412
|
byTorrent.set(torrent, { contents, fileCount: files.length });
|
|
381
413
|
return contents;
|
|
@@ -23,6 +23,9 @@ import { unionOf, Window } from "./Window.js";
|
|
|
23
23
|
export class DemandRegister {
|
|
24
24
|
/** Claimant → the one window it currently states. */
|
|
25
25
|
#windows = new Map();
|
|
26
|
+
#revision = 0;
|
|
27
|
+
|
|
28
|
+
get revision() { return this.#revision; }
|
|
26
29
|
|
|
27
30
|
/**
|
|
28
31
|
* State a need, replacing whatever that claimant said before.
|
|
@@ -39,9 +42,12 @@ export class DemandRegister {
|
|
|
39
42
|
* @param {number} params.urgency
|
|
40
43
|
* @returns {Window}
|
|
41
44
|
*/
|
|
42
|
-
state(
|
|
43
|
-
const window = new Window(
|
|
44
|
-
this.#windows.
|
|
45
|
+
state(params) {
|
|
46
|
+
const window = new Window(params);
|
|
47
|
+
const previous = this.#windows.get(window.claimant);
|
|
48
|
+
if (previous?.equals(window) && previous.requestId === window.requestId) return previous;
|
|
49
|
+
this.#windows.set(window.claimant, window);
|
|
50
|
+
this.#revision++;
|
|
45
51
|
return window;
|
|
46
52
|
}
|
|
47
53
|
|
|
@@ -57,7 +63,9 @@ export class DemandRegister {
|
|
|
57
63
|
* @returns {boolean} Whether there was anything to withdraw.
|
|
58
64
|
*/
|
|
59
65
|
withdraw(claimant) {
|
|
60
|
-
|
|
66
|
+
const removed = this.#windows.delete(claimant);
|
|
67
|
+
if (removed) this.#revision++;
|
|
68
|
+
return removed;
|
|
61
69
|
}
|
|
62
70
|
|
|
63
71
|
/** How many claimants are stating something. */
|
|
@@ -195,6 +203,7 @@ export class DemandRegister {
|
|
|
195
203
|
|
|
196
204
|
/** Forget everything. The torrent is going. */
|
|
197
205
|
clear() {
|
|
206
|
+
if (this.#windows.size) this.#revision++;
|
|
198
207
|
this.#windows.clear();
|
|
199
208
|
}
|
|
200
209
|
}
|
|
@@ -26,7 +26,7 @@ export class Window {
|
|
|
26
26
|
* @param {number} params.byteEnd - Last byte, inclusive.
|
|
27
27
|
* @param {number} params.urgency - A value of {@link import("./Urgency.js").Urgency}.
|
|
28
28
|
*/
|
|
29
|
-
constructor({ claimant, fileIndex, byteStart, byteEnd, urgency }) {
|
|
29
|
+
constructor({ claimant, fileIndex, byteStart, byteEnd, urgency, priority = 1, deadlineAt = Number.POSITIVE_INFINITY, requestId = claimant, order = 0 }) {
|
|
30
30
|
if (typeof claimant !== "string" || claimant.length === 0) {
|
|
31
31
|
throw new Error("A window needs a claimant to release it by.");
|
|
32
32
|
}
|
|
@@ -47,6 +47,14 @@ export class Window {
|
|
|
47
47
|
this.byteStart = byteStart;
|
|
48
48
|
this.byteEnd = byteEnd;
|
|
49
49
|
this.urgency = urgency;
|
|
50
|
+
if (!Number.isFinite(priority) || priority < 1 || priority > 100 ||
|
|
51
|
+
!(Number.isFinite(deadlineAt) || deadlineAt === Number.POSITIVE_INFINITY) || !Number.isSafeInteger(order) || order < 0) {
|
|
52
|
+
throw new TypeError("Invalid download priority or deadline.");
|
|
53
|
+
}
|
|
54
|
+
this.priority = priority;
|
|
55
|
+
this.deadlineAt = deadlineAt;
|
|
56
|
+
this.requestId = requestId;
|
|
57
|
+
this.order = order;
|
|
50
58
|
Object.freeze(this);
|
|
51
59
|
}
|
|
52
60
|
|
|
@@ -80,7 +88,10 @@ export class Window {
|
|
|
80
88
|
&& this.fileIndex === other.fileIndex
|
|
81
89
|
&& this.byteStart === other.byteStart
|
|
82
90
|
&& this.byteEnd === other.byteEnd
|
|
83
|
-
&& this.urgency === other.urgency
|
|
91
|
+
&& this.urgency === other.urgency
|
|
92
|
+
&& this.priority === other.priority
|
|
93
|
+
&& this.deadlineAt === other.deadlineAt
|
|
94
|
+
&& this.order === other.order;
|
|
84
95
|
}
|
|
85
96
|
|
|
86
97
|
/** @returns {string} */
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/** Count exact wanted bytes and their available residence without a read window. */
|
|
2
|
+
export function demandProgress({ windows, file, pieceLength, locationOf }) {
|
|
3
|
+
if (!Number.isSafeInteger(pieceLength) || pieceLength <= 0 || typeof locationOf !== "function") return null;
|
|
4
|
+
const ordered = windows.map(window => [Math.max(0, window.byteStart), Math.min(file.length - 1, window.byteEnd)])
|
|
5
|
+
.filter(([from, to]) => to >= from).sort((left, right) => left[0] - right[0]);
|
|
6
|
+
const ranges = [];
|
|
7
|
+
for (const range of ordered) {
|
|
8
|
+
const previous = ranges.at(-1);
|
|
9
|
+
if (previous && range[0] <= previous[1] + 1) previous[1] = Math.max(previous[1], range[1]);
|
|
10
|
+
else ranges.push(range);
|
|
11
|
+
}
|
|
12
|
+
if (!ranges.length) return null;
|
|
13
|
+
let totalBytes = 0, downloadedBytes = 0;
|
|
14
|
+
for (const [from, to] of ranges) {
|
|
15
|
+
totalBytes += to - from + 1;
|
|
16
|
+
const first = Math.floor((file.offset + from) / pieceLength), last = Math.floor((file.offset + to) / pieceLength);
|
|
17
|
+
for (let index = first; index <= last; index++) {
|
|
18
|
+
if (locationOf(index) === "missing") continue;
|
|
19
|
+
const start = Math.max(from, index * pieceLength - file.offset);
|
|
20
|
+
const end = Math.min(to, (index + 1) * pieceLength - file.offset - 1);
|
|
21
|
+
downloadedBytes += end - start + 1;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
return { totalBytes, downloadedBytes };
|
|
25
|
+
}
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import { piecesOf } from "../demand/pieces.js";
|
|
2
|
+
import { pieceStoreOf } from "../piece-store-of.js";
|
|
3
|
+
|
|
4
|
+
/** The block length and one-second request allowance used by WebTorrent. */
|
|
5
|
+
const BLOCK_BYTES = 16 * 1024;
|
|
6
|
+
const compiledDemand = new WeakMap();
|
|
7
|
+
const requestArrivals = new WeakMap();
|
|
8
|
+
|
|
9
|
+
/** Keep the same reassignment evidence in an isolated forecast request. */
|
|
10
|
+
export function copyRequestArrival(source, target) {
|
|
11
|
+
if (requestArrivals.has(source)) requestArrivals.set(target, requestArrivals.get(source));
|
|
12
|
+
return target;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function requestArrival(wire, request, now) {
|
|
16
|
+
if (!requestArrivals.has(request)) {
|
|
17
|
+
const speed = Number(wire.downloadSpeed?.());
|
|
18
|
+
let bytes = 0;
|
|
19
|
+
for (const queued of wire.requests ?? []) {
|
|
20
|
+
bytes += queued.length;
|
|
21
|
+
if (queued === request) break;
|
|
22
|
+
}
|
|
23
|
+
requestArrivals.set(request, speed > 0 ? now + bytes / speed * 1000 : Infinity);
|
|
24
|
+
}
|
|
25
|
+
return requestArrivals.get(request);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export function peerRequestCapacity(torrent, wire) {
|
|
29
|
+
const speed = Math.max(0, Number(wire.downloadSpeed?.()) || 0);
|
|
30
|
+
if (wire.type === "webSeed") {
|
|
31
|
+
return Math.min(1 + Math.ceil(speed / torrent.pieceLength), torrent.maxWebConns);
|
|
32
|
+
}
|
|
33
|
+
let capacity = 2 + Math.ceil(speed / BLOCK_BYTES);
|
|
34
|
+
const maximum = wire.peerExtendedHandshake?.reqq;
|
|
35
|
+
if (Number.isFinite(maximum) && maximum > 0) capacity = Math.min(capacity, Math.floor(maximum));
|
|
36
|
+
return capacity;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export function requestCompletionAt(wire, bytes, now) {
|
|
40
|
+
const speed = Number(wire.downloadSpeed?.());
|
|
41
|
+
if (!(speed > 0)) return Number.POSITIVE_INFINITY;
|
|
42
|
+
const queued = (wire.requests ?? []).reduce((total, request) => total + request.length, 0);
|
|
43
|
+
return now + (queued + bytes) / speed * 1000;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function peerCanServe(wire, piece) {
|
|
47
|
+
return !wire.destroyed && (!wire.peerChoking || (wire.hasFast && wire.peerAllowedFastSet?.includes(piece))) &&
|
|
48
|
+
wire.peerPieces?.get(piece);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** Identical map ordering for present requests and conditional future requests. */
|
|
52
|
+
export function compareDownloadCandidates(a, b) {
|
|
53
|
+
return a.deadlineAt - b.deadlineAt || b.priority - a.priority || a.order - b.order || a.piece - b.piece;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** One ordering across torrents; peers retain protocol-owned block verification. */
|
|
57
|
+
export class DeadlineScheduler {
|
|
58
|
+
#entries;
|
|
59
|
+
#running = false;
|
|
60
|
+
#store;
|
|
61
|
+
|
|
62
|
+
constructor({ entries, findStore = pieceStoreOf }) {
|
|
63
|
+
this.#entries = entries;
|
|
64
|
+
this.#store = findStore;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
reconcile(now = Date.now()) {
|
|
68
|
+
if (this.#running) return { requested: 0, reassigned: 0 };
|
|
69
|
+
this.#running = true;
|
|
70
|
+
try {
|
|
71
|
+
return dispatchDownloadCandidates(downloadCandidates(this.#entries(), this.#store), now);
|
|
72
|
+
} finally {
|
|
73
|
+
this.#running = false;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
/** The published map defines the same candidates for every scheduling pass. */
|
|
80
|
+
export function downloadCandidates(entries, findStore = pieceStoreOf) {
|
|
81
|
+
const candidates = [];
|
|
82
|
+
for (const { torrent, register } of entries) {
|
|
83
|
+
if (torrent.destroyed || torrent.ready === false || typeof torrent._request !== "function") continue;
|
|
84
|
+
const store = findStore(torrent);
|
|
85
|
+
let compiled = compiledDemand.get(register);
|
|
86
|
+
if (!Number.isSafeInteger(register.revision) || compiled?.revision !== register.revision ||
|
|
87
|
+
compiled.torrent !== torrent || compiled.files !== torrent.files || compiled.pieceLength !== torrent.pieceLength) {
|
|
88
|
+
const byPiece = new Map();
|
|
89
|
+
for (const window of register.windows()) {
|
|
90
|
+
const file = torrent.files?.[window.fileIndex];
|
|
91
|
+
if (!file) continue;
|
|
92
|
+
const range = piecesOf({ fileOffset: file.offset, byteStart: window.byteStart, byteEnd: Math.min(window.byteEnd, file.length - 1), pieceLength: torrent.pieceLength });
|
|
93
|
+
if (!range) continue;
|
|
94
|
+
for (let piece = range.from; piece <= range.to; piece++) {
|
|
95
|
+
const prior = byPiece.get(piece);
|
|
96
|
+
if (!prior || window.deadlineAt < prior.deadlineAt ||
|
|
97
|
+
(window.deadlineAt === prior.deadlineAt && (window.priority > prior.priority ||
|
|
98
|
+
(window.priority === prior.priority && window.order < prior.order)))) {
|
|
99
|
+
byPiece.set(piece, { torrent, piece, deadlineAt: window.deadlineAt, priority: window.priority, order: window.order, requestId: window.requestId });
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
compiled = { torrent, revision: register.revision, files: torrent.files,
|
|
104
|
+
pieceLength: torrent.pieceLength, pieces: [...byPiece.values()] };
|
|
105
|
+
compiledDemand.set(register, compiled);
|
|
106
|
+
}
|
|
107
|
+
for (const candidate of compiled.pieces) {
|
|
108
|
+
const present = store?.locationOf ? store.locationOf(candidate.piece) !== "missing" : torrent.bitfield?.get(candidate.piece);
|
|
109
|
+
if (!present) candidates.push(candidate);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
candidates.sort(compareDownloadCandidates);
|
|
113
|
+
return candidates;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** Run against protocol peers or an isolated future-state copy. */
|
|
117
|
+
export function dispatchDownloadCandidates(candidates, now) {
|
|
118
|
+
let requested = 0, reassigned = 0;
|
|
119
|
+
const hasCapacity = torrent => (torrent.wires ?? []).some(wire =>
|
|
120
|
+
!wire.destroyed && (wire.requests?.length ?? 0) < peerRequestCapacity(torrent, wire));
|
|
121
|
+
const writable = new Set(candidates.map(candidate => candidate.torrent).filter(hasCapacity));
|
|
122
|
+
for (const candidate of candidates) {
|
|
123
|
+
if (!writable.size) break;
|
|
124
|
+
const { torrent, piece, deadlineAt, requestId } = candidate;
|
|
125
|
+
if (!writable.has(torrent)) continue;
|
|
126
|
+
const wires = (torrent.wires ?? []).filter(wire => peerCanServe(wire, piece) &&
|
|
127
|
+
(wire.requests?.length ?? 0) < peerRequestCapacity(torrent, wire));
|
|
128
|
+
while (wires.length) {
|
|
129
|
+
wires.sort((a, b) => requestCompletionAt(a, BLOCK_BYTES, now) - requestCompletionAt(b, BLOCK_BYTES, now) ||
|
|
130
|
+
(a.requests?.length ?? 0) - (b.requests?.length ?? 0));
|
|
131
|
+
const wire = wires[0];
|
|
132
|
+
const before = wire.requests?.length ?? 0;
|
|
133
|
+
let accepted = torrent._request(wire, piece, false);
|
|
134
|
+
if (!accepted && Number.isFinite(deadlineAt)) {
|
|
135
|
+
const fasterAt = requestCompletionAt(wire, BLOCK_BYTES, now);
|
|
136
|
+
for (const slower of torrent.wires ?? []) {
|
|
137
|
+
if (slower === wire || typeof slower.cancel !== "function") continue;
|
|
138
|
+
const lateAt = requestCompletionAt(slower, 0, now);
|
|
139
|
+
if (!Number.isFinite(lateAt) || !(lateAt > deadlineAt && fasterAt < lateAt)) continue;
|
|
140
|
+
const request = slower.requests?.find(one => one.piece === piece);
|
|
141
|
+
if (!request) continue;
|
|
142
|
+
// A later queue estimate must not move the same block between peers
|
|
143
|
+
// indefinitely. Reassignment must improve its previous arrival;
|
|
144
|
+
// protocol completion or timeout ends this request's evidence.
|
|
145
|
+
if (!(fasterAt < requestArrival(slower, request, now))) continue;
|
|
146
|
+
slower.cancel(piece, request.offset, request.length);
|
|
147
|
+
reassigned++;
|
|
148
|
+
torrent.emit?.("download-request-cancelled", { ...request, reason: "deadline-reassigned", requestId });
|
|
149
|
+
accepted = torrent._request(wire, piece, false);
|
|
150
|
+
if (accepted) {
|
|
151
|
+
const replacement = wire.requests?.find(one => one.piece === piece && one.offset === request.offset);
|
|
152
|
+
if (replacement) requestArrivals.set(replacement, fasterAt);
|
|
153
|
+
}
|
|
154
|
+
if (accepted) break;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
if (accepted) {
|
|
158
|
+
requested++;
|
|
159
|
+
torrent.emit?.("download-requested", { piece, deadlineAt, requestId, peer: wire.remoteAddress, queued: wire.requests?.length ?? 0 });
|
|
160
|
+
}
|
|
161
|
+
if (!accepted || (wire.requests?.length ?? 0) <= before ||
|
|
162
|
+
(wire.requests?.length ?? 0) >= peerRequestCapacity(torrent, wire)) wires.shift();
|
|
163
|
+
}
|
|
164
|
+
if (!hasCapacity(torrent)) writable.delete(torrent);
|
|
165
|
+
}
|
|
166
|
+
return { requested, reassigned };
|
|
167
|
+
}
|