@torrent-tv/proxy 2.101.9 → 2.101.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/server.js +1 -0
- package/services/encode/EncodeOrchestrator.js +102 -0
- package/services/encode/PauseRule.js +23 -0
- package/services/encode/quality/HostLoad.js +43 -0
- package/services/server/ViewerRequests.js +5 -0
- package/services/server/wire-outputs.js +51 -1
- package/services/torrent/download/registry.js +27 -1
- package/services/torrent/torrent-pool.js +15 -1
- package/services/torrent/worker/client.js +11 -0
- package/services/torrent/worker/pool-adapter.js +11 -0
- package/services/torrent/worker/protocol.js +6 -0
- package/services/torrent/worker/worker.js +5 -0
package/package.json
CHANGED
package/server.js
CHANGED
|
@@ -1068,6 +1068,7 @@ export async function startProxyServer({
|
|
|
1068
1068
|
// seconds into bytes, what to ask the swarm for, what to keep in memory —
|
|
1069
1069
|
// is its own business.
|
|
1070
1070
|
setDownloadRate: (bytesPerSecond, because) => torrentPool.setDownloadRate?.(bytesPerSecond, because),
|
|
1071
|
+
setSpeculativeHold: (held, because) => torrentPool.setSpeculativeHold?.(held, because),
|
|
1071
1072
|
setPriorityMap: async ({ sourceKey, fileIndex, durationSeconds, zones }) => {
|
|
1072
1073
|
void sourcePreparation?.refresh();
|
|
1073
1074
|
const record = sourceRegistry.get(sourceKey);
|
|
@@ -152,6 +152,23 @@ export class EncodeOrchestrator {
|
|
|
152
152
|
*/
|
|
153
153
|
#pauseRule = new PauseRule();
|
|
154
154
|
|
|
155
|
+
/**
|
|
156
|
+
* Whether the run a viewer needs first is short of the processor, from the
|
|
157
|
+
* last time who has the processor was decided (`processorShortfall`).
|
|
158
|
+
*
|
|
159
|
+
* @type {{ address: string, speed: { middle: number, scatter: number }, need: number, seconds: number,
|
|
160
|
+
* waitedFor?: number } | null}
|
|
161
|
+
*/
|
|
162
|
+
#shortfall = null;
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* The piece each waiting viewer's start is held by the production of, as
|
|
166
|
+
* the start forecast last said it (`noteViewerWaiting`), by viewer.
|
|
167
|
+
*
|
|
168
|
+
* @type {Map<string, { address: string, index: number }>}
|
|
169
|
+
*/
|
|
170
|
+
#viewersWaiting = new Map();
|
|
171
|
+
|
|
155
172
|
/**
|
|
156
173
|
* What starting and stopping an encoder was measured to cost on this host,
|
|
157
174
|
* before any viewer existed. Told once; real runs replace it as they end.
|
|
@@ -238,6 +255,8 @@ export class EncodeOrchestrator {
|
|
|
238
255
|
* @param {{ freeCores: () => number | null, otherCores: () => number | null } | null} [params.processor] -
|
|
239
256
|
* The processor left free and the processor other work takes, in cores
|
|
240
257
|
* (`ProcessorBudget`); absent, both are unknown.
|
|
258
|
+
* @param {() => void} [params.shortfallNoted] - Told each time who has the
|
|
259
|
+
* processor was decided, so the download can read `processorShortfall`.
|
|
241
260
|
*/
|
|
242
261
|
constructor({
|
|
243
262
|
maxRunsFor,
|
|
@@ -253,6 +272,7 @@ export class EncodeOrchestrator {
|
|
|
253
272
|
describeWaits = null,
|
|
254
273
|
admission = null,
|
|
255
274
|
processor = null,
|
|
275
|
+
shortfallNoted = null,
|
|
256
276
|
logger,
|
|
257
277
|
now
|
|
258
278
|
}) {
|
|
@@ -262,6 +282,7 @@ export class EncodeOrchestrator {
|
|
|
262
282
|
// is about one output, and then only the per-output limit binds.
|
|
263
283
|
this.admission = admission && typeof admission.placesFor === "function" ? admission : null;
|
|
264
284
|
this.processor = processor && typeof processor.freeCores === "function" ? processor : null;
|
|
285
|
+
this.shortfallNoted = typeof shortfallNoted === "function" ? shortfallNoted : () => undefined;
|
|
265
286
|
this.#describeWaits = typeof describeWaits === "function" ? describeWaits : () => "";
|
|
266
287
|
// Only the segment facts and operations this planner needs cross the
|
|
267
288
|
// storage boundary; the concrete storage remains in the server assembly.
|
|
@@ -443,6 +464,7 @@ export class EncodeOrchestrator {
|
|
|
443
464
|
this.#lastError.delete(address);
|
|
444
465
|
this.#lastProgress.delete(address);
|
|
445
466
|
this.#failedStarts.delete(address);
|
|
467
|
+
this.#forgetWaitsOn(address);
|
|
446
468
|
}
|
|
447
469
|
|
|
448
470
|
/** @param {string} address @returns {{ at: number, count: number, blocked: boolean, limit: number }} */
|
|
@@ -598,6 +620,37 @@ export class EncodeOrchestrator {
|
|
|
598
620
|
// right whatever the last one met.
|
|
599
621
|
if (!Array.isArray(zones) || zones.length === 0) {
|
|
600
622
|
this.#forgetInputState(address);
|
|
623
|
+
this.#forgetWaitsOn(address);
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
/**
|
|
628
|
+
* What a viewer's start waits for, as the start forecast says each time it
|
|
629
|
+
* is asked: the production of piece `index` of `address`, or nothing an
|
|
630
|
+
* encoder makes (null). The map's deadlines can be met while the forecast
|
|
631
|
+
* still holds the viewer for the stock it wants after the piece they stand
|
|
632
|
+
* in: on Home Assistant (r87, torrent-tv/meta#180) the download was let go
|
|
633
|
+
* 5.3 s before a seek's viewer started, while the forecast held them for
|
|
634
|
+
* the production of the next piece. Who has the processor is decided again
|
|
635
|
+
* when it changes.
|
|
636
|
+
*
|
|
637
|
+
* @param {string} viewer
|
|
638
|
+
* @param {{ address: string, index: number } | null} wait
|
|
639
|
+
*/
|
|
640
|
+
noteViewerWaiting(viewer, wait) {
|
|
641
|
+
const said = this.#viewersWaiting.get(viewer) ?? null;
|
|
642
|
+
const next = typeof wait?.address === "string" && Number.isInteger(wait.index)
|
|
643
|
+
? { address: wait.address, index: wait.index } : null;
|
|
644
|
+
if (said?.address === next?.address && said?.index === next?.index) return;
|
|
645
|
+
if (next) this.#viewersWaiting.set(viewer, next);
|
|
646
|
+
else this.#viewersWaiting.delete(viewer);
|
|
647
|
+
this.shareTheMachine();
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
/** @param {string} address */
|
|
651
|
+
#forgetWaitsOn(address) {
|
|
652
|
+
for (const [viewer, wait] of this.#viewersWaiting) {
|
|
653
|
+
if (wait.address === address) this.#viewersWaiting.delete(viewer);
|
|
601
654
|
}
|
|
602
655
|
}
|
|
603
656
|
|
|
@@ -720,12 +773,21 @@ export class EncodeOrchestrator {
|
|
|
720
773
|
const leading = this.#leadingBand();
|
|
721
774
|
this.#leading = leading;
|
|
722
775
|
if (!leading) {
|
|
776
|
+
this.#noteShortfall(null);
|
|
723
777
|
this.#placeHeldBack(null);
|
|
724
778
|
return;
|
|
725
779
|
}
|
|
726
780
|
const now = this.now();
|
|
727
781
|
const running = this.#runningNow();
|
|
728
782
|
const need = this.#needOf(leading);
|
|
783
|
+
// Only a run in a band a viewer is in can be short for a viewer, and a run
|
|
784
|
+
// making a piece the start forecast holds a viewer for needs all it can
|
|
785
|
+
// make, whatever the map's deadlines say.
|
|
786
|
+
const waited = this.#runAViewerWaitsFor();
|
|
787
|
+
const judged = waited ?? (leading.urgent === true ? leading : null);
|
|
788
|
+
const short = judged
|
|
789
|
+
? this.#pauseRule.shortfall({ leader: judged.run, need: waited ? Number.POSITIVE_INFINITY : need }) : null;
|
|
790
|
+
this.#noteShortfall(short ? { address: judged.address, ...short, ...(waited ? { waitedFor: waited.index } : {}) } : null);
|
|
729
791
|
const free = this.#processorReading("freeCores");
|
|
730
792
|
const others = this.#processorReading("otherCores");
|
|
731
793
|
for (const [address, runs] of this.#runs) {
|
|
@@ -752,6 +814,46 @@ export class EncodeOrchestrator {
|
|
|
752
814
|
this.#placeHeldBack(leading);
|
|
753
815
|
}
|
|
754
816
|
|
|
817
|
+
/**
|
|
818
|
+
* The live run making a piece the start forecast holds a viewer for, the
|
|
819
|
+
* most urgent if there are several; null where none is.
|
|
820
|
+
*
|
|
821
|
+
* @returns {{ address: string, index: number, run: object, band: { urgent: boolean, priority: number } } | null}
|
|
822
|
+
*/
|
|
823
|
+
#runAViewerWaitsFor() {
|
|
824
|
+
let found = null;
|
|
825
|
+
for (const { address, index } of this.#viewersWaiting.values()) {
|
|
826
|
+
for (const run of this.#runs.get(address) ?? []) {
|
|
827
|
+
if (!run.isAlive || run.isStopping || run.isSuspended === true || index < run.head || index > run.to) continue;
|
|
828
|
+
const band = this.#bandAt(address, run.head);
|
|
829
|
+
if (!found || compareUrgency(band, found.band) < 0) found = { address, index, run, band };
|
|
830
|
+
}
|
|
831
|
+
}
|
|
832
|
+
return found;
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
/**
|
|
836
|
+
* Whether the run a viewer needs first is short of the processor, and by
|
|
837
|
+
* what: its output, the speed it made over its seconds kept (median and
|
|
838
|
+
* scatter) and the speed its deadlines need, infinite where a viewer waits
|
|
839
|
+
* for it (`PauseRule.shortfall`, torrent-tv/meta#180). The run the start
|
|
840
|
+
* forecast holds a viewer for is judged before the map's leader, and names
|
|
841
|
+
* the piece it is waited for by (`waitedFor`). Null where it is not short,
|
|
842
|
+
* or nothing measured it.
|
|
843
|
+
*
|
|
844
|
+
* @returns {{ address: string, speed: { middle: number, scatter: number }, need: number, seconds: number,
|
|
845
|
+
* waitedFor?: number } | null}
|
|
846
|
+
*/
|
|
847
|
+
processorShortfall() {
|
|
848
|
+
return this.#shortfall;
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
/** @param {{ address: string, speed: { middle: number, scatter: number }, need: number, seconds: number, waitedFor?: number } | null} short */
|
|
852
|
+
#noteShortfall(short) {
|
|
853
|
+
this.#shortfall = short;
|
|
854
|
+
this.shortfallNoted();
|
|
855
|
+
}
|
|
856
|
+
|
|
755
857
|
/**
|
|
756
858
|
* One second of each encoder's processor and speed, by process
|
|
757
859
|
* (`storage/processor-report.js`): what `PauseRule` judges from. Who has the
|
|
@@ -198,6 +198,29 @@ export class PauseRule {
|
|
|
198
198
|
return { runs: true, because: measured };
|
|
199
199
|
}
|
|
200
200
|
|
|
201
|
+
/**
|
|
202
|
+
* Whether the leader is short of the processor: its last second made film
|
|
203
|
+
* without waiting for its input, and over the seconds kept it made less than
|
|
204
|
+
* its need — anything at all where a viewer waits for it. Null where it is
|
|
205
|
+
* not short, waits for its input, or has fewer seconds than a median is
|
|
206
|
+
* taken from (torrent-tv/meta#180, phase 5: what the download's speculative
|
|
207
|
+
* levels give way to).
|
|
208
|
+
*
|
|
209
|
+
* @param {object} params
|
|
210
|
+
* @param {object} params.leader
|
|
211
|
+
* @param {number} params.need - `s*`, as for `judge`.
|
|
212
|
+
* @returns {{ speed: { middle: number, scatter: number }, need: number, seconds: number } | null}
|
|
213
|
+
*/
|
|
214
|
+
shortfall({ leader, need }) {
|
|
215
|
+
const kept = this.#seconds.get(leader) ?? [];
|
|
216
|
+
if (!(need > 0) || kept.length === 0 || kept.at(-1).waiting) return null;
|
|
217
|
+
const lead = kept.filter((second) => !second.waiting && second.speed !== null);
|
|
218
|
+
if (lead.length < LEAST_SECONDS) return null;
|
|
219
|
+
const speed = middleAndScatter(lead.map((second) => second.speed));
|
|
220
|
+
if (Number.isFinite(need) && speed.middle >= need) return null;
|
|
221
|
+
return { speed, need, seconds: lead.length };
|
|
222
|
+
}
|
|
223
|
+
|
|
201
224
|
/**
|
|
202
225
|
* Whether a run not started yet, on `address` and less urgent than
|
|
203
226
|
* `leader`, would be let go at once: a start the rule would pause is not
|
|
@@ -123,6 +123,15 @@ export class HostLoad {
|
|
|
123
123
|
/** @type {number[]} Recent readings behind that median. */
|
|
124
124
|
#torrentCostReadings = [];
|
|
125
125
|
|
|
126
|
+
/**
|
|
127
|
+
* What every torrent carried per interval, in bytes per second: in all, and
|
|
128
|
+
* over the time our selection held no connection back (`SelectionLimit`).
|
|
129
|
+
* Kept as readings for `swarmRate`.
|
|
130
|
+
*
|
|
131
|
+
* @type {{ carried: number[], unheld: number[] }}
|
|
132
|
+
*/
|
|
133
|
+
#swarmReadings = { carried: [], unheld: [] };
|
|
134
|
+
|
|
126
135
|
/**
|
|
127
136
|
* The share of the machine this process draws with nothing encoding and the
|
|
128
137
|
* torrents moving nothing — the spending that would have happened anyway.
|
|
@@ -264,6 +273,7 @@ export class HostLoad {
|
|
|
264
273
|
// worker starts its clock again: such an interval says nothing.
|
|
265
274
|
if (!(elapsedSec > 0) || megabytes < 0 || now.threadCpuSeconds < previous.threadCpuSeconds) return;
|
|
266
275
|
const cpuSeconds = (now.threadCpuSeconds - previous.threadCpuSeconds) / Math.max(1, os.cpus().length);
|
|
276
|
+
this.#noteSwarm(now, previous, megabytes * 1e6 / elapsedSec);
|
|
267
277
|
const limit = this.downloadLimit;
|
|
268
278
|
// What else the thread moved over the same interval, across every torrent:
|
|
269
279
|
// sent back to the swarm, and handed to readers (encoder inputs, container
|
|
@@ -311,6 +321,39 @@ export class HostLoad {
|
|
|
311
321
|
);
|
|
312
322
|
}
|
|
313
323
|
|
|
324
|
+
/**
|
|
325
|
+
* What the swarm can give every torrent, in bytes per second: the median of
|
|
326
|
+
* what it carried while our selection held no connection back, but never
|
|
327
|
+
* less than the median of what it carried in all — the same choice the
|
|
328
|
+
* start forecast makes for one source (`ViewerRequests` `sourceService`).
|
|
329
|
+
* Time our selection held a connection back measures our asking, not the
|
|
330
|
+
* swarm, so holding the speculative levels back does not lower it
|
|
331
|
+
* (torrent-tv/meta#180). Null before any interval was read.
|
|
332
|
+
*
|
|
333
|
+
* @returns {number | null}
|
|
334
|
+
*/
|
|
335
|
+
swarmRate() {
|
|
336
|
+
const carried = medianOf(this.#swarmReadings.carried);
|
|
337
|
+
const unheld = medianOf(this.#swarmReadings.unheld);
|
|
338
|
+
if (carried === null && unheld === null) return null;
|
|
339
|
+
return Math.max(carried ?? 0, unheld ?? 0);
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
/**
|
|
343
|
+
* @param {{ selection: { freeBytes?: number, freeMs?: number } | null }} now
|
|
344
|
+
* @param {{ selection: { freeBytes?: number, freeMs?: number } | null }} previous
|
|
345
|
+
* @param {number} carried - Bytes per second over the interval, every torrent.
|
|
346
|
+
*/
|
|
347
|
+
#noteSwarm(now, previous, carried) {
|
|
348
|
+
this.#swarmReadings.carried = [...this.#swarmReadings.carried, carried].slice(-READINGS_KEPT);
|
|
349
|
+
const freeMs = (now.selection?.freeMs ?? 0) - (previous.selection?.freeMs ?? 0);
|
|
350
|
+
const freeBytes = (now.selection?.freeBytes ?? 0) - (previous.selection?.freeBytes ?? 0);
|
|
351
|
+
// Totals that went back belong to a torrent that left: nothing to read.
|
|
352
|
+
if (freeMs > 0 && freeBytes >= 0) {
|
|
353
|
+
this.#swarmReadings.unheld = [...this.#swarmReadings.unheld, freeBytes * 1000 / freeMs].slice(-READINGS_KEPT);
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
|
|
314
357
|
/**
|
|
315
358
|
* What the torrent thread draws in an interval that downloaded nothing.
|
|
316
359
|
*
|
|
@@ -790,6 +790,11 @@ export class ViewerRequests {
|
|
|
790
790
|
})}`);
|
|
791
791
|
}
|
|
792
792
|
this.#sayHolding(session, consumerId, measurement, forecast, input, now);
|
|
793
|
+
// The encoder this viewer waits for, told to whoever shares the processor
|
|
794
|
+
// (torrent-tv/meta#180): the forecast knows it before the map's deadlines do.
|
|
795
|
+
const holding = forecast.holdingBy;
|
|
796
|
+
this.#host.noteViewerWaiting?.(consumerId || session.id,
|
|
797
|
+
holding?.kind === "production" ? { address: holding.track, index: holding.segment } : null);
|
|
793
798
|
return forecast;
|
|
794
799
|
}
|
|
795
800
|
|
|
@@ -205,6 +205,7 @@ export function wireOutputs({
|
|
|
205
205
|
getSourceStats = null,
|
|
206
206
|
setPriorityMap = null,
|
|
207
207
|
setDownloadRate = null,
|
|
208
|
+
setSpeculativeHold = null,
|
|
208
209
|
// What a second job costs on this host, measured at startup. Null when it
|
|
209
210
|
// could not be measured, and then nothing is corrected — the alternative
|
|
210
211
|
// is inventing a penalty, which is the same fault as inventing a fill rate.
|
|
@@ -630,6 +631,7 @@ export function wireOutputs({
|
|
|
630
631
|
disposeSession: (...args) => parts.lifecycle.disposeSession(...args),
|
|
631
632
|
planEncodersSoon: (...args) => parts.encodeRuns.planEncodersSoon(...args),
|
|
632
633
|
waitUntilReady: (...args) => parts.serving.waitUntilReady(...args),
|
|
634
|
+
noteViewerWaiting: (viewer, wait) => parts.encodeOrchestrator?.noteViewerWaiting(viewer, wait),
|
|
633
635
|
get encodeRuns() { return parts.encodeRuns; },
|
|
634
636
|
// When this viewer's playback can start and run to the end: the viewer
|
|
635
637
|
// component's forecast, which models the viewer's browser.
|
|
@@ -753,6 +755,49 @@ export function wireOutputs({
|
|
|
753
755
|
void Promise.resolve(parts.setDownloadRate(share.bytesPerSecond, `film speed ${share.filmSpeed.toFixed(2)}x ` +
|
|
754
756
|
`over ${watched.length} file(s), ${Math.round(share.capacity * 100)}% of the machine for the proxy`)).catch(() => {});
|
|
755
757
|
};
|
|
758
|
+
parts.setSpeculativeHold = typeof setSpeculativeHold === "function" ? setSpeculativeHold : null;
|
|
759
|
+
// THE DOWNLOAD GIVES WAY TO THE ENCODER A VIEWER NEEDS FIRST
|
|
760
|
+
// (torrent-tv/meta#180, phase 5). While that encoder makes film on the
|
|
761
|
+
// processor and less than its deadlines need, what nobody needs yet — the
|
|
762
|
+
// film's tail, the gap behind a seek — is not downloaded, so the torrent
|
|
763
|
+
// thread's processor goes to it. Only where the swarm gives more than the
|
|
764
|
+
// film needs at the speed asked of it (its need, or with a viewer waiting
|
|
765
|
+
// what it makes up to its scatter): then every byte an encoder will reach
|
|
766
|
+
// still comes first and in time, and nothing is lost by not fetching ahead.
|
|
767
|
+
// Where the swarm is slower than that, the bytes fetched ahead are what the
|
|
768
|
+
// film will play from, and they are fetched. Decided each time who has the
|
|
769
|
+
// processor is decided and each time the machine is read.
|
|
770
|
+
parts.speculativeHeld = false;
|
|
771
|
+
parts.applySpeculativeHold = () => {
|
|
772
|
+
if (!parts.setSpeculativeHold || !parts.hostLoad || !parts.encodeOrchestrator) return;
|
|
773
|
+
const short = parts.encodeOrchestrator.processorShortfall();
|
|
774
|
+
let held = false;
|
|
775
|
+
let because = "the encoder a viewer needs first is not short of the processor";
|
|
776
|
+
if (short) {
|
|
777
|
+
const output = parts.outputs.outputsOn(short.address)[0] ?? null;
|
|
778
|
+
const length = Number(output?.file?.lengthBytes ?? parts.hostLoad.fileLengthByKey.get(output?.file?.key));
|
|
779
|
+
const bytesPerFilmSecond = length / Number(output?.file?.durationSeconds);
|
|
780
|
+
const asked = Number.isFinite(short.need) ? short.need : short.speed.middle + short.speed.scatter;
|
|
781
|
+
const needs = bytesPerFilmSecond * asked;
|
|
782
|
+
const swarm = parts.hostLoad.swarmRate();
|
|
783
|
+
const made = `the encoder a viewer needs first made ${short.speed.middle.toFixed(2)}x ` +
|
|
784
|
+
`(scatter ${short.speed.scatter.toFixed(2)}) over ${short.seconds}s` +
|
|
785
|
+
(Number.isFinite(short.need) ? `, needing ${short.need.toFixed(2)}x` : " with a viewer waiting") +
|
|
786
|
+
(Number.isInteger(short.waitedFor) ? ` for its #${short.waitedFor} by the start forecast` : "");
|
|
787
|
+
if (!Number.isFinite(needs) || needs <= 0) {
|
|
788
|
+
because = `${made}; its film's bytes per second are unknown`;
|
|
789
|
+
} else if (swarm === null) {
|
|
790
|
+
because = `${made}; the swarm's rate is not measured yet`;
|
|
791
|
+
} else {
|
|
792
|
+
held = swarm > needs;
|
|
793
|
+
because = `${made}; the swarm gives ${(swarm / 1e6).toFixed(2)} MB/s, ` +
|
|
794
|
+
`${held ? "more than" : "not more than"} the ${(needs / 1e6).toFixed(2)} MB/s its film needs at ${asked.toFixed(2)}x`;
|
|
795
|
+
}
|
|
796
|
+
}
|
|
797
|
+
if (held === parts.speculativeHeld) return;
|
|
798
|
+
parts.speculativeHeld = held;
|
|
799
|
+
void Promise.resolve(parts.setSpeculativeHold(held, because)).catch(() => {});
|
|
800
|
+
};
|
|
756
801
|
// The one owner of what the processor gives the encoders: the share of the
|
|
757
802
|
// machine other work leaves and the measured contention penalties, read by
|
|
758
803
|
// every place that decides on the processor (torrent-tv/meta#180).
|
|
@@ -1090,6 +1135,7 @@ export function wireOutputs({
|
|
|
1090
1135
|
// The processor left free and what other work takes, for the pause rule
|
|
1091
1136
|
// (torrent-tv/meta#180, phase 4).
|
|
1092
1137
|
processor: parts.processor,
|
|
1138
|
+
shortfallNoted: () => parts.applySpeculativeHold?.(),
|
|
1093
1139
|
startingSpeedFor: (address) => parts.encodeCost.speedForOutput(address),
|
|
1094
1140
|
segmentCoverage,
|
|
1095
1141
|
// HOW IT ASKS TO DECIDE AGAIN. A plan that refuses to place anything
|
|
@@ -1257,7 +1303,11 @@ export function wireOutputs({
|
|
|
1257
1303
|
liveConsumers: (...args) => parts.renditions.liveConsumers(...args),
|
|
1258
1304
|
liveRunsOf: (...args) => parts.encodeRuns.liveRunsOf(...args),
|
|
1259
1305
|
producedNumbers: (...args) => parts.serving.producedNumbers(...args),
|
|
1260
|
-
reportHostLoad: async (...args) => {
|
|
1306
|
+
reportHostLoad: async (...args) => {
|
|
1307
|
+
await parts.hostLoad.reportHostLoad(...args);
|
|
1308
|
+
parts.applyDownloadShare?.();
|
|
1309
|
+
parts.applySpeculativeHold?.();
|
|
1310
|
+
},
|
|
1261
1311
|
runStateOf: (...args) => parts.encodeRuns.runStateOf(...args),
|
|
1262
1312
|
sampleDownloadRates: (...args) => parts.hostLoad.sampleDownloadRates(...args),
|
|
1263
1313
|
get encodeCost() { return parts.encodeCost; },
|
|
@@ -30,6 +30,12 @@ let futurePending = null;
|
|
|
30
30
|
let publishingSelections = false;
|
|
31
31
|
/** What the last reconciliation told the swarm to fetch first, for the pool to say. */
|
|
32
32
|
let firstClassStated = "none";
|
|
33
|
+
/**
|
|
34
|
+
* Whether the speculative levels are held back for the processor: the encoder
|
|
35
|
+
* a viewer needs first is short of it, and the swarm gives more than its film
|
|
36
|
+
* needs (`holdSpeculative`, torrent-tv/meta#180).
|
|
37
|
+
*/
|
|
38
|
+
let speculativeHeld = false;
|
|
33
39
|
|
|
34
40
|
/**
|
|
35
41
|
* The last forecast that finished on the maps it was begun on: its result, the
|
|
@@ -222,6 +228,26 @@ export function firstClassStatement() {
|
|
|
222
228
|
return firstClassStated;
|
|
223
229
|
}
|
|
224
230
|
|
|
231
|
+
/**
|
|
232
|
+
* Hold the speculative levels back, or let them go again, on every torrent.
|
|
233
|
+
*
|
|
234
|
+
* Downloading what nobody needs yet takes the torrent thread's processor
|
|
235
|
+
* (Home Assistant r84: 9-19 % of the machine while it downloaded 7-14 MB/s
|
|
236
|
+
* and handed readers about 0.5 MB/s in most intervals, with 0-6 % of the
|
|
237
|
+
* machine free). The side that measures the encoders decides when that processor is
|
|
238
|
+
* worth more to them (torrent-tv/meta#180, phase 5); the urgent levels are
|
|
239
|
+
* never held, so every byte an encoder will reach is still asked for first.
|
|
240
|
+
*
|
|
241
|
+
* @param {boolean} held
|
|
242
|
+
* @returns {boolean} Whether this changed anything.
|
|
243
|
+
*/
|
|
244
|
+
export function holdSpeculative(held) {
|
|
245
|
+
const next = held === true;
|
|
246
|
+
if (next === speculativeHeld) return false;
|
|
247
|
+
speculativeHeld = next;
|
|
248
|
+
return true;
|
|
249
|
+
}
|
|
250
|
+
|
|
225
251
|
/**
|
|
226
252
|
* Bring every torrent's download set into line with what is stated.
|
|
227
253
|
*
|
|
@@ -235,7 +261,7 @@ export function firstClassStatement() {
|
|
|
235
261
|
export function reconcileAll() {
|
|
236
262
|
if (publishingSelections) return { torrents: live.size, speculativeAllowed: false, stated: 0, withdrawn: 0 };
|
|
237
263
|
const entries = [...live];
|
|
238
|
-
const speculativeAllowed = !entries.some((entry) => entry.selection.hasUrgentMissing());
|
|
264
|
+
const speculativeAllowed = !speculativeHeld && !entries.some((entry) => entry.selection.hasUrgentMissing());
|
|
239
265
|
// The first class across every torrent: the most urgent missing level and
|
|
240
266
|
// its highest priority. Lower classes stay out of every torrent's selection.
|
|
241
267
|
const bands = entries.map(entry => entry.selection.missingBand()).filter(Boolean);
|
|
@@ -18,7 +18,7 @@ import { logger } from "../../utils/logger.js";
|
|
|
18
18
|
import { pieceStoreOf } from "./piece-store-of.js";
|
|
19
19
|
import { demandProgress } from "./demand-progress.js";
|
|
20
20
|
import { Urgency, urgencyName } from "./demand/index.js";
|
|
21
|
-
import { demandFor, firstClassStatement, forgetTorrent, reconcileAll, hasUnmetDemand, selectionLimitOf } from "./download/registry.js";
|
|
21
|
+
import { demandFor, firstClassStatement, forgetTorrent, holdSpeculative, reconcileAll, hasUnmetDemand, selectionLimitOf } from "./download/registry.js";
|
|
22
22
|
import { applyDownloadRate } from "./download/throttle.js";
|
|
23
23
|
import { withdrawClaim } from "./download/withdraw-claim.js";
|
|
24
24
|
import { deadlineOfMapZone, levelOfMapZone, priorityOfMapZone } from "./map-zones.js";
|
|
@@ -1309,6 +1309,20 @@ export class TorrentPool {
|
|
|
1309
1309
|
/** The power of two the last limit said was nearest, null for none. */
|
|
1310
1310
|
#downloadLimitScale = null;
|
|
1311
1311
|
|
|
1312
|
+
/**
|
|
1313
|
+
* Hold back, or let go, what every torrent downloads speculatively: the
|
|
1314
|
+
* film's tail and the gap behind a seek (`download/registry.js`
|
|
1315
|
+
* `holdSpeculative`, torrent-tv/meta#180). Said when it changes.
|
|
1316
|
+
*
|
|
1317
|
+
* @param {boolean} held
|
|
1318
|
+
* @param {string} [because]
|
|
1319
|
+
*/
|
|
1320
|
+
setSpeculativeHold(held, because = "") {
|
|
1321
|
+
if (!holdSpeculative(held)) return;
|
|
1322
|
+
logger.info(`torrent-pool: speculative download ${held === true ? "held" : "let go"}${because ? ` (${because})` : ""}`);
|
|
1323
|
+
reconcileAll();
|
|
1324
|
+
}
|
|
1325
|
+
|
|
1312
1326
|
applyPriorityMap(torrent, fileIndex, zones, durationSeconds) {
|
|
1313
1327
|
// A MAP WITH NOTHING IN IT IS A STATEMENT, and it is answered before
|
|
1314
1328
|
// anything else is looked at. It says nobody wants any of this file, and
|
|
@@ -511,6 +511,17 @@ export class TorrentWorkerClient {
|
|
|
511
511
|
return this.#caller.call(Command.DOWNLOAD_RATE, { bytesPerSecond: Number.isFinite(bytesPerSecond) ? bytesPerSecond : null, because });
|
|
512
512
|
}
|
|
513
513
|
|
|
514
|
+
/**
|
|
515
|
+
* Whether the speculative levels are held back (`Command.SPECULATIVE_HOLD`).
|
|
516
|
+
*
|
|
517
|
+
* @param {boolean} held
|
|
518
|
+
* @param {string} [because]
|
|
519
|
+
* @returns {Promise<void>}
|
|
520
|
+
*/
|
|
521
|
+
async setSpeculativeHold(held, because = "") {
|
|
522
|
+
return this.#caller.call(Command.SPECULATIVE_HOLD, { held: held === true, because });
|
|
523
|
+
}
|
|
524
|
+
|
|
514
525
|
async setPriorityMap({ sourceKey, fileIndex, zones, durationSeconds, postedAt }) {
|
|
515
526
|
return this.#caller.call(Command.PRIORITY_MAP, {
|
|
516
527
|
sourceKey,
|
|
@@ -179,6 +179,17 @@ export class WorkerTorrentPool {
|
|
|
179
179
|
return this.#client.setDownloadRate(bytesPerSecond, because);
|
|
180
180
|
}
|
|
181
181
|
|
|
182
|
+
/**
|
|
183
|
+
* Whether the speculative levels are held back for the encoders.
|
|
184
|
+
*
|
|
185
|
+
* @param {boolean} held
|
|
186
|
+
* @param {string} [because]
|
|
187
|
+
* @returns {Promise<void>}
|
|
188
|
+
*/
|
|
189
|
+
async setSpeculativeHold(held, because = "") {
|
|
190
|
+
return this.#client.setSpeculativeHold(held, because);
|
|
191
|
+
}
|
|
192
|
+
|
|
182
193
|
async setPriorityMap({ sourceKey, fileIndex, durationSeconds, zones, postedAt }) {
|
|
183
194
|
if (!sourceKey) {
|
|
184
195
|
return;
|
|
@@ -72,6 +72,12 @@ export const Command = {
|
|
|
72
72
|
* (`storage/processor-share.js`, torrent-tv/meta#178).
|
|
73
73
|
*/
|
|
74
74
|
DOWNLOAD_RATE: "download-rate",
|
|
75
|
+
/**
|
|
76
|
+
* Whether the speculative levels are held back for the processor the
|
|
77
|
+
* encoders need (`download/registry.js` `holdSpeculative`,
|
|
78
|
+
* torrent-tv/meta#180).
|
|
79
|
+
*/
|
|
80
|
+
SPECULATIVE_HOLD: "speculative-hold",
|
|
75
81
|
/** Read a byte range; the body arrives as CHUNK messages. */
|
|
76
82
|
READ_RANGE: "read-range",
|
|
77
83
|
/** Abandon an in-flight READ_RANGE (viewer gone, seek superseded). */
|
|
@@ -525,6 +525,11 @@ async function runCommand(command, params, id) {
|
|
|
525
525
|
return true;
|
|
526
526
|
}
|
|
527
527
|
|
|
528
|
+
case Command.SPECULATIVE_HOLD: {
|
|
529
|
+
pool.setSpeculativeHold(params.held === true, params.because);
|
|
530
|
+
return true;
|
|
531
|
+
}
|
|
532
|
+
|
|
528
533
|
case Command.READ_RANGE: {
|
|
529
534
|
// Streams its own reply; the caller's promise resolves once the body has
|
|
530
535
|
// been fully sent, which is what lets the client await completion.
|