@torrent-tv/proxy 2.101.6 → 2.101.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@torrent-tv/proxy",
3
- "version": "2.101.6",
3
+ "version": "2.101.7",
4
4
  "description": "Torrent proxy client that exposes webseed-like HTTP stream endpoint.",
5
5
  "license": "GPL-3.0-or-later",
6
6
  "repository": {
@@ -1,4 +1,5 @@
1
1
  import { parseRange } from "../../utils/parse-range.js";
2
+ import { Readable } from "node:stream";
2
3
 
3
4
  /** FFmpeg reads only original byte ranges already owned by its live run. */
4
5
  export async function handleEncodeInputGet(req, reply, { inputOf, refused = () => {} }) {
@@ -53,10 +54,12 @@ export async function handleEncodeInputGet(req, reply, { inputOf, refused = () =
53
54
  const socket = req.raw?.socket;
54
55
  const writtenBefore = socket?.bytesWritten ?? 0;
55
56
  reply.raw?.once?.("close", () => {
57
+ held.release?.();
56
58
  input.noteAnswer?.({ fileIndex, start: range.start, end: held.end,
57
59
  written: Math.max(0, (socket?.bytesWritten ?? writtenBefore) - writtenBefore),
58
60
  handedOver: reply.raw?.writableFinished === true });
59
61
  });
60
- return reply.code(206).header("Content-Length", String(held.bytes.length))
61
- .header("Content-Range", `bytes ${range.start}-${held.end}/${length}`).send(held.bytes);
62
+ return reply.code(206).header("Content-Length", String(held.size ?? held.bytes.length))
63
+ .header("Content-Range", `bytes ${range.start}-${held.end}/${length}`)
64
+ .send(held.stream ?? (held.chunks ? Readable.from(held.chunks, { objectMode: false }) : held.bytes));
62
65
  }
package/server.js CHANGED
@@ -978,22 +978,26 @@ export async function startProxyServer({
978
978
  return answer && typeof answer.then !== "function" && answer.end === end ? answer.bytes : null;
979
979
  };
980
980
  const holdsUntil = start => {
981
+ if (input.heldUntil) return input.heldUntil(source.fileIndex, start);
981
982
  const range = source.input.ranges.find(([first, last]) => first <= start && start <= last);
982
983
  return range ? range[1] + 1 : null;
983
984
  };
984
985
  const startedAt = Date.now();
986
+ const releaseInput = input.retain?.();
985
987
  void container.learnPackets({ from: source.input.from, to: source.input.to, trackIds: [...sounds] }, read, holdsUntil).then(learned =>
986
988
  logger.info(`encode input #${from}..#${to} of ${output.outputKey}: soundtrack packets of track(s) ${[...sounds].join(",")} ` +
987
989
  `over ${source.input.from.toFixed(3)}-${source.input.to.toFixed(3)}s ` +
988
990
  (learned === "walked" ? "walked" : `not all walked (${learned === "memory" ? "memory refused" : "the copy does not hold the rest"})`) +
989
991
  ` from its copy in ${Date.now() - startedAt}ms`),
990
- error => logger.warn(`encode input #${from}..#${to} of ${output.outputKey}: walking the soundtrack from its copy failed: ${error?.message ?? error}`));
992
+ error => logger.warn(`encode input #${from}..#${to} of ${output.outputKey}: walking the soundtrack from its copy failed: ${error?.message ?? error}`))
993
+ .finally(() => releaseInput?.());
991
994
  }
992
995
  },
993
996
  readEncodeRanges: async (source, ranges, maxBytes) => {
994
997
  const torrent = torrentPool.knownTorrent(source.sourceKey);
995
998
  return torrent ? torrentPool.readHeldRangesOf(torrent, source.fileIndex, ranges, maxBytes) : null;
996
999
  },
1000
+ openEncodeWholeFile: source => torrentPool.acquireWholeFile(source),
997
1001
  // Which bytes of a file are downloaded whole now — what an original
998
1002
  // input's stretch may extend over. A snapshot: the copy that follows
999
1003
  // holds the pieces it reads, and refuses where one has gone since.
@@ -1036,6 +1040,7 @@ export async function startProxyServer({
1036
1040
  wholeFiles: typeof torrentPool.allowWholeFileBytes === "function"
1037
1041
  ? {
1038
1042
  held: () => torrentPool.wholeFileBytes ?? 0,
1043
+ required: () => torrentPool.wholeFileRequiredBytes ?? 0,
1039
1044
  allow: (bytes) => torrentPool.allowWholeFileBytes(bytes),
1040
1045
  root: completedFilesRoot()
1041
1046
  }
@@ -1,5 +1,6 @@
1
1
  import { admitInput } from "./AdmittedInput.js";
2
2
  import { admitOriginalInput, originalInputBytes } from "./OriginalInput.js";
3
+ import { OriginalInputStore } from "./OriginalInputStore.js";
3
4
 
4
5
  /**
5
6
  * How many recent readings a learned figure here is taken from: long enough
@@ -32,6 +33,7 @@ export class EncodeInputs {
32
33
  #preparedIn = new WeakMap();
33
34
  #memoryRevision = 0;
34
35
  #held = 0;
36
+ #shared = new OriginalInputStore({ changed: () => queueMicrotask(() => this.memoryChanged()) });
35
37
  #allowed = 0;
36
38
  /** What each preparing request asks of memory: all it would use, and the least it can run with. */
37
39
  #wanted = new Map();
@@ -46,6 +48,7 @@ export class EncodeInputs {
46
48
  #openOf;
47
49
  #resolve;
48
50
  #read;
51
+ #openWholeFile;
49
52
  #heldRanges;
50
53
  #revise;
51
54
  #changed;
@@ -63,12 +66,13 @@ export class EncodeInputs {
63
66
  */
64
67
  #stretchCopied;
65
68
 
66
- constructor({ resolve, readRanges, heldRanges = null, reviseBudget, changed, failed, capacity = () => null, urgent = () => true,
69
+ constructor({ resolve, readRanges, openWholeFile = null, heldRanges = null, reviseBudget, changed, failed, capacity = () => null, urgent = () => true,
67
70
  rankAt = () => null, openOf = () => null, log = () => {}, now = Date.now, stretchCopied = () => {} }) {
68
71
  this.#stretchCopied = stretchCopied;
69
72
  this.#openOf = openOf;
70
73
  this.#resolve = resolve;
71
74
  this.#read = readRanges;
75
+ this.#openWholeFile = openWholeFile;
72
76
  this.#heldRanges = heldRanges;
73
77
  this.#revise = reviseBudget;
74
78
  this.#changed = changed;
@@ -95,7 +99,7 @@ export class EncodeInputs {
95
99
  const serves = request.serves ?? request.from;
96
100
  const rank = this.#rankAt(request.output, serves);
97
101
  this.#log(`encode input #${request.from}..#${request.to} of ${request.output.outputKey ?? request.output.id}: ${state}` +
98
- ` (held ${this.#held} of ${this.#allowed} allowed; for #${serves}` +
102
+ ` (held ${this.held()} of ${this.#allowed} allowed; for #${serves}` +
99
103
  `${rank ? `, rank ${rank.rank} of ${rank.topRank}` : ""})${detail}`);
100
104
  }
101
105
 
@@ -120,10 +124,10 @@ export class EncodeInputs {
120
124
  }
121
125
  }
122
126
 
123
- held() { return this.#held; }
124
- wanted() { return this.#held + [...this.#wanted.values()].reduce((sum, asked) => sum + asked.wanted, 0); }
127
+ held() { return this.#held + this.#shared.bytes; }
128
+ wanted() { return this.held() + [...this.#wanted.values()].reduce((sum, asked) => sum + asked.wanted, 0); }
125
129
  required() {
126
- let bytes = this.#held;
130
+ let bytes = this.#held + this.#shared.required;
127
131
  for (const [key, asked] of this.#wanted) {
128
132
  // The encode plan already chose this finite input. Its least runnable
129
133
  // part must fit before production can advance, even when playback has
@@ -137,6 +141,7 @@ export class EncodeInputs {
137
141
  allow(bytes) {
138
142
  const changed = this.#allowed !== Math.max(0, bytes);
139
143
  this.#allowed = Math.max(0, bytes);
144
+ this.#shared.trim(this.#allowed - this.#held);
140
145
  if (changed) this.memoryChanged();
141
146
  }
142
147
 
@@ -320,6 +325,7 @@ export class EncodeInputs {
320
325
  this.#wanted.delete(key);
321
326
  request.result?.release?.();
322
327
  }
328
+ this.#shared.forget(output);
323
329
  }
324
330
 
325
331
  /**
@@ -409,7 +415,23 @@ export class EncodeInputs {
409
415
  if (resolved.kind !== "result") {
410
416
  result = resolved;
411
417
  } else if (resolved.sources?.every(source => source.input.original === true)) {
412
- result = await this.#admitStretch(request, resolved.sources);
418
+ const sharing = resolved.sources.every(source => Array.isArray(source.input.commonRanges));
419
+ if (sharing) this.#shared.retain(request.output, resolved.sources);
420
+ const files = new Map();
421
+ let unpin = () => {};
422
+ try {
423
+ for (const source of resolved.sources) {
424
+ const file = await this.#openWholeFile?.(source);
425
+ if (file && file.length === source.input.fileLength) files.set(source.fileIndex, file);
426
+ else if (file) await file.release();
427
+ if (this.#requests.get(request.key) !== request) return;
428
+ }
429
+ if (sharing) unpin = this.#shared.protect(resolved.sources.filter(source => !files.has(source.fileIndex)));
430
+ result = await this.#admitStretch(request, resolved.sources, files);
431
+ } finally {
432
+ unpin();
433
+ if (result?.kind !== "result") await Promise.all([...files.values()].map(file => file.release()));
434
+ }
413
435
  if (!result) return;
414
436
  } else {
415
437
  result = await admitInput({ sources: resolved.sources, readRanges: this.#read, reserve: this.#reserveFor(request) });
@@ -475,7 +497,9 @@ export class EncodeInputs {
475
497
 
476
498
  /** Take `bytes` of the allowance for this request now, or answer null. */
477
499
  #hold(request, bytes) {
478
- if (this.#requests.get(request.key) !== request || this.#allowed - this.#held < bytes) return null;
500
+ if (this.#requests.get(request.key) !== request) return null;
501
+ this.#shared.trim(this.#allowed - this.#held - bytes);
502
+ if (this.#allowed - this.held() < bytes) return null;
479
503
  this.#wanted.delete(request.key);
480
504
  this.#held += bytes;
481
505
  let released = false;
@@ -497,8 +521,11 @@ export class EncodeInputs {
497
521
  *
498
522
  * @returns {Promise<object | undefined>} Undefined when the request was withdrawn.
499
523
  */
500
- async #admitStretch(request, firstSources) {
524
+ async #admitStretch(request, firstSources, files = new Map()) {
501
525
  const { output, from } = request;
526
+ const sharing = files.size > 0 || firstSources.every(source => Array.isArray(source.input.commonRanges));
527
+ const inMemory = sources => sources.filter(source => !files.has(source.fileIndex));
528
+ const memoryBytes = sources => sharing ? this.#shared.bytesNeeded(inMemory(sources)) : originalInputBytes(sources);
502
529
  // Record existing decisions only: diagnostics must not resolve another
503
530
  // interval or ask storage for an additional availability reading.
504
531
  const probes = [], reservations = [];
@@ -515,12 +542,13 @@ export class EncodeInputs {
515
542
  }
516
543
  return stretches.get(end);
517
544
  };
518
- const fits = async (end, maxBytes) => {
545
+ const fits = async (end, maxBytes, memory = false) => {
519
546
  const stretch = await at(end);
520
- const withinLimit = Boolean(stretch) && stretch.bytes <= maxBytes;
547
+ const withinLimit = Boolean(stretch) && (memory ? memoryBytes(stretch.sources) <= maxBytes && stretch.bytes <= (target ?? minimum) : stretch.bytes <= maxBytes);
521
548
  const result = !stretch ? "unresolved" : !withinLimit ? "size"
522
- : await this.#present(stretch.sources, held, request.timing) ? "fits" : "missing-bytes";
523
- probes.push({ end, limitBytes: maxBytes, bytes: stretch?.bytes ?? null, result });
549
+ : await this.#present(stretch.sources, held, request.timing, files) ? "fits" : "missing-bytes";
550
+ probes.push({ end, limitBytes: maxBytes, bytes: stretch?.bytes ?? null, result,
551
+ ...(sharing ? { bound: memory ? "memory" : "target", memoryBytes: stretch ? memoryBytes(stretch.sources) : null } : {}) });
524
552
  return result === "fits";
525
553
  };
526
554
  // Grows by doubling and then halves back: a handful of resolutions find
@@ -534,17 +562,17 @@ export class EncodeInputs {
534
562
  const covering = await at(serves);
535
563
  if (covering) { floor = serves; minimum = covering.bytes; }
536
564
  }
537
- const longest = async maxBytes => {
565
+ const longest = async (maxBytes, memory = false) => {
538
566
  let good = floor, bad = null, step = 1;
539
567
  while (bad === null) {
540
568
  const end = good + step;
541
569
  if (end > request.to) bad = request.to + 1;
542
- else if (await fits(end, maxBytes)) { good = end; step *= 2; }
570
+ else if (await fits(end, maxBytes, memory)) { good = end; step *= 2; }
543
571
  else bad = end;
544
572
  }
545
573
  while (bad - good > 1) {
546
574
  const middle = Math.floor((good + bad) / 2);
547
- if (await fits(middle, maxBytes)) good = middle; else bad = middle;
575
+ if (await fits(middle, maxBytes, memory)) good = middle; else bad = middle;
548
576
  }
549
577
  return good;
550
578
  };
@@ -554,24 +582,26 @@ export class EncodeInputs {
554
582
  let end = target === null || this.#heldRanges === null ? floor : await longest(target);
555
583
  const targetEnd = end;
556
584
  if (this.#requests.get(request.key) !== request) return undefined;
557
- this.#wanted.set(request.key, { wanted: stretches.get(end).bytes, required: minimum });
585
+ const minimumMemory = () => memoryBytes(stretches.get(floor).sources);
586
+ this.#wanted.set(request.key, { wanted: memoryBytes(stretches.get(end).sources), required: minimumMemory() });
558
587
  await this.#spent(request.timing, "budgetMs", () => this.#revise());
559
588
  if (this.#requests.get(request.key) !== request) return undefined;
560
589
  const capacity = this.#capacity();
561
- if (capacity !== null && minimum > capacity && this.#urgent(output, from)) {
562
- return { kind: "terminal", reason: "source-input-exceeds-memory-capacity", bytes: minimum, capacity };
590
+ const minimumCapacity = originalInputBytes(inMemory(stretches.get(floor).sources));
591
+ if (capacity !== null && minimumCapacity > capacity && this.#urgent(output, from)) {
592
+ return { kind: "terminal", reason: "source-input-exceeds-memory-capacity", bytes: minimumCapacity, capacity };
563
593
  }
564
- const roomNow = () => Math.min(capacity ?? Number.POSITIVE_INFINITY, this.#allowed - this.#held);
594
+ const roomNow = () => Math.min(capacity ?? Number.POSITIVE_INFINITY, this.#allowed - this.held());
565
595
  let room = roomNow();
566
596
  let result;
567
597
  for (;;) {
568
- if (minimum > room) {
598
+ if (minimumMemory() > room) {
569
599
  // Wait for memory with the least it can run with, not the larger stretch.
570
- this.#wanted.set(request.key, { wanted: minimum, required: minimum });
571
- return { kind: "needs-memory", bytes: minimum };
600
+ this.#wanted.set(request.key, { wanted: minimumMemory(), required: minimumMemory() });
601
+ return { kind: "needs-memory", bytes: minimumMemory() };
572
602
  }
573
- if (stretches.get(end).bytes > room) {
574
- end = await longest(Math.min(target ?? minimum, room));
603
+ if (memoryBytes(stretches.get(end).sources) > room) {
604
+ end = await longest(sharing ? room : Math.min(target ?? minimum, room), sharing);
575
605
  if (this.#requests.get(request.key) !== request) return undefined;
576
606
  }
577
607
  const chosen = stretches.get(end);
@@ -581,17 +611,26 @@ export class EncodeInputs {
581
611
  // it waits for the bytes as one read of everything used to. Field
582
612
  // 2026-10-12: a cold MKV's first piece was answered, its stretch read
583
613
  // found pieces missing, and the run was refused for good (torrent-tv/meta#178).
584
- if (this.#heldRanges !== null && !await this.#present(chosen.sources, held, request.timing)) {
614
+ if (this.#heldRanges !== null && !await this.#present(chosen.sources, held, request.timing, files)) {
585
615
  const [first] = chosen.sources;
586
616
  return { kind: "needs-bytes", sourceKey: first.sourceKey, fileIndex: first.fileIndex, ranges: first.input.ranges,
587
617
  // Every source of the stretch, so the bytes of any of them wake it.
588
618
  awaiting: chosen.sources.map(source => ({ sourceKey: source.sourceKey, fileIndex: source.fileIndex, ranges: source.input.ranges })) };
589
619
  }
590
620
  if (this.#requests.get(request.key) !== request) return undefined;
591
- this.#wanted.set(request.key, { wanted: chosen.bytes, required: chosen.bytes });
592
- result = await admitOriginalInput({ sources: chosen.sources, readRanges: this.#read, now: this.#now,
621
+ const needed = memoryBytes(chosen.sources);
622
+ this.#wanted.set(request.key, { wanted: needed, required: needed });
623
+ if (sharing) {
624
+ const reservation = { bytes: needed, minimumMemoryBytes: minimumMemory(), allowed: this.#allowed, held: this.held(), capacity, room: roomNow(), end };
625
+ const storage = this.#shared.acquire(inMemory(chosen.sources), roomNow(), { readRanges: this.#read, now: this.#now });
626
+ reservations.push({ ...reservation, admitted: storage !== null });
627
+ if (storage) {
628
+ this.#wanted.delete(request.key);
629
+ result = await admitOriginalInput({ sources: chosen.sources, storage, files, now: this.#now });
630
+ } else result = { kind: "needs-memory", bytes: needed };
631
+ } else result = await admitOriginalInput({ sources: chosen.sources, readRanges: this.#read, now: this.#now,
593
632
  reserve: async bytes => {
594
- const reservation = { bytes, allowed: this.#allowed, held: this.#held, capacity,
633
+ const reservation = { bytes, allowed: this.#allowed, held: this.held(), capacity,
595
634
  room: roomNow(), end };
596
635
  const release = this.#hold(request, bytes);
597
636
  reservations.push({ ...reservation, admitted: typeof release === "function" });
@@ -603,11 +642,12 @@ export class EncodeInputs {
603
642
  if (result.kind !== "needs-memory" || this.#requests.get(request.key) !== request || roomNow() >= room) break;
604
643
  room = roomNow();
605
644
  }
606
- if (result.kind === "needs-memory") this.#wanted.set(request.key, { wanted: minimum, required: minimum });
645
+ if (result.kind === "needs-memory") this.#wanted.set(request.key, { wanted: minimumMemory(), required: minimumMemory() });
607
646
  if (result.kind === "result") {
608
647
  this.#log(`encode input selection of ${addressOf(output)}: ${JSON.stringify({
609
648
  from, requestedTo: request.to, serves, floor, targetEnd, chosenEnd: end,
610
649
  targetBytes: target, minimumBytes: minimum, inputBytes: result.bytes,
650
+ ...(sharing ? { minimumMemoryBytes: reservations.at(-1)?.minimumMemoryBytes ?? null, reservedMemoryBytes: reservations.at(-1)?.bytes ?? null } : {}),
611
651
  copyBytesPerMs: sizing?.copyBytesPerMs ?? null, openSeconds: sizing?.openSeconds ?? null,
612
652
  openMeasured: sizing?.openMeasured ?? null, reservations, probes
613
653
  })}`);
@@ -616,13 +656,13 @@ export class EncodeInputs {
616
656
  // is the whole copy's, learned once it is done.
617
657
  const copied = result.whole ?? Promise.resolve(true);
618
658
  void copied.then(whole => {
619
- if (whole && result.copyMs > 0) keep(this.#copyRates, result.bytes / result.copyMs);
620
- if (whole) {
659
+ if (whole && result.copyMs > 0 && result.copiedBytes > 0) keep(this.#copyRates, result.copiedBytes / result.copyMs);
660
+ if (whole && result.copiedBytes + result.sharedBytes > 0) {
621
661
  try { this.#stretchCopied(output, from, end, result); }
622
662
  catch (error) { this.#log(`encode input #${from}..#${end} of ${addressOf(output)}: what was told of its copy failed: ${error?.message ?? error}`); }
623
663
  }
624
664
  this.#log(`encode input #${from}..#${end} of ${addressOf(output)}: ` + (whole
625
- ? `stretch copied, ${result.bytes} bytes in ${result.copyMs}ms of copying since it was admitted`
665
+ ? `stretch copied, ${result.bytes} bytes in ${result.copyMs}ms of copying since it was admitted; copied ${result.copiedBytes}, shared ${result.sharedBytes}, disk ${result.diskBytes} bytes`
626
666
  : result.copyFailed?.() ? "the stretch could not be copied: a piece it needs is no longer held"
627
667
  : "released before its stretch was copied"));
628
668
  });
@@ -631,12 +671,21 @@ export class EncodeInputs {
631
671
  }
632
672
 
633
673
  /** Whether every range of these sources is downloaded whole now. */
634
- async #present(sources, held, timing) {
674
+ async #present(sources, held, timing, files = new Map()) {
635
675
  for (const source of sources) {
676
+ if (files.has(source.fileIndex)) continue;
636
677
  const name = `${source.sourceKey}:${source.fileIndex}`;
637
678
  if (!held.has(name)) held.set(name, await this.#spent(timing, "heldMs", () => Promise.resolve().then(() => this.#heldRanges(source)).catch(() => null)) ?? []);
638
- const ranges = held.get(name);
639
- if (!source.input.ranges.every(([start, end]) => ranges.some(([first, last]) => first <= start && last >= end))) return false;
679
+ const ranges = [...held.get(name), ...this.#shared.available(source)].sort((a, b) => a[0] - b[0]);
680
+ if (!source.input.ranges.every(([start, end]) => {
681
+ let at = start;
682
+ for (const [first, last] of ranges) {
683
+ if (first > at) return false;
684
+ if (last >= at) at = last + 1;
685
+ if (at > end) return true;
686
+ }
687
+ return false;
688
+ })) return false;
640
689
  }
641
690
  return true;
642
691
  }
@@ -1220,7 +1220,11 @@ export class EncodeRuns {
1220
1220
  `ended at ${Date.now() - run.startedAt}ms` + describeTaken(reads.answers, run.startedAt, ended.firstNamedWallMs));
1221
1221
  }
1222
1222
  this.#noteOpen(session, run, ended);
1223
- admittedInput.release();
1223
+ void Promise.resolve(admittedInput.release()).then(() => {
1224
+ this.#host.logger.info(`encode input storage of run #${run.from}..#${run.to} on ${session.outputKey}: ` +
1225
+ `input ${admittedInput.bytes} bytes, copied ${admittedInput.copiedBytes ?? 0}, ` +
1226
+ `shared ${admittedInput.sharedBytes ?? 0}, disk ${admittedInput.diskBytes ?? 0} bytes`);
1227
+ });
1224
1228
  }
1225
1229
  this.#sayRunCpu(session, run);
1226
1230
  this.noteRunEnded(session, run, ended);
@@ -13,10 +13,10 @@ import { createHash } from "node:crypto";
13
13
  * (torrent-tv/meta#178). `whole` settles when everything is copied — true — or
14
14
  * when the copy failed or the input was released first — false.
15
15
  */
16
- export async function admitOriginalInput({ sources, reserve, readRanges, now = Date.now }) {
16
+ export async function admitOriginalInput({ sources, reserve, readRanges, now = Date.now, storage = null, files = new Map() }) {
17
17
  const bytes = originalInputBytes(sources);
18
18
  if (!Number.isSafeInteger(bytes) || bytes <= 0) throw new TypeError("Original input requires complete finite byte ranges.");
19
- const releaseBudget = await reserve(bytes);
19
+ const releaseBudget = storage ? storage.release : await reserve(bytes);
20
20
  if (typeof releaseBudget !== "function") return releaseBudget?.kind === "terminal" ? releaseBudget : { kind: "needs-memory", bytes };
21
21
  const held = new Map();
22
22
  const hash = createHash("sha256");
@@ -24,10 +24,13 @@ export async function admitOriginalInput({ sources, reserve, readRanges, now = D
24
24
  // How long the copy itself took, apart from any wait for memory or bytes
25
25
  // before it: the figure a run's stretch is sized from, read where the copy is.
26
26
  let copyMs = 0;
27
+ let diskBytes = 0;
27
28
  // Per source, the range copied after the answer: its position in the
28
29
  // source's ranges, and the copy's promise once it has begun.
29
30
  const later = [];
30
31
  const read = async (source, ranges) => {
32
+ if (files.has(source.fileIndex)) return ranges.map(() => true);
33
+ if (storage) return await storage.copy(source, ranges) ? ranges.map(() => true) : null;
31
34
  const startedAt = now();
32
35
  const buffers = await readRanges(source, ranges, bytes);
33
36
  copyMs += Math.max(0, now() - startedAt);
@@ -43,7 +46,7 @@ export async function admitOriginalInput({ sources, reserve, readRanges, now = D
43
46
  const { ranges } = source.input;
44
47
  const largest = ranges.reduce((best, range, index) =>
45
48
  range[1] - range[0] > ranges[best][1] - ranges[best][0] ? index : best, 0);
46
- const deferred = ranges.length > 1 ? largest : -1;
49
+ const deferred = !files.has(source.fileIndex) && ranges.length > 1 ? largest : -1;
47
50
  const first = ranges.filter((_, index) => index !== deferred);
48
51
  const buffers = await read(source, first);
49
52
  if (buffers === null) return { kind: "needs-bytes", sourceKey: source.sourceKey, fileIndex: source.fileIndex, ranges };
@@ -59,6 +62,8 @@ export async function admitOriginalInput({ sources, reserve, readRanges, now = D
59
62
  if (deferred >= 0) later.push({ entry, position: deferred });
60
63
  }
61
64
  let released = false;
65
+ let runReleased = false, owners = 1, finishClose;
66
+ const closed = new Promise(resolve => { finishClose = resolve; });
62
67
  let failed = false;
63
68
  retained = true;
64
69
  // ONE READ FOR THE STRETCH. Copied a torrent piece per read (2.97.12), a
@@ -102,19 +107,46 @@ export async function admitOriginalInput({ sources, reserve, readRanges, now = D
102
107
  // When FFmpeg's reads waited for a stretch still being copied.
103
108
  /** @type {{ from: number, to: number | null }[]} */
104
109
  const waits = [];
105
- const answer = (entry, position, start, stop) => {
110
+ const answer = (entry, position, start, stop, answeringHttp = false) => {
106
111
  const now = Date.now();
107
112
  reads.count += 1;
108
113
  reads.bytes += stop - start + 1;
109
114
  reads.firstAt ||= now;
110
115
  reads.lastAt = now;
111
116
  const [from] = entry.source.input.ranges[position];
117
+ const file = files.get(entry.source.fileIndex);
118
+ if (file) {
119
+ // Packet learning reads only an already-owned memory copy; disk has no
120
+ // such copy. Its ordinary container reader already reads this file.
121
+ if (!answeringHttp) return null;
122
+ const stream = file.stream(start, stop, bytes => { diskBytes += bytes; });
123
+ return stream ? { length: file.length, end: stop, size: stop - start + 1, stream } : null;
124
+ }
125
+ if (storage) {
126
+ const held = storage.slices(entry.source, start, stop, answeringHttp);
127
+ if (!held) return null;
128
+ return { length: entry.source.input.fileLength, end: stop, size: stop - start + 1,
129
+ ...(answeringHttp && held.buffers.length > 1 ? { chunks: held.buffers }
130
+ : { bytes: held.buffers.length === 1 ? held.buffers[0] : Buffer.concat(held.buffers) }),
131
+ release: held.release };
132
+ }
112
133
  return { length: entry.source.input.fileLength, end: stop, bytes: entry.buffers[position].subarray(start - from, stop - from + 1) };
113
134
  };
135
+ const drop = () => {
136
+ owners--;
137
+ if (owners || released) return;
138
+ released = true;
139
+ held.clear();
140
+ releaseBudget();
141
+ void Promise.all([whole, ...[...files.values()].map(file => file.release())]).then(finishClose);
142
+ };
114
143
  return {
115
144
  kind: "result", original: true, sources, bytes, fingerprint: hash.digest("hex"),
116
145
  // What the copy took so far; final once `whole` has settled.
117
- get copyMs() { return copyMs; },
146
+ get copyMs() { return storage ? storage.stats.copyMs : copyMs; },
147
+ get copiedBytes() { return storage ? storage.stats.copiedBytes : bytes; },
148
+ get sharedBytes() { return storage ? storage.stats.sharedBytes : 0; },
149
+ get diskBytes() { return diskBytes; },
118
150
  whole,
119
151
  // Bytes of the stretches still to be copied when the answer was given.
120
152
  deferredBytes: later.reduce((sum, job) => {
@@ -126,6 +158,13 @@ export async function admitOriginalInput({ sources, reserve, readRanges, now = D
126
158
  // not where each track stops. The final piece is judged by the input
127
159
  // reaching the end of the file instead (`piece-completeness.js`).
128
160
  tracks: sources.flatMap(source => (source.input.selections ?? []).map(selection => ({ track: selection.track }))),
161
+ heldUntil(fileIndex, start) {
162
+ if (released) return null;
163
+ const entry = held.get(fileIndex);
164
+ if (!entry || files.has(fileIndex)) return null;
165
+ const range = entry.source.input.ranges.find(([first, last]) => first <= start && last >= start);
166
+ return range ? Math.min(range[1] + 1, storage?.endOf(entry.source, start) ?? range[1] + 1) : null;
167
+ },
129
168
  /**
130
169
  * The answer to a read from `start`, or null when it lies outside what
131
170
  * the run holds. Inside a stretch still being copied the answer is a
@@ -141,12 +180,12 @@ export async function admitOriginalInput({ sources, reserve, readRanges, now = D
141
180
  const position = ranges.findIndex(([from, to]) => from <= start && (partial ? start : stop) <= to);
142
181
  if (position < 0) return null;
143
182
  stop = Math.min(stop, ranges[position][1]);
144
- if (entry.buffers[position] !== null) return answer(entry, position, start, stop);
183
+ if (entry.buffers[position] !== null) return answer(entry, position, start, stop, partial);
145
184
  const wait = { from: Date.now(), to: null };
146
185
  waits.push(wait);
147
186
  return copied(entry, position).then(ok => {
148
187
  wait.to = Date.now();
149
- return ok && !released ? answer(entry, position, start, stop) : null;
188
+ return ok && !released ? answer(entry, position, start, stop, partial) : null;
150
189
  });
151
190
  },
152
191
  /**
@@ -163,11 +202,16 @@ export async function admitOriginalInput({ sources, reserve, readRanges, now = D
163
202
  noteAnswer(answer) {
164
203
  if (!released) answers.push({ ...answer, at: Date.now() });
165
204
  },
205
+ /** Packet learning can finish after the FFmpeg run that supplied it. */
206
+ retain() {
207
+ if (released || runReleased) return null;
208
+ owners++;
209
+ let dropped = false;
210
+ return () => { if (!dropped) { dropped = true; drop(); } };
211
+ },
166
212
  release() {
167
- if (released) return;
168
- released = true;
169
- held.clear();
170
- releaseBudget();
213
+ if (!runReleased) { runReleased = true; drop(); }
214
+ return closed;
171
215
  }
172
216
  };
173
217
  } finally {
@@ -0,0 +1,217 @@
1
+ /** Copied, admitted file ranges shared by live inputs and HTTP responses. */
2
+ export class OriginalInputStore {
3
+ #files = new Map();
4
+ #owners = new Map();
5
+ #parts = new Set();
6
+ #changed;
7
+
8
+ constructor({ changed = () => {} } = {}) { this.#changed = changed; }
9
+
10
+ get bytes() { return [...this.#parts].reduce((sum, part) => sum + (part.charged ? sizeOf(part) : 0), 0); }
11
+ get required() { return [...this.#parts].reduce((sum, part) => sum + (part.charged && (part.refs || part.loading) ? sizeOf(part) : 0), 0); }
12
+
13
+ #file(source) {
14
+ const key = JSON.stringify([source.sourceKey, source.fileIndex]);
15
+ if (!this.#files.has(key)) this.#files.set(key, { key, parts: [], owners: new Set() });
16
+ return this.#files.get(key);
17
+ }
18
+
19
+ /** Keep common parts across seeks while an output of this file exists. */
20
+ retain(owner, sources) {
21
+ const files = this.#owners.get(owner) ?? new Set();
22
+ for (const source of sources) {
23
+ const file = this.#file(source);
24
+ file.owners.add(owner);
25
+ files.add(file);
26
+ }
27
+ this.#owners.set(owner, files);
28
+ }
29
+
30
+ forget(owner) {
31
+ for (const file of this.#owners.get(owner) ?? []) {
32
+ file.owners.delete(owner);
33
+ for (const part of [...file.parts]) this.#discardUnused(part);
34
+ this.#prune(file);
35
+ }
36
+ this.#owners.delete(owner);
37
+ }
38
+
39
+ /** Unused common bytes are wanted, never required, and can leave for room. */
40
+ trim(limit) {
41
+ let bytes = this.bytes;
42
+ for (const part of this.#parts) {
43
+ if (bytes <= limit) break;
44
+ if (part.refs || part.loading) continue;
45
+ bytes -= part.charged ? sizeOf(part) : 0;
46
+ this.#discard(part);
47
+ }
48
+ }
49
+
50
+ #prune(file) {
51
+ if (!file.parts.length && !file.owners.size) this.#files.delete(file.key);
52
+ }
53
+
54
+ #discard(part) {
55
+ part.charged = false;
56
+ part.buffer = null;
57
+ part.ensure = null;
58
+ this.#parts.delete(part);
59
+ const position = part.file.parts.indexOf(part);
60
+ if (position >= 0) part.file.parts.splice(position, 1);
61
+ this.#prune(part.file);
62
+ this.#changed();
63
+ }
64
+
65
+ #discardUnused(part) {
66
+ if (part.refs || part.loading) return;
67
+ if (!part.failed && part.buffer && part.common && part.file.owners.size) return;
68
+ this.#discard(part);
69
+ }
70
+
71
+ #unpin(parts) {
72
+ let released = false;
73
+ return () => {
74
+ if (released) return;
75
+ released = true;
76
+ for (const part of parts) { part.refs--; this.#discardUnused(part); }
77
+ };
78
+ }
79
+
80
+ /** Protect existing bytes during an asynchronous budget revision. */
81
+ protect(sources) {
82
+ const parts = new Set();
83
+ for (const source of sources) {
84
+ for (const part of this.#file(source).parts) {
85
+ if (source.input.ranges.some(([start, end]) => start <= part.end && end >= part.start)) parts.add(part);
86
+ }
87
+ }
88
+ for (const part of parts) part.refs++;
89
+ return this.#unpin(parts);
90
+ }
91
+
92
+ available(source) {
93
+ const file = this.#files.get(JSON.stringify([source.sourceKey, source.fileIndex]));
94
+ return (file?.parts ?? []).filter(part => !part.failed).map(part => [part.start, part.end]);
95
+ }
96
+
97
+ #missing(source) {
98
+ const file = this.#file(source);
99
+ const missing = [];
100
+ for (const [start, end] of source.input.ranges) {
101
+ let at = start;
102
+ for (const part of file.parts) {
103
+ if (part.end < at || part.failed) continue;
104
+ if (part.start > end) break;
105
+ if (at < part.start) missing.push([at, Math.min(end, part.start - 1)]);
106
+ at = Math.max(at, part.end + 1);
107
+ if (at > end) break;
108
+ }
109
+ if (at <= end) missing.push([at, end]);
110
+ }
111
+ return missing;
112
+ }
113
+
114
+ bytesNeeded(sources) { return sources.reduce((sum, source) => sum + this.#missing(source).reduce((n, [start, end]) => n + end - start + 1, 0), 0); }
115
+
116
+ /** Reserve missing ranges synchronously; a second admission sees them pending. */
117
+ acquire(sources, room, { readRanges, now = Date.now }) {
118
+ const additionalBytes = this.bytesNeeded(sources);
119
+ if (additionalBytes > room) return null;
120
+ const parts = new Set();
121
+ const owned = new Set();
122
+ const stats = { copiedBytes: 0, sharedBytes: 0, copyMs: 0 };
123
+ for (const source of sources) {
124
+ const file = this.#file(source);
125
+ for (const [start, end] of this.#missing(source)) {
126
+ // A common prefix must never retain the rest of a large media buffer.
127
+ const cuts = new Set([start, end + 1]);
128
+ for (const [first, last] of source.input.commonRanges ?? []) {
129
+ if (first > start && first <= end) cuts.add(first);
130
+ if (last >= start && last < end) cuts.add(last + 1);
131
+ }
132
+ const offsets = [...cuts].sort((a, b) => a - b);
133
+ for (let i = 1; i < offsets.length; i++) {
134
+ const first = offsets[i - 1], last = offsets[i] - 1;
135
+ const part = { file, source, start: first, end: last, refs: 0, charged: true,
136
+ common: (source.input.commonRanges ?? []).some(([a, b]) => a <= first && b >= last),
137
+ buffer: null, loading: false, failed: false, promise: null, ensure: null };
138
+ file.parts.push(part);
139
+ this.#parts.add(part);
140
+ owned.add(part);
141
+ }
142
+ }
143
+ file.parts.sort((a, b) => a.start - b.start);
144
+ for (const part of file.parts) {
145
+ for (const [start, end] of source.input.ranges) {
146
+ const bytes = Math.max(0, Math.min(end, part.end) - Math.max(start, part.start) + 1);
147
+ if (!bytes) continue;
148
+ parts.add(part);
149
+ if (!owned.has(part)) stats.sharedBytes += bytes;
150
+ }
151
+ }
152
+ }
153
+ for (const part of parts) part.refs++;
154
+ let released = false;
155
+ const unpin = this.#unpin(parts);
156
+ const copy = async (source, ranges) => {
157
+ const selected = [...parts].filter(part => part.file.key === JSON.stringify([source.sourceKey, source.fileIndex]) &&
158
+ ranges.some(([start, end]) => start <= part.end && end >= part.start));
159
+ const fresh = selected.filter(part => owned.has(part) && !part.promise);
160
+ if (fresh.length) {
161
+ for (const part of fresh) part.loading = true;
162
+ const startedAt = now();
163
+ // Defer the read until every part names the same pending operation.
164
+ const operation = Promise.resolve().then(() => readRanges(source, fresh.map(part => [part.start, part.end]), additionalBytes))
165
+ .then(buffers => {
166
+ if (buffers === null) return false;
167
+ if (!Array.isArray(buffers) || buffers.length !== fresh.length || buffers.some((buffer, i) => !Buffer.isBuffer(buffer) || buffer.length !== sizeOf(fresh[i]))) {
168
+ throw new Error("Original input reader returned incomplete ranges.");
169
+ }
170
+ for (let i = 0; i < fresh.length; i++) fresh[i].buffer = buffers[i];
171
+ stats.copiedBytes += fresh.reduce((sum, part) => sum + sizeOf(part), 0);
172
+ return true;
173
+ }).catch(error => { for (const part of fresh) part.error = error; return false; }).then(ok => {
174
+ stats.copyMs += Math.max(0, now() - startedAt);
175
+ for (const part of fresh) {
176
+ part.loading = false;
177
+ if (!ok) { part.failed = true; this.#discard(part); }
178
+ else this.#discardUnused(part);
179
+ }
180
+ return ok;
181
+ });
182
+ for (const part of fresh) { part.promise = operation; part.ensure = null; }
183
+ }
184
+ for (const part of selected) if (!part.promise) part.ensure();
185
+ const answers = await Promise.all(selected.map(part => part.promise));
186
+ const failure = selected.find(part => part.error);
187
+ if (failure) throw failure.error;
188
+ return !released && answers.every(Boolean);
189
+ };
190
+ for (const part of owned) part.ensure = () => copy(part.source, [[part.start, part.end]]);
191
+ return {
192
+ stats, additionalBytes,
193
+ copy,
194
+ endOf(source, start) {
195
+ if (released) return null;
196
+ const part = [...parts].find(part => part.file.key === JSON.stringify([source.sourceKey, source.fileIndex]) &&
197
+ part.buffer && part.start <= start && part.end >= start);
198
+ return part ? part.end + 1 : null;
199
+ },
200
+ slices(source, start, end, pin = false) {
201
+ if (released) return null;
202
+ const selected = [...parts].filter(part => part.file.key === JSON.stringify([source.sourceKey, source.fileIndex]) && part.start <= end && part.end >= start)
203
+ .sort((a, b) => a.start - b.start);
204
+ if (selected.some(part => !part.buffer || part.failed)) return null;
205
+ const buffers = selected.map(part => part.buffer.subarray(Math.max(start, part.start) - part.start, Math.min(end, part.end) - part.start + 1));
206
+ if (buffers.reduce((sum, buffer) => sum + buffer.length, 0) !== end - start + 1) return null;
207
+ if (pin) for (const part of selected) part.refs++;
208
+ return { buffers, release: pin ? this.releaseParts(selected) : () => {} };
209
+ },
210
+ // Bound separately because slices' `this` is the lease, not the store.
211
+ releaseParts: selected => this.#unpin(selected),
212
+ release() { if (!released) { released = true; unpin(); } }
213
+ };
214
+ }
215
+ }
216
+
217
+ const sizeOf = part => part.end - part.start + 1;
@@ -275,9 +275,10 @@ export class AviContainer extends Container {
275
275
  // media time: every stream's packets in file order, from the first.
276
276
  const search = index.mediaRanges({ from: 0, to: FFMPEG_STREAM_SEARCH_SECONDS, picture: null,
277
277
  streams: requested.map(track => track.trackNumber) });
278
- const ranges = [[0, movi.start + 3], ...index.indexRanges, ...index.firstPackets(), ...search,
279
- ...index.mediaRanges({ from: interval.from, to: interval.to, picture: picture?.trackNumber ?? null,
280
- streams: requested.map(track => track.trackNumber) })].map(padded);
278
+ const commonRanges = [[0, movi.start + 3], ...index.indexRanges, ...index.firstPackets(), ...search].map(padded);
279
+ const intervalRanges = index.mediaRanges({ from: interval.from, to: interval.to, picture: picture?.trackNumber ?? null,
280
+ streams: requested.map(track => track.trackNumber) }).map(padded);
281
+ const ranges = [...commonRanges, ...intervalRanges];
281
282
  ranges.sort((left, right) => left[0] - right[0]);
282
283
  const union = [];
283
284
  for (const range of ranges) {
@@ -285,7 +286,7 @@ export class AviContainer extends Container {
285
286
  if (previous && range[0] <= previous[1] + 1) previous[1] = Math.max(previous[1], range[1]);
286
287
  else union.push([...range]);
287
288
  }
288
- return { kind: "result", from: interval.from, to: interval.to, ranges: union, fileLength: this.fileSize,
289
+ return { kind: "result", from: interval.from, to: interval.to, ranges: union, commonRanges, intervalRanges, fileLength: this.fileSize,
289
290
  streamSearchSeconds: FFMPEG_STREAM_SEARCH_SECONDS };
290
291
  }
291
292
 
@@ -761,8 +761,8 @@ export class MatroskaContainer extends Container {
761
761
  // media time and stops on the first packet past it, which lies before the
762
762
  // first Cue later than that time; the padding holds its read-ahead.
763
763
  const searchEnd = points[firstAt(points[0].seconds + FFMPEG_STREAM_SEARCH_SECONDS, true)];
764
- const ranges = [[0, paddedEnd((searchEnd ? searchEnd.at : layout.segmentEnd) - 1)],
765
- [points[first].at, paddedEnd(mediaEnd)], [layout.cuesAt, paddedEnd(cuesEnd - 1)],
764
+ const commonRanges = [[0, paddedEnd((searchEnd ? searchEnd.at : layout.segmentEnd) - 1)],
765
+ [layout.cuesAt, paddedEnd(cuesEnd - 1)],
766
766
  [Math.max(0, this.fileSize - requestBytes), this.fileSize - 1]];
767
767
  // SeekHead may name top-level declarations after media clusters. Preserve
768
768
  // their original bytes instead of reconstructing declarations for FFmpeg.
@@ -770,8 +770,10 @@ export class MatroskaContainer extends Container {
770
770
  if (at === null || at < layout.firstClusterAt) continue;
771
771
  const end = await elementEnd(at);
772
772
  if (end === null) return { kind: "terminal", reason: "source-declarations-are-unbounded" };
773
- ranges.push([at, paddedEnd(end - 1)]);
773
+ commonRanges.push([at, paddedEnd(end - 1)]);
774
774
  }
775
+ const intervalRanges = [[points[first].at, paddedEnd(mediaEnd)]];
776
+ const ranges = [...commonRanges, ...intervalRanges];
775
777
  ranges.sort((left, right) => left[0] - right[0]);
776
778
  const union = [];
777
779
  for (const range of ranges) {
@@ -779,7 +781,7 @@ export class MatroskaContainer extends Container {
779
781
  if (previous && range[0] <= previous[1] + 1) previous[1] = Math.max(previous[1], range[1]);
780
782
  else union.push([...range]);
781
783
  }
782
- return { kind: "result", from: interval.from, to: interval.to, ranges: union, fileLength: this.fileSize,
784
+ return { kind: "result", from: interval.from, to: interval.to, ranges: union, commonRanges, intervalRanges, fileLength: this.fileSize,
783
785
  streamSearchSeconds: FFMPEG_STREAM_SEARCH_SECONDS };
784
786
  }
785
787
 
@@ -230,6 +230,7 @@ export function wireOutputs({
230
230
  readMetadataActivity = null,
231
231
  sourceInputsFor = null,
232
232
  readEncodeRanges = null,
233
+ openEncodeWholeFile = null,
233
234
  stretchCopied = () => {},
234
235
  readEncodeHeldRanges = null,
235
236
  encodeInputs = null,
@@ -1117,6 +1118,7 @@ export function wireOutputs({
1117
1118
  parts.encodeInputs = encodeInputs ?? (typeof resolveEncodeInput === "function" ? new EncodeInputs({
1118
1119
  resolve: resolveEncodeInput,
1119
1120
  readRanges: readEncodeRanges,
1121
+ openWholeFile: openEncodeWholeFile,
1120
1122
  heldRanges: readEncodeHeldRanges,
1121
1123
  stretchCopied,
1122
1124
  reviseBudget: () => parts.machineBudget.revise("memory", "admitted encode inputs"),
@@ -54,6 +54,8 @@ export class CompletedFiles {
54
54
 
55
55
  /** Assemblies in flight, so two passes cannot write one file at once. @type {Set<string>} */
56
56
  #writing = new Set();
57
+ #readers = new Map();
58
+ #removeAfterRead = new Set();
57
59
 
58
60
  /**
59
61
  * What this store may hold, handed to it by the owner of the disk.
@@ -140,6 +142,7 @@ export class CompletedFiles {
140
142
  */
141
143
  find(infoHash, fileIndex) {
142
144
  const key = this.#keyOf(infoHash, fileIndex);
145
+ if (this.#removeAfterRead.has(key)) return null;
143
146
  const file = this.#held.get(key) ?? null;
144
147
  if (file) {
145
148
  // WHEN IT WAS LAST WANTED, which is the order it leaves in. A whole file
@@ -150,6 +153,26 @@ export class CompletedFiles {
150
153
  return file;
151
154
  }
152
155
 
156
+ /** Pin a completed file against eviction until all its offset reads finish. */
157
+ acquire(infoHash, fileIndex) {
158
+ const key = this.#keyOf(infoHash, fileIndex);
159
+ const file = this.find(infoHash, fileIndex);
160
+ if (!file) return null;
161
+ this.#readers.set(key, (this.#readers.get(key) ?? 0) + 1);
162
+ let released = false;
163
+ return { ...file, release: async () => {
164
+ if (released) return { bytes: this.bytes, removed: 0 };
165
+ released = true;
166
+ const remaining = this.#readers.get(key) - 1;
167
+ if (remaining) this.#readers.set(key, remaining);
168
+ else {
169
+ this.#readers.delete(key);
170
+ if (this.#removeAfterRead.delete(key)) await this.#remove(key);
171
+ }
172
+ return this.allow(this.#allowanceBytes);
173
+ } };
174
+ }
175
+
153
176
  /**
154
177
  * Say how much disk these files may take between them.
155
178
  *
@@ -204,7 +227,7 @@ export class CompletedFiles {
204
227
  let oldest = null;
205
228
  let oldestAt = Number.POSITIVE_INFINITY;
206
229
  for (const key of this.#held.keys()) {
207
- if (this.#writing.has(key)) {
230
+ if (this.#writing.has(key) || this.#readers.has(key)) {
208
231
  continue;
209
232
  }
210
233
  const at = this.#lastRead.get(key) ?? 0;
@@ -223,6 +246,7 @@ export class CompletedFiles {
223
246
  * @returns {Promise<void>}
224
247
  */
225
248
  async #remove(key) {
249
+ if (this.#readers.has(key)) { this.#removeAfterRead.add(key); return; }
226
250
  const file = this.#held.get(key);
227
251
  this.#held.delete(key);
228
252
  this.#lastRead.delete(key);
@@ -400,11 +424,11 @@ export class CompletedFiles {
400
424
  async forget(infoHash) {
401
425
  for (const key of [...this.#held.keys()]) {
402
426
  if (key.startsWith(`${infoHash}/`)) {
403
- this.#held.delete(key);
427
+ await this.#remove(key);
404
428
  }
405
429
  }
406
- await fs.rm(path.join(this.#root, String(infoHash)), { ...REMOVAL, recursive: true }).catch(
407
- () => undefined
408
- );
430
+ if (![...this.#held.keys()].some(key => key.startsWith(`${infoHash}/`))) {
431
+ await fs.rm(path.join(this.#root, String(infoHash)), { ...REMOVAL, recursive: true }).catch(() => undefined);
432
+ }
409
433
  }
410
434
  }
@@ -147,6 +147,7 @@ export function wireMachineBudget({
147
147
  name: "whole files",
148
148
  resource: filesOn,
149
149
  held: () => wholeFiles.held?.() ?? 0,
150
+ required: () => wholeFiles.required?.() ?? 0,
150
151
  // A whole file is worth keeping while anybody may read it again, and
151
152
  // there is no smaller honest answer than "all of them" — like the rest,
152
153
  // it is cut in proportion.
@@ -21,6 +21,7 @@ import { fileURLToPath } from "node:url";
21
21
  import { logger, writeAlreadyDecided } from "../../../utils/logger.js";
22
22
  import { createCaller, createReceiveStream, workerReplyError } from "./channel.js";
23
23
  import { Command, Event } from "./protocol.js";
24
+ import { openCompletedInputFile } from "./completed-input-file.js";
24
25
 
25
26
  const WORKER_URL = new URL("./worker.js", import.meta.url);
26
27
  /**
@@ -106,6 +107,29 @@ export class TorrentWorkerClient {
106
107
  * @type {Map<string, { path: string, length: number, name: string }>}
107
108
  */
108
109
  wholeFiles = new Map();
110
+ #wholeInputPins = new Map();
111
+
112
+ get wholeFileRequiredBytes() {
113
+ const files = new Map();
114
+ for (const file of this.#wholeInputPins.values()) files.set(JSON.stringify([file.sourceKey, file.fileIndex]), file.length);
115
+ return [...files.values()].reduce((sum, length) => sum + length, 0);
116
+ }
117
+
118
+ /** No source is opened or revived: only the worker's completed-file store answers. */
119
+ async acquireWholeFile({ sourceKey, fileIndex }) {
120
+ if (!this.#wholeFileOf(sourceKey, fileIndex)) return null;
121
+ const file = await this.#caller.call(Command.ACQUIRE_WHOLE_FILE, { sourceKey, fileIndex });
122
+ if (!file) {
123
+ this.wholeFiles.delete(`${String(sourceKey).slice("torrent:".length).toLowerCase()}/${fileIndex}`);
124
+ return null;
125
+ }
126
+ this.#wholeInputPins.set(file.token, { sourceKey, fileIndex, length: file.length });
127
+ this.wholeFileBytes = file.heldBytes;
128
+ return openCompletedInputFile(file, async () => {
129
+ try { this.wholeFileBytes = await this.#caller.call(Command.RELEASE_WHOLE_FILE, { token: file.token }); }
130
+ finally { this.#wholeInputPins.delete(file.token); }
131
+ });
132
+ }
109
133
 
110
134
  /** @type {(event: { sourceKey: string, fileIndex: number, trackIndex: number, cues: object[], language: string }) => void} */
111
135
  /** @type {(event: { sourceKey: string, fileIndexes: number[] }) => void} */
@@ -0,0 +1,50 @@
1
+ import { open } from "node:fs/promises";
2
+ import { Readable } from "node:stream";
3
+
4
+ /** Open one pinned completed file; every HTTP response owns a separate read. */
5
+ export async function openCompletedInputFile(file, release) {
6
+ let handle;
7
+ try {
8
+ handle = await open(file.path, "r");
9
+ if ((await handle.stat()).size !== file.length) throw new Error("Completed file length changed.");
10
+ } catch {
11
+ await handle?.close();
12
+ await release();
13
+ return null;
14
+ }
15
+ let references = 1, released = false, closing = false;
16
+ let finish;
17
+ const closed = new Promise(resolve => { finish = resolve; });
18
+ const drop = () => {
19
+ references--;
20
+ if (references || closing) return;
21
+ closing = true;
22
+ void handle.close().finally(release).then(() => finish(), error => finish(error));
23
+ };
24
+ return {
25
+ length: file.length, closed,
26
+ stream(start, end, onRead) {
27
+ if (released || !Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start < 0 || end < start || end >= file.length) return null;
28
+ references++;
29
+ let dropped = false;
30
+ const done = () => { if (!dropped) { dropped = true; drop(); } };
31
+ const stream = Readable.from((async function* () {
32
+ for (let at = start; at <= end;) {
33
+ const buffer = Buffer.allocUnsafe(Math.min(64 * 1024, end - at + 1));
34
+ const { bytesRead } = await handle.read(buffer, 0, buffer.length, at);
35
+ if (!bytesRead) throw new Error("Completed file ended inside an admitted range.");
36
+ at += bytesRead;
37
+ onRead(bytesRead);
38
+ yield buffer.subarray(0, bytesRead);
39
+ }
40
+ })(), { objectMode: false, highWaterMark: 64 * 1024 });
41
+ stream.once("end", done);
42
+ stream.once("close", done);
43
+ return stream;
44
+ },
45
+ release() {
46
+ if (!released) { released = true; drop(); }
47
+ return closed;
48
+ }
49
+ };
50
+ }
@@ -122,6 +122,10 @@ export class WorkerTorrentPool {
122
122
  return this.#client.wholeFiles;
123
123
  }
124
124
 
125
+ get wholeFileRequiredBytes() { return this.#client.wholeFileRequiredBytes ?? 0; }
126
+
127
+ acquireWholeFile(source) { return this.#client.acquireWholeFile(source); }
128
+
125
129
  allowWholeFileBytes(bytes) {
126
130
  return this.#client.allowWholeFileBytes(bytes);
127
131
  }
@@ -94,6 +94,9 @@ export const Command = {
94
94
  READ_HELD: "read-held",
95
95
  /** Acquire all source pieces of an admitted input before copying its ranges. */
96
96
  READ_HELD_RANGES: "read-held-ranges",
97
+ /** Pin a completed file; only a matching release permits its eviction. */
98
+ ACQUIRE_WHOLE_FILE: "acquire-whole-file",
99
+ RELEASE_WHOLE_FILE: "release-whole-file",
97
100
 
98
101
  /** Shut the client down, optionally deleting downloaded data. */
99
102
  /**
@@ -595,6 +595,22 @@ async function runCommand(command, params, id) {
595
595
  return memoryClaim();
596
596
  }
597
597
 
598
+ case Command.ACQUIRE_WHOLE_FILE: {
599
+ const hash = String(params.sourceKey).startsWith("torrent:") ? params.sourceKey.slice("torrent:".length).toLowerCase() : null;
600
+ const lease = hash === null ? null : completedFiles.acquire(hash, params.fileIndex);
601
+ if (!lease) return null;
602
+ wholeFileReaders.set(id, lease);
603
+ return { token: id, path: lease.path, length: lease.length, name: lease.name, heldBytes: completedFiles.bytes };
604
+ }
605
+
606
+ case Command.RELEASE_WHOLE_FILE: {
607
+ const lease = wholeFileReaders.get(params.token);
608
+ wholeFileReaders.delete(params.token);
609
+ const after = await lease?.release();
610
+ if (after?.removed) holdingsChanged();
611
+ return completedFiles.bytes;
612
+ }
613
+
598
614
  case Command.WHOLE_FILES_ALLOWANCE: {
599
615
  // Whole files are held on this thread and the disk has one owner, on the
600
616
  // other. What arrives is their share; what goes back is what they hold
@@ -980,6 +996,7 @@ function holdingsChanged() {
980
996
  * thread writes them, the main thread serves them without asking anybody.
981
997
  */
982
998
  const completedFiles = new CompletedFiles({ root: completedFilesRoot() });
999
+ const wholeFileReaders = new Map();
983
1000
  const wholeSources = createWholeSources({ find: (infoHash, fileIndex) => completedFiles.find(infoHash, fileIndex) });
984
1001
 
985
1002
  /** Adopted files and newly completed files state the same availability event. */