@torrent-tv/proxy 2.64.9 → 2.65.0

package/services/sidecar-files.js

@@ -0,0 +1,352 @@
+/**
+ * Which files of a torrent belong to one video file as its sound or its
+ * subtitles.
+ *
+ * Releases often ship a dub or a subtitle set as SEPARATE FILES beside the
+ * picture — `Rus Sound/<name>.mka`, `Sub/[group]/<name>.ass` — and a viewer who
+ * cannot reach them is watching the release without half of what it carries.
+ * This module answers only which file goes with which; what is inside a file is
+ * the container layer's business (`services/container/`), and what a viewer is
+ * shown is composed in the browser, where the locale is known.
+ *
+ * Nothing here reads bytes, waits on the swarm or knows about ffmpeg. It is a
+ * function of the torrent's own list of names, which is why it can be tested
+ * outright.
+ *
+ * A note on the axes this fits into. A file beside the video is NOT a third kind
+ * of track: `<name>.mka` is a Matroska container holding an audio track, and
+ * `MatroskaContainer` reads it exactly as it reads the picture's own tracks. So
+ * "external" is not a type — it is only the answer to WHERE a track lives, and
+ * that answer belongs here rather than in `tracks/`, which describes what a
+ * container declares and must stay free of torrent knowledge.
+ */
+
+/**
+ * Containers and elementary streams that can carry a soundtrack on their own.
+ *
+ * Two groups, and the difference matters to the caller: `.mka` and `.m4a` have a
+ * track table that `ContainerFactory` can read, so their language, title and
+ * flags are available; the rest are raw elementary streams that carry exactly
+ * one track and declare nothing about it.
+ */
+export const AUDIO_SIDECAR_EXTENSIONS = new Set([
+  ".mka",
+  ".m4a",
+  ".aac",
+  ".ac3",
+  ".eac3",
+  ".dts",
+  ".dtshd",
+  ".flac",
+  ".mp3",
+  ".mp2",
+  ".ogg",
+  ".oga",
+  ".opus",
+  ".wav",
+  ".thd",
+  ".mlp",
+  ".wma"
+]);
+
+/**
+ * Subtitle files. Image-based ones (`.sup`, `.sub`, `.idx`) are listed because
+ * they exist and must be RECOGNISED — a file we cannot show is still not a
+ * soundtrack — but this module does not decide what is offerable; that is the
+ * subtitle track's own `isTextBased()`.
+ */
+export const SUBTITLE_SIDECAR_EXTENSIONS = new Set([
+  ".srt",
+  ".ass",
+  ".ssa",
+  ".vtt",
+  ".webvtt",
+  ".sub",
+  ".sup",
+  ".idx",
+  ".ttml",
+  ".smi",
+  ".txt"
+]);
+
+/**
+ * Containers that carry a picture.
+ *
+ * Here to COUNT them, not to decide what plays: whether a file is playable is
+ * the browser's answer, made from what its own media stack accepts, and this
+ * proxy must not acquire a second opinion about it. The count is needed for one
+ * decision — whether a torrent holds exactly one picture, in which case a
+ * soundtrack beside it has nothing else it could belong to.
+ */
+export const VIDEO_FILE_EXTENSIONS = new Set([
+  ".mp4",
+  ".mkv",
+  ".webm",
+  ".mov",
+  ".m4v",
+  ".avi",
+  ".mpg",
+  ".mpeg",
+  ".ts",
+  ".m2ts",
+  ".mts",
+  ".wmv",
+  ".asf",
+  ".flv",
+  ".f4v",
+  ".ogv",
+  ".ogm",
+  ".3gp",
+  ".3g2",
+  ".divx",
+  ".vob",
+  ".m2v",
+  ".m2p",
+  ".mxf",
+  ".rm",
+  ".rmvb"
+]);
+
+/**
+ * How many files of a torrent carry a picture.
+ *
+ * @param {Array<{ path?: string, name?: string }>} files
+ * @returns {number}
+ */
+export function countVideoFiles(files) {
+  let count = 0;
+  for (const file of Array.isArray(files) ? files : []) {
+    const name = typeof file?.name === "string" && file.name.length > 0
+      ? file.name
+      : String(file?.path ?? "");
+    if (VIDEO_FILE_EXTENSIONS.has(extensionOf(name))) {
+      count += 1;
+    }
+  }
+  return count;
+}
+
+/**
+ * Subtitle formats that are text, and therefore small by construction — a whole
+ * episode of dialogue is tens of kilobytes.
+ *
+ * The distinction is used to decide how much of such a file to fetch ahead of
+ * the viewer: a text file is fetched WHOLE, because it is smaller than the
+ * torrent's own piece and reading its edges would cost the same pieces as
+ * reading all of it. An image-based one (`.sup`, `.sub`+`.idx`) is a picture per
+ * cue and runs to tens of megabytes, so it gets the same edges treatment as
+ * anything else.
+ */
+export const TEXT_SUBTITLE_SIDECAR_EXTENSIONS = new Set([
+  ".srt",
+  ".ass",
+  ".ssa",
+  ".vtt",
+  ".webvtt",
+  ".ttml",
+  ".smi",
+  ".txt"
+]);
+
+/**
+ * Extensions whose track table a `Container` subclass can read. Everything else
+ * in {@link AUDIO_SIDECAR_EXTENSIONS} is a bare stream: one track, no metadata.
+ */
+const CONTAINER_BACKED_AUDIO = new Set([".mka", ".m4a", ".mp4", ".mkv", ".webm"]);
+
+/**
+ * The extension of a name, lowercased, including the dot. Empty when there is
+ * none.
+ *
+ * @param {string} name
+ * @returns {string}
+ */
+export function extensionOf(name) {
+  const text = typeof name === "string" ? name : "";
+  const dot = text.lastIndexOf(".");
+  // A leading dot is a hidden file, not an extension.
+  return dot > 0 ? text.slice(dot).toLowerCase() : "";
+}
+
+/**
+ * A name without its extension.
+ *
+ * @param {string} name
+ * @returns {string}
+ */
+export function baseNameOf(name) {
+  const text = typeof name === "string" ? name : "";
+  const dot = text.lastIndexOf(".");
+  return dot > 0 ? text.slice(0, dot) : text;
+}
+
+/**
+ * Whether a sidecar file of this extension has a readable track table.
+ *
+ * @param {string} extension - Lowercased, with the dot.
+ * @returns {boolean}
+ */
+export function declaresItsOwnTracks(extension) {
+  return CONTAINER_BACKED_AUDIO.has(extension);
+}
+
+/**
+ * Split a torrent file's path into the folders above it and its own name, with
+ * the torrent's own name removed from the front.
+ *
+ * WebTorrent prefixes every path in a multi-file torrent with the torrent name;
+ * the browser strips it again to show a playlist. Doing it once, here, means the
+ * folders travel to the browser already relative to the torrent root, so there
+ * is one stripping rule in the system rather than two that can disagree.
+ *
+ * @param {string} path - `file.path` as WebTorrent reports it.
+ * @param {string} torrentName
+ * @returns {{ folders: string[], name: string }}
+ */
+export function splitTorrentPath(path, torrentName) {
+  const text = typeof path === "string" ? path.replace(/\\/g, "/") : "";
+  const prefix = typeof torrentName === "string" && torrentName.length > 0 ? `${torrentName}/` : "";
+  const relative = prefix && text.startsWith(prefix) ? text.slice(prefix.length) : text;
+  const segments = relative.split("/").filter((segment) => segment.length > 0);
+  const name = segments.length > 0 ? segments[segments.length - 1] : "";
+  return { folders: segments.slice(0, -1), name };
+}
+
+/**
+ * Bracketed groups of a name, in order: `[HorribleSubs] X [1080p]` → both.
+ *
+ * @param {string} text
+ * @returns {string[]}
+ */
+export function bracketTokensOf(text) {
+  const source = typeof text === "string" ? text : "";
+  const tokens = [];
+  for (const match of source.matchAll(/\[([^\]]+)\]/g)) {
+    const token = match[1].trim();
+    if (token.length > 0) {
+      tokens.push(token);
+    }
+  }
+  return tokens;
+}
+
+/**
+ * Bracketed groups that look like a release hash — `[78EFD746]`. Anime releases
+ * carry the CRC of the file, and two files sharing one are certainly a pair.
+ *
+ * @param {string} text
+ * @returns {string[]}
+ */
+function hashTokensOf(text) {
+  return bracketTokensOf(text)
+    .filter((token) => /^[0-9a-f]{4,10}$/i.test(token))
+    .map((token) => token.toLowerCase());
+}
+
+/**
+ * Whether a sidecar and a video are the same release of the same episode.
+ *
+ * Two rules, both taken from what releases actually do, and neither of them
+ * involving the folder — a dub lives in a folder of its own by construction, so
+ * requiring the folders to match would reject every case this exists for:
+ *
+ * 1. the base names are equal, which is the common shape (`X.mkv` / `X.mka`);
+ * 2. they share a release hash, which anime releases carry.
+ *
+ * @param {string} sidecarName
+ * @param {string} videoName
+ * @returns {boolean}
+ */
+export function namesPair(sidecarName, videoName) {
+  const sidecarBase = baseNameOf(sidecarName).toLowerCase();
+  const videoBase = baseNameOf(videoName).toLowerCase();
+  if (sidecarBase.length === 0 || videoBase.length === 0) {
+    return false;
+  }
+  if (sidecarBase === videoBase) {
+    return true;
+  }
+  const videoHashes = new Set(hashTokensOf(videoBase));
+  if (videoHashes.size > 0) {
+    for (const token of hashTokensOf(sidecarBase)) {
+      if (videoHashes.has(token)) {
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+/**
+ * @typedef {object} SidecarFile
+ * @property {number} fileIndex - Index in the torrent.
+ * @property {string} name - File name, extension included.
+ * @property {string[]} folders - Folders above it, relative to the torrent root.
+ * @property {string} extension - Lowercased, with the dot.
+ * @property {number} length - Bytes.
+ * @property {boolean} declaresTracks - Whether its own track table can be read.
+ */
+
+/**
+ * The audio and subtitle files of a torrent that belong to one video file.
+ *
+ * The pairing is by name (see {@link namesPair}), with ONE relaxation: a torrent
+ * holding a single video file has nothing else its sidecars could belong to, so
+ * there every sidecar is taken. That covers the very common
+ * `Film.mkv` + `Rus.mka` shape, where the two names have nothing in common. It
+ * is deliberately not extended to a torrent with several videos, where a wrong
+ * pairing would put the sound of one episode over the picture of another.
+ *
+ * @param {object} params
+ * @param {Array<{ path?: string, name?: string, length?: number }>} params.files
+ * @param {number} params.videoIndex
+ * @param {string} [params.torrentName]
+ * @param {number} [params.videoCount] - How many playable video files the
+ *   torrent holds. When 1, the relaxation above applies. Counted by the caller,
+ *   which is the side that knows what counts as playable.
+ * @returns {{ audio: SidecarFile[], subtitles: SidecarFile[] }}
+ */
+export function matchSidecarFiles({ files, videoIndex, torrentName = "", videoCount = 0 }) {
+  const list = Array.isArray(files) ? files : [];
+  const video = list[videoIndex];
+  if (!video) {
+    return { audio: [], subtitles: [] };
+  }
+  const videoPath = splitTorrentPath(video.path ?? video.name ?? "", torrentName);
+  const onlyVideo = videoCount === 1;
+  /** @type {SidecarFile[]} */
+  const audio = [];
+  /** @type {SidecarFile[]} */
+  const subtitles = [];
+
+  for (const [fileIndex, file] of list.entries()) {
+    if (fileIndex === videoIndex) {
+      continue;
+    }
+    const { folders, name } = splitTorrentPath(file?.path ?? file?.name ?? "", torrentName);
+    const extension = extensionOf(name);
+    const isAudio = AUDIO_SIDECAR_EXTENSIONS.has(extension);
+    const isSubtitle = !isAudio && SUBTITLE_SIDECAR_EXTENSIONS.has(extension);
+    if (!isAudio && !isSubtitle) {
+      continue;
+    }
+    if (!onlyVideo && !namesPair(name, videoPath.name)) {
+      continue;
+    }
+    const entry = {
+      fileIndex,
+      name,
+      folders,
+      extension,
+      length: Number.isFinite(file?.length) ? file.length : 0,
+      declaresTracks: declaresItsOwnTracks(extension)
+    };
+    if (isAudio) {
+      audio.push(entry);
+    } else {
+      subtitles.push(entry);
+    }
+  }
+
+  return { audio, subtitles };
+}

package/services/torrent-worker/client.js

@@ -325,6 +325,20 @@ export class TorrentWorkerClient {
     return this.#caller.call(Command.SUBTITLE_TRACKS, { sourceKey, fileIndex });
   }
 
+  /**
+   * Every track one file declares, read from its own header.
+   *
+   * Asked of the picture for the flags ffmpeg's banner does not carry, and of a
+   * soundtrack shipped as its own file beside it — the same question about a
+   * different file, which is why there is one command rather than two.
+   *
+   * @param {{ sourceKey: string, fileIndex: number }} params
+   * @returns {Promise<{ tracks: object[] }>}
+   */
+  async getContainerTracks({ sourceKey, fileIndex }) {
+    return this.#caller.call(Command.CONTAINER_TRACKS, { sourceKey, fileIndex });
+  }
+
   /**
    * The cues of one subtitle track that can be read from what is downloaded.
    *

package/services/torrent-worker/container-tracks.js

@@ -0,0 +1,243 @@
+/**
+ * The tracks a file declares, read from the file itself on the thread that owns
+ * the torrent.
+ *
+ * This exists because the container layer (`services/container/`) reads bytes,
+ * and the bytes live here: the torrent client runs on this worker thread, so the
+ * main thread cannot open a read stream on one of its files. The main thread
+ * asks over the channel (`Command.CONTAINER_TRACKS`) and gets plain objects
+ * back, because a class instance does not survive the boundary as a class.
+ *
+ * It reads whatever container the file is — the same `ContainerFactory` that
+ * serves the picture serves a `.mka` beside it, since a `.mka` IS Matroska and
+ * differs only in having no video track. That is the whole reason no new class
+ * was needed for a soundtrack shipped as its own file.
+ *
+ * Unlike the subtitle walk, the reads here DO wait on the swarm. A sidecar file
+ * has usually had nothing downloaded at all when this is first asked, and its
+ * header is a few hundred kilobytes; the alternative is offering the viewer a
+ * soundtrack with nothing known about it.
+ */
+
+import { containerOrchestrator } from "../orchestrators/ContainerOrchestrator.js";
+import { logger } from "../../utils/logger.js";
+
+/**
+ * How long one range read may take before it is given up.
+ *
+ * Not a measurement and nothing is derived from it: it is the point past which a
+ * read of a file nobody is playing is presumed lost, so that one stream which
+ * never ends cannot hold the answer — and with it the viewer's audio menu — for
+ * the rest of the session.
+ */
+const READ_ABANDON_MS = 60_000;
+
+/** Bytes of the head a container's track table lives in. */
+const HEAD_BYTES = 256 * 1024;
+
+/**
+ * Tracks already read, by `sourceKey:fileIndex`. Only a non-empty reading is
+ * kept: an empty one usually means the header has not arrived yet, and caching
+ * that would hide the file's tracks for the life of the process.
+ *
+ * @type {Map<string, object[]>}
+ */
+const byFile = new Map();
+
+/** @type {Map<string, Promise<object[]>>} */
+const inFlight = new Map();
+
+/**
+ * Read a byte range of a file, fetching what is missing.
+ *
+ * @param {object} file
+ * @param {number} start
+ * @param {number} end - Inclusive.
+ * @returns {Promise<Buffer | null>}
+ */
+function readFetching(file, start, end) {
+  return new Promise((resolve) => {
+    const chunks = [];
+    let stream;
+    try {
+      stream = file.createReadStream({ start, end });
+    } catch {
+      resolve(null);
+      return;
+    }
+    let settled = false;
+    /** @type {ReturnType<typeof setTimeout> | null} */
+    let abandon = null;
+    const settle = (value) => {
+      if (settled) {
+        return;
+      }
+      settled = true;
+      if (abandon !== null) {
+        clearTimeout(abandon);
+      }
+      if (value === null) {
+        stream.destroy?.();
+      }
+      resolve(value);
+    };
+    abandon = setTimeout(() => {
+      logger.info(
+        `container-tracks: a read of ${start}-${end} in "${String(file.name).slice(0, 40)}" ` +
+        `did not finish in ${READ_ABANDON_MS / 1000}s and was given up`
+      );
+      settle(null);
+    }, READ_ABANDON_MS);
+    abandon.unref?.();
+    stream.on("data", (chunk) => chunks.push(chunk));
+    stream.on("end", () => settle(Buffer.concat(chunks)));
+    stream.on("error", () => settle(null));
+  });
+}
+
+/**
+ * One track as it crosses the thread boundary: the fields the inventory and the
+ * master playlist read, and nothing that would not survive being cloned.
+ *
+ * @param {import("../tracks/index.js").ContainerTrack} track
+ * @returns {object}
+ */
+function plainTrack(track) {
+  return {
+    type: track.type,
+    trackNumber: track.trackNumber,
+    declaredIndex: track.declaredIndex,
+    codecId: track.codecId,
+    language: track.resolvedLanguage(),
+    languageBcp47: track.languageBcp47,
+    name: track.name,
+    isEnabled: track.isEnabled,
+    isDefault: track.isDefault,
+    declaresDefault: track.declaresDefault,
+    // Audio-only, per RFC 9559 — absent on other types, and read as false there
+    // rather than being placed on the base track where they do not belong.
+    isOriginal: track.isOriginal === true,
+    isCommentary: track.isCommentary === true,
+    isVisualImpaired: track.isVisualImpaired === true,
+    channels: Number.isFinite(track.channels) ? track.channels : null,
+    samplingFrequency: Number.isFinite(track.samplingFrequency) ? track.samplingFrequency : null
+  };
+}
+
+/**
+ * Every track one file declares, in container order.
+ *
+ * @param {object} torrent
+ * @param {number} fileIndex
+ * @param {string} sourceKey
+ * @param {{ prefetchEdges?: () => Promise<unknown> }} [options]
+ * @returns {Promise<object[]>}
+ */
+export async function containerTracksOf(torrent, fileIndex, sourceKey, options = {}) {
+  const key = `${sourceKey}:${fileIndex}`;
+  const held = byFile.get(key);
+  if (Array.isArray(held)) {
+    return held;
+  }
+  const running = inFlight.get(key);
+  if (running) {
+    return running;
+  }
+  const work = readTracks(torrent, fileIndex, sourceKey, options)
+    .then((tracks) => {
+      if (tracks.length > 0) {
+        byFile.set(key, tracks);
+      }
+      return tracks;
+    })
+    .finally(() => {
+      inFlight.delete(key);
+    });
+  inFlight.set(key, work);
+  return work;
+}
+
+/**
+ * @param {object} torrent
+ * @param {number} fileIndex
+ * @param {string} sourceKey
+ * @param {{ prefetchEdges?: () => Promise<unknown> }} options
+ * @returns {Promise<object[]>}
+ */
+async function readTracks(torrent, fileIndex, sourceKey, options) {
+  const file = torrent?.files?.[fileIndex];
+  if (!file || !Number.isFinite(file.length) || file.length <= 0) {
+    return [];
+  }
+  if (typeof options.prefetchEdges === "function") {
+    try {
+      await options.prefetchEdges();
+    } catch {
+      // A prefetch that failed is not a reason to skip the read: the read
+      // fetches what it needs itself, only more slowly.
+    }
+  }
+  const readRange = async (start, end) =>
+    readFetching(file, start, Math.min(end, file.length - 1));
+  try {
+    const tracks = await containerOrchestrator.getTracks({
+      sourceKey,
+      fileIndex,
+      readRange,
+      fileSize: file.length,
+      label: String(file.name ?? "")
+    });
+    const plain = tracks.map(plainTrack);
+    if (plain.length > 0) {
+      logger.info(
+        `container-tracks: "${String(file.name).slice(0, 40)}" declares ` +
+        `${plain.filter((track) => track.type === "video").length} video, ` +
+        `${plain.filter((track) => track.type === "audio").length} audio, ` +
+        `${plain.filter((track) => track.type === "subtitle").length} subtitle track(s)`
+      );
+    }
+    return plain;
+  } catch (error) {
+    logger.warn(
+      `container-tracks: "${String(file.name).slice(0, 40)}" could not be read: ${error?.message ?? error}`
+    );
+    return [];
+  }
+}
+
+/**
+ * The audio tracks of one file, in the order ffmpeg numbers them `0:a:N`.
+ *
+ * @param {object} torrent
+ * @param {number} fileIndex
+ * @param {string} sourceKey
+ * @param {{ prefetchEdges?: () => Promise<unknown> }} [options]
+ * @returns {Promise<object[]>}
+ */
+export async function containerAudioTracksOf(torrent, fileIndex, sourceKey, options = {}) {
+  const tracks = await containerTracksOf(torrent, fileIndex, sourceKey, options);
+  return tracks
+    .filter((track) => track.type === "audio")
+    .sort((left, right) => left.declaredIndex - right.declaredIndex);
+}
+
+/** Bytes of a file's head worth fetching before its track table is read. */
+export const CONTAINER_HEAD_BYTES = HEAD_BYTES;
+
+/**
+ * Forget one file's tracks, or every file of a source.
+ *
+ * @param {string} sourceKey
+ * @param {number} [fileIndex]
+ */
+export function forgetContainerTracks(sourceKey, fileIndex) {
+  if (fileIndex === undefined) {
+    for (const key of [...byFile.keys()]) {
+      if (key.startsWith(`${sourceKey}:`)) {
+        byFile.delete(key);
+      }
+    }
+    return;
+  }
+  byFile.delete(`${sourceKey}:${fileIndex}`);
+}

package/services/torrent-worker/pool-adapter.js

@@ -161,6 +161,44 @@ export class WorkerTorrentPool {
     return Array.isArray(answer?.declared) ? answer.declared : [];
   }
 
+  /**
+   * Every track one file declares, read from its own header by the container
+   * layer.
+   *
+   * The audio menu is built from ffmpeg's `-i` banner, which carries neither
+   * `FlagOriginal`, `FlagCommentary`, `FlagVisualImpaired`, `FlagEnabled` nor
+   * `LanguageBCP47` — so without this a director's commentary and the film
+   * itself are indistinguishable in it. Also how a soundtrack shipped as its own
+   * file is read: a `.mka` is Matroska and the same reader serves it.
+   *
+   * @param {object} torrent
+   * @param {number} fileIndex
+   * @returns {Promise<object[]>}
+   */
+  async getContainerTracks(torrent, fileIndex) {
+    const sourceKey = torrent?.sourceKey;
+    if (!sourceKey) {
+      return [];
+    }
+    const answer = await this.#client.getContainerTracks({ sourceKey, fileIndex });
+    return Array.isArray(answer?.tracks) ? answer.tracks : [];
+  }
+
+  /**
+   * The audio tracks one file declares, in the order ffmpeg numbers them
+   * `0:a:N`.
+   *
+   * @param {object} torrent
+   * @param {number} fileIndex
+   * @returns {Promise<object[]>}
+   */
+  async getDeclaredAudioTracks(torrent, fileIndex) {
+    const tracks = await this.getContainerTracks(torrent, fileIndex);
+    return tracks
+      .filter((track) => track?.type === "audio")
+      .sort((left, right) => (left.declaredIndex ?? 0) - (right.declaredIndex ?? 0));
+  }
+
   /**
    * The cues of one subtitle track that the downloaded clusters already carry.
    *

package/services/torrent-worker/protocol.js

@@ -67,6 +67,14 @@ export const Command = {
   PREFETCH_EDGES: "prefetch-edges",
   /** The text subtitle tracks a file carries, for the viewer's menu. */
   SUBTITLE_TRACKS: "subtitle-tracks",
+  /**
+   * Every track a file declares, read from its own header by the container
+   * layer. Answers what ffmpeg's `-i` banner cannot: FlagOriginal,
+   * FlagCommentary, FlagVisualImpaired, FlagEnabled and LanguageBCP47 appear
+   * nowhere in it. Asked of the picture AND of a soundtrack shipped as its own
+   * file beside it, which is the same question about a different file.
+   */
+  CONTAINER_TRACKS: "container-tracks",
   /** Cues of one subtitle track, from the clusters already downloaded. */
   SUBTITLE_CUES: "subtitle-cues",
   /** Shut the client down, optionally deleting downloaded data. */