@lincoln504/pi-research 0.1.12 → 1.0.0

package/src/youtube/potoken.ts

@@ -0,0 +1,259 @@
+/**
+ * YouTube PoToken minting (BotGuard / Proof-of-Origin)
+ *
+ * YouTube's 2026 BotGuard attestation requires a Proof-of-Origin Token (PoToken)
+ * for caption/transcript requests. Without one, the timedtext endpoint returns
+ * HTTP 200 with an EMPTY body (a silent failure). This module mints tokens with
+ * `bgutils-js`, running the BotGuard VM under a `jsdom` DOM.
+ *
+ * Hard-won implementation details (validated live against youtubei.js@17 /
+ * bgutils-js@3 / jsdom@29):
+ *
+ *  1. Cross-realm typed arrays. The BotGuard VM must run in the SAME JS realm as
+ *     bgutils' `base64ToU8` (the Node realm). So the interpreter is eval'd via
+ *     `new Function(interp)()` in the Node realm and jsdom's browser globals
+ *     (window/document/…) are bridged onto Node `globalThis`. Running the VM in
+ *     jsdom's own realm makes the minter factory return a non-function
+ *     ("APF:Failed") because of an `instanceof Uint8Array` realm mismatch.
+ *
+ *  2. The VM references `window` during EVERY mint call (not just setup), so we
+ *     mint all identifiers (the session visitorData + every video ID) up front
+ *     while the realm is bridged, then unbridge and close the DOM.
+ *
+ *  3. Mutating `globalThis` is process-global, so the whole bridged section is
+ *     serialized behind a module-level async mutex to prevent concurrent
+ *     researchers from racing on the shared globals.
+ *
+ *     The mutex only guards youtube-vs-youtube. The one other main-process
+ *     consumer that feature-detects `window` is @huggingface/transformers, which
+ *     computes `IS_BROWSER_ENV = typeof window !== 'undefined' && …` ONCE at
+ *     import time and freezes it (Object.freeze). That import is forced — and the
+ *     constant frozen as NODE — before any mint can bridge: the deep orchestrator
+ *     awaits `getService(KNOWLEDGE_STORE)` (→ static transformers import) in its
+ *     pre-research phase before researchers spawn, and quick mode runs a single
+ *     agent whose tools never overlap. So the bridge cannot flip the embedder's
+ *     environment. If you ever make the store load lazily AFTER gathering begins,
+ *     or make quick mode multi-agent, pre-warm the transformers import explicitly
+ *     or this invariant breaks.
+ *
+ *  4. Content binding. timedtext needs a token minted with `identifier =
+ *     videoId`; the session token (identifier = visitorData) is only used to
+ *     create the Innertube session. We mint both from a single challenge.
+ */
+
+import { logger } from '../logger.ts';
+import { metrics } from '../utils/metrics.ts';
+
+/**
+ * Well-known request key for the YouTube web PoToken program. This is a public
+ * constant baked into the YouTube web player; it is overridable via env in case
+ * YouTube rotates it (see config YOUTUBE_POTOKEN_REQUEST_KEY).
+ */
+export const DEFAULT_REQUEST_KEY = 'O43z0dpjhgX20SCx4KAo';
+
+/** Browser globals the BotGuard VM probes; bridged from jsdom onto globalThis. */
+const BRIDGED_GLOBALS = ['window', 'document', 'location', 'navigator', 'origin'] as const;
+
+export interface MintOptions {
+  /** Override the PoToken program request key (defaults to the web key). */
+  requestKey?: string;
+  /** Fetch implementation (defaults to global fetch); lets callers route/proxy. */
+  fetchImpl?: typeof fetch;
+  /** Cancellation signal — aborts the BotGuard challenge/integrity network calls. */
+  signal?: AbortSignal;
+}
+
+/** A bgutils-js error carrying a `.code` (e.g. VM_ERROR, INTEGRITY_ERROR). */
+function describeError(error: unknown): string {
+  if (error && typeof error === 'object') {
+    const e = error as { code?: string; message?: string };
+    if (e.code && e.message) return `${e.code}: ${e.message}`;
+    if (e.message) return e.message;
+  }
+  return String(error);
+}
+
+// Module-level mutex: serialize all globalThis mutation across concurrent callers.
+let mintChain: Promise<unknown> = Promise.resolve();
+
+/**
+ * Mint a session-bound token (for `identifiers[0]`, the visitorData) plus one
+ * content-bound token per remaining identifier (the video IDs), all from a
+ * single BotGuard challenge. Returns a map keyed by identifier.
+ *
+ * Throws on challenge/attestation/integrity failure — the caller decides how to
+ * degrade (typically: mark every video's transcript as unavailable).
+ */
+export async function mintPoTokens(
+  identifiers: string[],
+  opts: MintOptions = {},
+): Promise<Map<string, string>> {
+  // Chain on the mutex so only one bridged section runs at a time.
+  const run = mintChain.then(() => doMint(identifiers, opts), () => doMint(identifiers, opts));
+  // Keep the chain alive regardless of this call's success/failure.
+  mintChain = run.then(() => undefined, () => undefined);
+  return run;
+}
+
+async function doMint(
+  identifiers: string[],
+  opts: MintOptions,
+): Promise<Map<string, string>> {
+  if (identifiers.length === 0) return new Map();
+
+  const requestKey = opts.requestKey || process.env['PI_RESEARCH_YOUTUBE_POTOKEN_REQUEST_KEY'] || DEFAULT_REQUEST_KEY;
+  const fetchImpl = opts.fetchImpl ?? fetch;
+  const started = Date.now();
+
+  // Lazy-load heavy deps — never imported at module load (keeps startup native-free).
+  const bgMod = (await import('bgutils-js')) as unknown as BgModule;
+  const BG = bgMod.BG ?? bgMod.default;
+  const { buildURL, getHeaders } = bgMod;
+  const { JSDOM } = await import('jsdom');
+
+  const dom = new JSDOM('<!DOCTYPE html><html lang="en"><body></body></html>', {
+    url: 'https://www.youtube.com/',
+  });
+
+  const savedDescriptors: Record<string, PropertyDescriptor | undefined> = {};
+  const bridge = () => {
+    for (const key of BRIDGED_GLOBALS) {
+      savedDescriptors[key] = Object.getOwnPropertyDescriptor(globalThis, key);
+      try {
+        Object.defineProperty(globalThis, key, {
+          value: (dom.window as unknown as Record<string, unknown>)[key],
+          configurable: true,
+          writable: true,
+        });
+      } catch {
+        // Some globals (e.g. navigator) may be non-configurable on certain
+        // runtimes; skip — the VM only needs the ones it can read.
+      }
+    }
+  };
+  const unbridge = () => {
+    for (const key of BRIDGED_GLOBALS) {
+      try {
+        const saved = savedDescriptors[key];
+        if (saved) Object.defineProperty(globalThis, key, saved);
+        else delete (globalThis as Record<string, unknown>)[key];
+      } catch {
+        // Best-effort restore: never let one non-restorable global abort the
+        // loop (which would leave later globals dirty and skip DOM teardown).
+      }
+    }
+  };
+
+  bridge();
+  try {
+    const bgConfig: BgConfig = {
+      fetch: (input: unknown, init?: unknown) =>
+        fetchImpl(input as Parameters<typeof fetch>[0], {
+          ...(init as RequestInit | undefined),
+          signal: (init as RequestInit | undefined)?.signal ?? opts.signal,
+        }),
+      globalObj: globalThis,
+      identifier: identifiers[0]!,
+      requestKey,
+    };
+
+    const challenge = await BG.Challenge.create(bgConfig);
+    if (!challenge) throw new Error('BotGuard challenge unavailable');
+
+    const interpreter = challenge.interpreterJavascript?.privateDoNotAccessOrElseSafeScriptWrappedValue;
+    if (!interpreter) throw new Error('BotGuard interpreter script missing');
+    // Eval in the Node realm so the VM and bgutils share one Uint8Array realm.
+    // The interpreter is YouTube's BotGuard VM, fetched over HTTPS from Google's
+    // attestation endpoint by bgutils — running it is required and inherent to
+    // PoToken generation; trust rests on that TLS connection.
+    new Function(interpreter)();
+
+    const botguard = await BG.BotGuardClient.create({
+      program: challenge.program,
+      globalName: challenge.globalName,
+      globalObj: globalThis,
+    });
+
+    const webPoSignalOutput: unknown[] = [];
+    const botguardResponse = await botguard.snapshot({ webPoSignalOutput });
+
+    const integrityResponse = await fetchImpl(buildURL('GenerateIT', false), {
+      method: 'POST',
+      headers: getHeaders(),
+      body: JSON.stringify([requestKey, botguardResponse]),
+      signal: opts.signal,
+    });
+    const integrityJson = (await integrityResponse.json()) as unknown[];
+    const integrityToken = integrityJson?.[0] as string | undefined;
+    if (!integrityToken) {
+      throw new Error('Integrity token rejected by attestation server (likely a non-residential IP)');
+    }
+
+    const minter = await BG.WebPoMinter.create({ integrityToken }, webPoSignalOutput);
+
+    const tokens = new Map<string, string>();
+    for (const id of identifiers) {
+      tokens.set(id, await minter.mintAsWebsafeString(id));
+    }
+
+    metrics.observe('youtube_potoken_mint_duration_ms', Date.now() - started, { status: 'success' });
+    metrics.increment('youtube_potoken_mints_total', tokens.size, { status: 'success' });
+    logger.log(`[youtube] Minted ${tokens.size} PoToken(s) in ${Date.now() - started}ms`);
+    return tokens;
+  } catch (error) {
+    metrics.increment('youtube_potoken_mints_total', 1, { status: 'error' });
+    logger.warn(`[youtube] PoToken minting failed: ${describeError(error)}`);
+    throw error instanceof Error ? error : new Error(describeError(error));
+  } finally {
+    // Always restore globals and tear down the DOM, even if unbridge throws.
+    try {
+      unbridge();
+    } catch {
+      // unbridge is already per-key best-effort; this guards the loop itself.
+    }
+    try {
+      dom.window.close();
+    } catch {
+      // best-effort teardown
+    }
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Minimal structural types for the loosely-typed bgutils-js surface we touch.
+// ---------------------------------------------------------------------------
+
+interface BgConfig {
+  fetch: (input: unknown, init?: unknown) => Promise<Response>;
+  globalObj: unknown;
+  identifier: string;
+  requestKey: string;
+}
+
+interface BgNamespace {
+  Challenge: {
+    create(config: BgConfig): Promise<{
+      interpreterJavascript?: { privateDoNotAccessOrElseSafeScriptWrappedValue?: string };
+      program: string;
+      globalName: string;
+    } | undefined>;
+  };
+  BotGuardClient: {
+    create(opts: { program: string; globalName: string; globalObj: unknown }): Promise<{
+      snapshot(args: { webPoSignalOutput: unknown[] }): Promise<string>;
+    }>;
+  };
+  WebPoMinter: {
+    create(
+      integrity: { integrityToken: string },
+      webPoSignalOutput: unknown[],
+    ): Promise<{ mintAsWebsafeString(identifier: string): Promise<string> }>;
+  };
+}
+
+interface BgModule {
+  BG?: BgNamespace;
+  default: BgNamespace;
+  buildURL(endpoint: string, useYouTubeAPI?: boolean): string;
+  getHeaders(): Record<string, string>;
+}

package/src/youtube/transcript-client.ts

@@ -0,0 +1,390 @@
+/**
+ * YouTube transcript client
+ *
+ * Fetches transcripts for a batch of video IDs using youtubei.js (InnerTube) for
+ * metadata + caption-track discovery and a direct timedtext request for the
+ * caption content, authenticated with a content-bound PoToken.
+ *
+ * Design notes:
+ *  - `getTranscript()` (the get_transcript endpoint) is deliberately NOT used: it
+ *    returns 400 FAILED_PRECONDITION even with a valid PoToken. The caption track
+ *    (timedtext, fmt=json3) is fetched directly instead.
+ *  - Empty-body guard: the timedtext endpoint returns HTTP 200 with an empty body
+ *    when the PoToken is missing/invalid. Empty or whitespace-only transcripts are
+ *    treated as failures and never surfaced as blank successes.
+ *  - Per-video isolation: one failed/unavailable video never aborts the batch.
+ *  - All network work is bounded by a per-video timeout and guarded by a shared
+ *    circuit breaker so a YouTube outage fails fast instead of hanging researchers.
+ */
+
+import { withTimeout, retryWithBackoff, createTimeoutSignal } from '../web-research/retry-utils.ts';
+import { CircuitBreaker } from '../utils/circuit-breaker.ts';
+import { logger } from '../logger.ts';
+import { metrics } from '../utils/metrics.ts';
+import { mintPoTokens } from './potoken.ts';
+import { watchUrl } from './video-id.ts';
+
+export interface VideoTranscript {
+  videoId: string;
+  url: string;
+  success: boolean;
+  title?: string;
+  author?: string;
+  durationSeconds?: number;
+  lang?: string;
+  text?: string;
+  charCount?: number;
+  error?: string;
+}
+
+export interface FetchTranscriptsOptions {
+  /** Preferred caption language (BCP-47 prefix, e.g. 'en'). Defaults to 'en'. */
+  lang?: string;
+  /** Per-video timeout in milliseconds. Defaults to 20000. */
+  timeoutMs?: number;
+  /** PoToken request key override. */
+  requestKey?: string;
+  /** Caller cancellation signal. */
+  signal?: AbortSignal;
+  /** Fires as each video completes (success or failure) — used for TUI flashes. */
+  onVideoComplete?: (videoId: string, success: boolean) => void;
+}
+
+// Shared breaker: a sustained YouTube/attestation outage trips it so subsequent
+// researchers fail fast rather than each waiting out the full timeout.
+const breaker = new CircuitBreaker({
+  failureThreshold: 5,
+  resetTimeoutMs: 30000,
+  name: 'YouTube transcript',
+  isTransientError: (err) => {
+    const msg = (err instanceof Error ? err.message : String(err)).toLowerCase();
+    return (
+      msg.includes('timeout') ||
+      // withTimeout() rejects with "...cancelled or timed out" (no contiguous "timeout"),
+      // so without this the breaker never counted a genuine YouTube hang toward its
+      // failure threshold and every researcher paid the full timeout during an outage.
+      msg.includes('timed out') ||
+      msg.includes('network') ||
+      msg.includes('econn') ||
+      msg.includes('fetch failed') ||
+      msg.includes('429') ||
+      msg.includes('500') ||
+      msg.includes('502') ||
+      msg.includes('503')
+    );
+  },
+});
+
+interface CaptionTrack {
+  base_url: string;
+  language_code?: string;
+  kind?: string;
+}
+
+/**
+ * Fetch transcripts for up to a handful of video IDs. Always resolves with one
+ * result per input ID (never throws for individual-video problems); only a
+ * total infrastructure failure (e.g. youtubei.js unavailable) rejects.
+ */
+export async function fetchVideoTranscripts(
+  videoIds: string[],
+  opts: FetchTranscriptsOptions = {},
+): Promise<VideoTranscript[]> {
+  const lang = opts.lang || 'en';
+  const timeoutMs = opts.timeoutMs ?? 20000;
+  const signal = opts.signal;
+
+  if (videoIds.length === 0) return [];
+
+  // Lazy-load youtubei.js — never imported at module load (keeps startup native-free).
+  const yt = await import('youtubei.js');
+  const { Innertube } = yt;
+  // Silence youtubei.js's own console logging — it writes parser warnings
+  // straight to the console, which would corrupt the TUI (stdout is the UI).
+  try {
+    const log = (yt as unknown as { Log?: { setLevel?: (n: number) => void; Level?: { NONE: number } } }).Log;
+    if (log?.setLevel && log.Level) log.setLevel(log.Level.NONE);
+  } catch {
+    // non-fatal: logging stays at default if the API ever changes
+  }
+  const fetchImpl: typeof fetch = (input, init) => fetch(input, init);
+
+  // 1) Throwaway session to obtain stable visitor data for the PoToken binding.
+  let visitorData: string;
+  try {
+    // Route the once-per-batch seed session through the same breaker as fetchOne: the
+    // seed + mint are the steps most prone to a SYSTEMIC outage (BotGuard/attestation
+    // down, non-residential IP), and without the breaker every batch paid the full
+    // timeout on each one. Only transient errors count toward the threshold (the breaker
+    // gates on isTransientError), so a permanent "no residential IP" failure won't trip it.
+    const seed = await breaker.execute(() => withTimeout(
+      Innertube.create({ retrieve_player: false }),
+      timeoutMs,
+      'youtube:create-seed-session',
+      signal,
+    ));
+    visitorData = seed.session.context?.client?.visitorData ?? '';
+  } catch (error) {
+    logger.warn(`[youtube] Failed to create seed session: ${msg(error)}`);
+    return videoIds.map((videoId) => failure(videoId, 'Could not initialize YouTube session.'));
+  }
+  if (!visitorData) {
+    return videoIds.map((videoId) => failure(videoId, 'YouTube did not return visitor data.'));
+  }
+
+  // 2) Mint a session token + one content-bound token per video, all at once.
+  //    Bounded by withTimeout + the caller's signal so a hung BotGuard challenge
+  //    can never stall the researcher.
+  let tokens: Map<string, string>;
+  try {
+    // Pass a timeout-combined signal INTO the mint, not just to withTimeout's wrapper. doMint
+    // threads opts.signal into its BotGuard challenge/integrity fetches, so this makes the timeout
+    // actually abort an in-flight mint and release the process-wide mintChain mutex. Previously the
+    // raw researcher signal was passed, so on timeout withTimeout rejected the wrapper while doMint
+    // kept running and held the mutex forever — one stalled Google socket then hung every later
+    // YouTube mint in the run (each surfacing as a 20s "token unavailable").
+    const mintSignal = createTimeoutSignal(timeoutMs, signal);
+    tokens = await breaker.execute(() => withTimeout(
+      mintPoTokens([visitorData, ...videoIds], { requestKey: opts.requestKey, fetchImpl, signal: mintSignal }),
+      timeoutMs,
+      'youtube:mint',
+      signal,
+    ));
+  } catch (error) {
+    // Without a PoToken, timedtext silently returns empty — so degrade honestly:
+    // report every video as unavailable with the real reason.
+    const reason = `Bot-protection token unavailable (${msg(error)}). YouTube transcripts require a residential IP.`;
+    return videoIds.map((videoId) => failure(videoId, reason));
+  }
+
+  const sessionToken = tokens.get(visitorData);
+  if (!sessionToken) {
+    return videoIds.map((videoId) => failure(videoId, 'Failed to mint a YouTube session token.'));
+  }
+
+  // 3) Real session, bound to the session token + visitor data, with player
+  //    retrieval enabled so caption tracks are present on getInfo().
+  let session: Awaited<ReturnType<typeof Innertube.create>>;
+  try {
+    session = await withTimeout(
+      Innertube.create({ po_token: sessionToken, visitor_data: visitorData, retrieve_player: true }),
+      timeoutMs,
+      'youtube:create-session',
+      signal,
+    );
+  } catch (error) {
+    logger.warn(`[youtube] Failed to create authenticated session: ${msg(error)}`);
+    return videoIds.map((videoId) => failure(videoId, 'Could not initialize authenticated YouTube session.'));
+  }
+
+  // 4) Fetch each video's transcript independently (bounded + isolated).
+  const results = await Promise.all(
+    videoIds.map((videoId) =>
+      fetchOne(session, videoId, tokens.get(videoId), lang, timeoutMs, signal, fetchImpl)
+        .then((result) => {
+          // Guard the caller-supplied callback: a throwing callback must never
+          // reject Promise.all and break the never-throws contract.
+          try {
+            opts.onVideoComplete?.(videoId, result.success);
+          } catch {
+            // ignore callback errors
+          }
+          return result;
+        }),
+    ),
+  );
+
+  // youtubei.js Innertube sessions are plain GC'd objects with no disposer and
+  // no per-session sockets (Node fetch uses a shared agent), so there is nothing
+  // to close here.
+  return results;
+}
+
+async function fetchOne(
+  session: { getInfo(id: string): Promise<unknown> },
+  videoId: string,
+  contentToken: string | undefined,
+  lang: string,
+  timeoutMs: number,
+  signal: AbortSignal | undefined,
+  fetchImpl: typeof fetch,
+): Promise<VideoTranscript> {
+  const started = Date.now();
+  try {
+    if (!contentToken) {
+      return failure(videoId, 'No content-bound token was minted for this video.');
+    }
+
+    const result = await breaker.execute(() =>
+      retryWithBackoff(
+        async () => {
+          const info = (await withTimeout(
+            session.getInfo(videoId) as Promise<VideoInfoLike>,
+            timeoutMs,
+            `youtube:getInfo:${videoId}`,
+            signal,
+          )) as VideoInfoLike;
+
+          const tracks = (info.captions?.caption_tracks ?? []) as CaptionTrack[];
+          const track = pickTrack(tracks, lang);
+          if (!track) {
+            // No captions at all — a permanent condition; do not retry.
+            throw new PermanentError('No captions/transcript available for this video.');
+          }
+
+          // base_url is parsed out of YouTube's getInfo() response. Defense in
+          // depth: it must be an https YouTube/Google timedtext host before we
+          // append the PoToken and fetch it — never a private/metadata target.
+          assertTrustedTimedtextUrl(track.base_url);
+
+          const sep = track.base_url.includes('?') ? '&' : '?';
+          const url = `${track.base_url}${sep}fmt=json3&c=WEB&pot=${encodeURIComponent(contentToken)}`;
+          // Pass a combined timeout+caller signal straight into fetch so BOTH the
+          // response headers AND the body read are bounded and tear the socket
+          // down on timeout/cancellation (withTimeout alone would not bound text()).
+          const fetchSignal = createTimeoutSignal(timeoutMs, signal);
+          const response = await fetchImpl(url, {
+            signal: fetchSignal,
+            headers: { 'User-Agent': 'Mozilla/5.0', 'Accept-Language': lang },
+          });
+          // Surface the real HTTP status. Without this, a 403 (age-restricted) or
+          // 404 (removed) returns HTML that parseJson3 yields '' for, which would
+          // be misreported below as a bot-protection/PoToken failure.
+          if (!response.ok) {
+            throw new Error(`HTTP ${response.status} from timedtext endpoint`);
+          }
+          const body = await response.text();
+          const text = parseJson3(body);
+
+          // Empty-body guard: HTTP 200 + empty body == silent PoToken failure.
+          if (!text) {
+            throw new Error('Empty transcript body (bot-protection rejected the token).');
+          }
+
+          return {
+            info,
+            track,
+            text,
+          };
+        },
+        {
+          maxRetries: 1,
+          initialDelay: 500,
+          maxDelay: 2000,
+          label: `youtube:${videoId}`,
+          isTransientError: (err) => !(err instanceof PermanentError),
+        },
+      ),
+    );
+
+    const basic = (result.info.basic_info ?? {}) as Record<string, unknown>;
+    metrics.observe('youtube_transcript_fetch_duration_ms', Date.now() - started, { status: 'success' });
+    metrics.increment('youtube_transcript_videos_total', 1, { status: 'success' });
+    return {
+      videoId,
+      url: watchUrl(videoId),
+      success: true,
+      title: typeof basic['title'] === 'string' ? (basic['title'] as string) : undefined,
+      author: typeof basic['author'] === 'string' ? (basic['author'] as string) : undefined,
+      durationSeconds: typeof basic['duration'] === 'number' ? (basic['duration'] as number) : undefined,
+      lang: result.track.language_code,
+      text: result.text,
+      charCount: result.text.length,
+    };
+  } catch (error) {
+    metrics.observe('youtube_transcript_fetch_duration_ms', Date.now() - started, { status: 'error' });
+    metrics.increment('youtube_transcript_videos_total', 1, { status: 'error' });
+    logger.warn(`[youtube] Transcript fetch failed for ${videoId}: ${msg(error)}`);
+    return failure(videoId, normalizeVideoError(error));
+  }
+}
+
+/** Select the best caption track: exact lang → lang-prefix → human (non-asr) → first. */
+function pickTrack(tracks: CaptionTrack[], lang: string): CaptionTrack | undefined {
+  if (tracks.length === 0) return undefined;
+  return (
+    tracks.find((t) => t.language_code === lang) ??
+    tracks.find((t) => t.language_code?.startsWith(lang)) ??
+    tracks.find((t) => t.kind !== 'asr') ??
+    tracks[0]
+  );
+}
+
+/** Parse YouTube json3 caption payload into a single whitespace-collapsed string. */
+/**
+ * Defense-in-depth SSRF gate for the timedtext URL. The base_url comes from
+ * youtubei.js parsing YouTube's getInfo() response; this guarantees we only ever
+ * fetch it if it is https on a YouTube/Google-owned host, so a manipulated or
+ * misparsed value can never point the PoToken-bearing request at an internal IP
+ * or the cloud metadata endpoint. Throws PermanentError (no retry) on mismatch.
+ */
+function assertTrustedTimedtextUrl(rawUrl: string): void {
+  let parsed: URL;
+  try {
+    parsed = new URL(rawUrl);
+  } catch {
+    throw new PermanentError('Caption track URL is not a valid URL.');
+  }
+  if (parsed.protocol !== 'https:') {
+    throw new PermanentError(`Caption track URL must be https (got ${parsed.protocol}).`);
+  }
+  const host = parsed.hostname.toLowerCase();
+  const ok =
+    host === 'youtube.com' ||
+    host.endsWith('.youtube.com') ||
+    host.endsWith('.googlevideo.com') ||
+    host.endsWith('.google.com') ||
+    host.endsWith('.googleapis.com');
+  if (!ok) {
+    throw new PermanentError(`Caption track URL host not trusted: ${host}`);
+  }
+}
+
+function parseJson3(body: string): string {
+  if (!body) return '';
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(body);
+  } catch {
+    // Log the real cause: an unparseable body is NOT necessarily an empty
+    // bot-protection response (the caller's default diagnosis) — it can be an HTML
+    // interstitial or a response-shape change. A short snippet disambiguates without
+    // dumping a huge/again-sensitive body.
+    logger.debug(`[youtube] timedtext body was not valid json3 (first 120 chars): ${body.slice(0, 120)}`);
+    return '';
+  }
+  const events = (parsed as { events?: unknown[] }).events;
+  if (!Array.isArray(events)) return '';
+  let out = '';
+  for (const event of events) {
+    const segs = (event as { segs?: unknown[] }).segs;
+    if (!Array.isArray(segs)) continue;
+    for (const seg of segs) {
+      const utf8 = (seg as { utf8?: unknown }).utf8;
+      if (typeof utf8 === 'string') out += utf8;
+    }
+  }
+  return out.replace(/\s+/g, ' ').trim();
+}
+
+function failure(videoId: string, error: string): VideoTranscript {
+  return { videoId, url: watchUrl(videoId), success: false, error };
+}
+
+function normalizeVideoError(error: unknown): string {
+  const m = msg(error);
+  if (/cancelled or timed out/i.test(m)) return 'Timed out fetching this video.';
+  return m;
+}
+
+function msg(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}
+
+/** Marks a non-retryable condition (e.g. captions disabled). */
+class PermanentError extends Error {}
+
+interface VideoInfoLike {
+  basic_info?: Record<string, unknown>;
+  captions?: { caption_tracks?: CaptionTrack[] };
+}