extract-youtube 1.0.1

package/src/errors.ts

@@ -0,0 +1,364 @@
+/**
+ * @fileoverview All error classes for the YouTube Transcript API
+ */
+
+import type { ProxyConfig } from './proxies';
+
+const YOUTUBE_WATCH_URL = 'https://www.youtube.com/watch?v={video_id}';
+
+/**
+ * Base exception class for all YouTube Transcript API errors.
+ */
+export class YouTubeTranscriptApiException extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'YouTubeTranscriptApiException';
+    Object.setPrototypeOf(this, YouTubeTranscriptApiException.prototype);
+  }
+}
+
+/**
+ * Base exception class for transcript retrieval errors.
+ */
+export class CouldNotRetrieveTranscript extends YouTubeTranscriptApiException {
+  private static readonly ERROR_MESSAGE = '\nCould not retrieve a transcript for the video {video_url}!';
+  private static readonly CAUSE_MESSAGE_INTRO = ' This is most likely caused by:\n\n{cause}';
+  private static readonly GITHUB_REFERRAL =
+    '\n\nIf you are sure that the described cause is not responsible for this error ' +
+    'and that a transcript should be retrievable, please create an issue at ' +
+    'https://github.com/jdepoix/youtube-transcript-api/issues. ' +
+    'Please add which version of youtube_transcript_api you are using ' +
+    'and provide the information needed to replicate the error. ' +
+    'Also make sure that there are no open issues which already describe your problem!';
+
+  public videoId: string;
+  protected causeMessage: string = '';
+
+  constructor(videoId: string) {
+    super('');
+    this.videoId = videoId;
+    Object.setPrototypeOf(this, CouldNotRetrieveTranscript.prototype);
+    this.message = this.buildErrorMessage();
+  }
+
+  protected buildErrorMessage(): string {
+    const videoUrl = YOUTUBE_WATCH_URL.replace('{video_id}', this.videoId);
+    let errorMessage = CouldNotRetrieveTranscript.ERROR_MESSAGE.replace('{video_url}', videoUrl);
+
+    const cause = this.getCause();
+    if (cause) {
+      errorMessage += CouldNotRetrieveTranscript.CAUSE_MESSAGE_INTRO.replace('{cause}', cause);
+      errorMessage += CouldNotRetrieveTranscript.GITHUB_REFERRAL;
+    }
+
+    return errorMessage;
+  }
+
+  protected getCause(): string {
+    return this.causeMessage;
+  }
+}
+
+// Cookie errors
+export class CookieError extends YouTubeTranscriptApiException {
+  constructor(message: string) {
+    super(message);
+    this.name = 'CookieError';
+    Object.setPrototypeOf(this, CookieError.prototype);
+  }
+}
+
+export class CookiePathInvalid extends CookieError {
+  constructor(cookiePath: string) {
+    super(`Can't load the provided cookie file: ${cookiePath}`);
+    this.name = 'CookiePathInvalid';
+    Object.setPrototypeOf(this, CookiePathInvalid.prototype);
+  }
+}
+
+export class CookieInvalid extends CookieError {
+  constructor(cookiePath: string) {
+    super(`The cookies provided are not valid (may have expired): ${cookiePath}`);
+    this.name = 'CookieInvalid';
+    Object.setPrototypeOf(this, CookieInvalid.prototype);
+  }
+}
+
+// Video errors
+export class YouTubeDataUnparsable extends CouldNotRetrieveTranscript {
+  protected causeMessage =
+    'The data required to fetch the transcript is not parsable. This should ' +
+    'not happen, please open an issue (make sure to include the video ID)!';
+
+  constructor(videoId: string) {
+    super(videoId);
+    this.name = 'YouTubeDataUnparsable';
+    Object.setPrototypeOf(this, YouTubeDataUnparsable.prototype);
+    this.message = this.buildErrorMessage();
+  }
+}
+
+export class YouTubeRequestFailed extends CouldNotRetrieveTranscript {
+  private httpErrorReason: string;
+
+  constructor(videoId: string, httpError: Error) {
+    super(videoId);
+    this.httpErrorReason = httpError.message;
+    this.name = 'YouTubeRequestFailed';
+    this.message = this.buildErrorMessage();
+    Object.setPrototypeOf(this, YouTubeRequestFailed.prototype);
+  }
+
+  protected getCause(): string {
+    return `Request to YouTube failed: ${this.httpErrorReason}`;
+  }
+}
+
+export class VideoUnplayable extends CouldNotRetrieveTranscript {
+  private unplayableReason: string | null;
+  private additionalDetails: string[];
+
+  constructor(videoId: string, unplayableReason: string | null, additionalDetails: string[]) {
+    super(videoId);
+    this.unplayableReason = unplayableReason;
+    this.additionalDetails = additionalDetails;
+    this.name = 'VideoUnplayable';
+    this.message = this.buildErrorMessage();
+    Object.setPrototypeOf(this, VideoUnplayable.prototype);
+  }
+
+  protected getCause(): string {
+    let reason = this.unplayableReason ?? 'No reason specified!';
+
+    if (this.additionalDetails.length > 0) {
+      const formattedDetails = this.additionalDetails.map(detail => ` - ${detail}`).join('\n');
+      reason = `${reason}\n\nAdditional Details:\n${formattedDetails}`;
+    }
+
+    return `The video is unplayable for the following reason: ${reason}`;
+  }
+}
+
+export class VideoUnavailable extends CouldNotRetrieveTranscript {
+  protected causeMessage = 'The video is no longer available';
+
+  constructor(videoId: string) {
+    super(videoId);
+    this.name = 'VideoUnavailable';
+    Object.setPrototypeOf(this, VideoUnavailable.prototype);
+    this.message = this.buildErrorMessage();
+  }
+}
+
+export class InvalidVideoId extends CouldNotRetrieveTranscript {
+  protected causeMessage =
+    'You provided an invalid video id. Make sure you are using the video id and NOT the url!\n\n' +
+    'Do NOT run: `new YouTubeTranscriptApi().fetch("https://www.youtube.com/watch?v=1234")`\n' +
+    'Instead run: `new YouTubeTranscriptApi().fetch("1234")`';
+
+  constructor(videoId: string) {
+    super(videoId);
+    this.name = 'InvalidVideoId';
+    Object.setPrototypeOf(this, InvalidVideoId.prototype);
+    this.message = this.buildErrorMessage();
+  }
+}
+
+export class AgeRestricted extends CouldNotRetrieveTranscript {
+  protected causeMessage =
+    'This video is age-restricted. Therefore, you are unable to retrieve ' +
+    'transcripts for it without authenticating yourself.\n\n' +
+    'Unfortunately, Cookie Authentication is temporarily unsupported in ' +
+    'youtube-transcript-api, as recent changes in YouTube\'s API broke the previous ' +
+    'implementation. I will do my best to re-implement it as soon as possible.';
+
+  constructor(videoId: string) {
+    super(videoId);
+    this.name = 'AgeRestricted';
+    Object.setPrototypeOf(this, AgeRestricted.prototype);
+    this.message = this.buildErrorMessage();
+  }
+}
+
+export class FailedToCreateConsentCookie extends CouldNotRetrieveTranscript {
+  protected causeMessage = 'Failed to automatically give consent to saving cookies';
+
+  constructor(videoId: string) {
+    super(videoId);
+    this.name = 'FailedToCreateConsentCookie';
+    Object.setPrototypeOf(this, FailedToCreateConsentCookie.prototype);
+    this.message = this.buildErrorMessage();
+  }
+}
+
+// Transcript errors
+export class TranscriptsDisabled extends CouldNotRetrieveTranscript {
+  protected causeMessage = 'Subtitles are disabled for this video';
+
+  constructor(videoId: string) {
+    super(videoId);
+    this.name = 'TranscriptsDisabled';
+    Object.setPrototypeOf(this, TranscriptsDisabled.prototype);
+    this.message = this.buildErrorMessage();
+  }
+}
+
+export class NotTranslatable extends CouldNotRetrieveTranscript {
+  protected causeMessage = 'The requested language is not translatable';
+
+  constructor(videoId: string) {
+    super(videoId);
+    this.name = 'NotTranslatable';
+    Object.setPrototypeOf(this, NotTranslatable.prototype);
+    this.message = this.buildErrorMessage();
+  }
+}
+
+export class TranslationLanguageNotAvailable extends CouldNotRetrieveTranscript {
+  protected causeMessage = 'The requested translation language is not available';
+
+  constructor(videoId: string) {
+    super(videoId);
+    this.name = 'TranslationLanguageNotAvailable';
+    Object.setPrototypeOf(this, TranslationLanguageNotAvailable.prototype);
+    this.message = this.buildErrorMessage();
+  }
+}
+
+export class NoTranscriptFound extends CouldNotRetrieveTranscript {
+  private requestedLanguageCodes: string[];
+  private availableTranscriptData: any;
+
+  constructor(videoId: string, requestedLanguageCodes: string[], availableTranscriptData: any) {
+    super(videoId);
+    this.requestedLanguageCodes = requestedLanguageCodes;
+    this.availableTranscriptData = availableTranscriptData;
+    this.name = 'NoTranscriptFound';
+    this.message = this.buildErrorMessage();
+    Object.setPrototypeOf(this, NoTranscriptFound.prototype);
+  }
+
+  protected getCause(): string {
+    const requestedLanguages = this.requestedLanguageCodes.join(', ');
+    return (
+      `No transcripts were found for any of the requested language codes: ${requestedLanguages}\n\n` +
+      this.availableTranscriptData.toString()
+    );
+  }
+}
+
+export class PoTokenRequired extends CouldNotRetrieveTranscript {
+  protected causeMessage =
+    'The requested video cannot be retrieved without a PO Token. If this happens, ' +
+    'please open a GitHub issue!';
+
+  constructor(videoId: string) {
+    super(videoId);
+    this.name = 'PoTokenRequired';
+    Object.setPrototypeOf(this, PoTokenRequired.prototype);
+    this.message = this.buildErrorMessage();
+  }
+}
+
+// Network errors
+export class RequestBlocked extends CouldNotRetrieveTranscript {
+  private static readonly BASE_CAUSE_MESSAGE =
+    'YouTube is blocking requests from your IP. This usually is due to one of the ' +
+    'following reasons:\n' +
+    '- You have done too many requests and your IP has been blocked by YouTube\n' +
+    '- You are doing requests from an IP belonging to a cloud provider (like AWS, ' +
+    'Google Cloud Platform, Azure, etc.). Unfortunately, most IPs from cloud ' +
+    'providers are blocked by YouTube.\n\n';
+
+  private static readonly DEFAULT_CAUSE_MESSAGE =
+    RequestBlocked.BASE_CAUSE_MESSAGE +
+    'There are two things you can do to work around this:\n' +
+    '1. Use proxies to hide your IP address, as explained in the "Working around ' +
+    'IP bans" section of the README ' +
+    '(https://github.com/jdepoix/youtube-transcript-api' +
+    '?tab=readme-ov-file' +
+    '#working-around-ip-bans-requestblocked-or-ipblocked-exception).\n' +
+    '2. (NOT RECOMMENDED) If you authenticate your requests using cookies, you ' +
+    'will be able to continue doing requests for a while. However, YouTube will ' +
+    'eventually permanently ban the account that you have used to authenticate ' +
+    'with! So only do this if you don\'t mind your account being banned!';
+
+  private static readonly WITH_GENERIC_PROXY_CAUSE_MESSAGE =
+    'YouTube is blocking your requests, despite you using proxies. Keep in mind ' +
+    'that a proxy is just a way to hide your real IP behind the IP of that proxy, ' +
+    'but there is no guarantee that the IP of that proxy won\'t be blocked as ' +
+    'well.\n\n' +
+    'The only truly reliable way to prevent IP blocks is rotating through a large ' +
+    'pool of residential IPs, by using a provider like Webshare ' +
+    '(https://www.webshare.io/?referral_code=w0xno53eb50g), which provides you ' +
+    'with a pool of >30M residential IPs (make sure to purchase ' +
+    '"Residential" proxies, NOT "Proxy Server" or "Static Residential"!).\n\n' +
+    'You will find more information on how to easily integrate Webshare here: ' +
+    'https://github.com/jdepoix/youtube-transcript-api' +
+    '?tab=readme-ov-file#using-webshare';
+
+  private static readonly WITH_WEBSHARE_PROXY_CAUSE_MESSAGE =
+    'YouTube is blocking your requests, despite you using Webshare proxies. ' +
+    'Please make sure that you have purchased "Residential" proxies and ' +
+    'NOT "Proxy Server" or "Static Residential", as those won\'t work as ' +
+    'reliably! The free tier also uses "Proxy Server" and will NOT work!\n\n' +
+    'The only reliable option is using "Residential" proxies (not "Static ' +
+    'Residential"), as this allows you to rotate through a pool of over 30M IPs, ' +
+    'which means you will always find an IP that hasn\'t been blocked by YouTube ' +
+    'yet!\n\n' +
+    'You can support the development of this open source project by making your ' +
+    'Webshare purchases through this affiliate link: ' +
+    'https://www.webshare.io/?referral_code=w0xno53eb50g \n\n' +
+    'Thank you for your support! <3';
+
+  private proxyConfig: ProxyConfig | null = null;
+
+  constructor(videoId: string) {
+    super(videoId);
+    this.name = 'RequestBlocked';
+    Object.setPrototypeOf(this, RequestBlocked.prototype);
+    this.message = this.buildErrorMessage();
+  }
+
+  withProxyConfig(proxyConfig: ProxyConfig | null): RequestBlocked {
+    this.proxyConfig = proxyConfig;
+    this.message = this.buildErrorMessage();
+    return this;
+  }
+
+  protected getCause(): string {
+    // Check class name to avoid circular dependency with instanceof
+    if (this.proxyConfig && this.proxyConfig.constructor.name === 'WebshareProxyConfig') {
+      return RequestBlocked.WITH_WEBSHARE_PROXY_CAUSE_MESSAGE;
+    }
+    if (this.proxyConfig && this.proxyConfig.constructor.name === 'GenericProxyConfig') {
+      return RequestBlocked.WITH_GENERIC_PROXY_CAUSE_MESSAGE;
+    }
+    return RequestBlocked.DEFAULT_CAUSE_MESSAGE;
+  }
+}
+
+export class IpBlocked extends RequestBlocked {
+  protected causeMessage =
+    'YouTube is blocking requests from your IP. This usually is due to one of the ' +
+    'following reasons:\n' +
+    '- You have done too many requests and your IP has been blocked by YouTube\n' +
+    '- You are doing requests from an IP belonging to a cloud provider (like AWS, ' +
+    'Google Cloud Platform, Azure, etc.). Unfortunately, most IPs from cloud ' +
+    'providers are blocked by YouTube.\n\n' +
+    'Ways to work around this are explained in the "Working around IP ' +
+    'bans" section of the README (https://github.com/jdepoix/youtube-transcript-api' +
+    '?tab=readme-ov-file' +
+    '#working-around-ip-bans-requestblocked-or-ipblocked-exception).\n';
+
+  constructor(videoId: string) {
+    super(videoId);
+    this.name = 'IpBlocked';
+    Object.setPrototypeOf(this, IpBlocked.prototype);
+    this.message = this.buildErrorMessage();
+  }
+
+  protected getCause(): string {
+    return this.causeMessage;
+  }
+}

package/src/fetchers/transcript-list-fetcher.ts

@@ -0,0 +1,252 @@
+/**
+ * @fileoverview Fetcher for retrieving transcript lists from YouTube
+ */
+
+import { decode } from 'html-entities';
+import { HttpClient, CaptionsJson, InnerTubeData } from '../types';
+import { ProxyConfig } from '../proxies';
+import { TranscriptList } from '../models/transcript-list';
+import { RequestBlocked, FailedToCreateConsentCookie } from '../errors';
+import { handleHttpErrors } from '../utils/http-error-handler';
+import {
+  extractInnertubeApiKey,
+  extractCaptionsJson
+} from './youtube-data-extractor';
+
+/** YouTube watch page URL template */
+const YOUTUBE_WATCH_URL = 'https://www.youtube.com/watch?v={video_id}';
+
+/** YouTube InnerTube API URL template */
+const INNERTUBE_API_URL = 'https://www.youtube.com/youtubei/v1/player?key={api_key}';
+
+/** InnerTube API context configuration for requests */
+const INNERTUBE_CONTEXT = {
+  client: {
+    clientName: 'ANDROID',
+    clientVersion: '20.10.38'
+  }
+};
+
+/**
+ * Fetches transcript lists from YouTube for given video IDs.
+ * Handles consent cookies, retries on blocks, and extracts caption data.
+ *
+ * @example
+ * ```typescript
+ * const fetcher = new TranscriptListFetcher(httpClient, proxyConfig);
+ * const transcriptList = await fetcher.fetchTranscriptList('dQw4w9WgXcQ');
+ * ```
+ */
+export class TranscriptListFetcher {
+  private httpClient: HttpClient;
+  private proxyConfig: ProxyConfig | null;
+
+  /**
+   * Creates a new TranscriptListFetcher instance.
+   *
+   * @param {HttpClient} httpClient - The HTTP client to use for requests
+   * @param {ProxyConfig | null} proxyConfig - Optional proxy configuration
+   */
+  constructor(httpClient: HttpClient, proxyConfig: ProxyConfig | null) {
+    this.httpClient = httpClient;
+    this.proxyConfig = proxyConfig;
+  }
+
+  /**
+   * Fetches the list of available transcripts for a video.
+   *
+   * @param {string} videoId - The YouTube video ID
+   * @returns {Promise<TranscriptList>} The transcript list
+   * @throws {RequestBlocked} If requests are blocked and retries exhausted
+   * @throws {IpBlocked} If IP is blocked by YouTube
+   * @throws {YouTubeDataUnparsable} If YouTube data cannot be parsed
+   * @throws {TranscriptsDisabled} If transcripts are disabled for the video
+   *
+   * @example
+   * ```typescript
+   * const transcriptList = await fetcher.fetchTranscriptList('dQw4w9WgXcQ');
+   * const transcript = transcriptList.findTranscript(['en']);
+   * ```
+   */
+  async fetchTranscriptList(videoId: string): Promise<TranscriptList> {
+    const captionsJson = await this.fetchCaptionsJsonWithRetry(videoId);
+    return TranscriptList.buildFromCaptionsJson(this.httpClient, videoId, captionsJson);
+  }
+
+  /**
+   * Fetches captions JSON with retry logic for blocked requests.
+   *
+   * @private
+   * @param {string} videoId - The video ID
+   * @param {number} [attemptNumber=0] - Current attempt number (for retry logic)
+   * @returns {Promise<CaptionsJson>} The captions JSON data
+   */
+  private async fetchCaptionsJsonWithRetry(
+    videoId: string,
+    attemptNumber: number = 0
+  ): Promise<CaptionsJson> {
+    try {
+      const htmlContent = await this.fetchVideoPageHtml(videoId);
+      const apiKey = extractInnertubeApiKey(htmlContent, videoId);
+      const innertubeData = await this.fetchInnertubePlayerData(videoId, apiKey);
+      return extractCaptionsJson(innertubeData, videoId);
+    } catch (error) {
+      if (error instanceof RequestBlocked) {
+        return this.handleBlockedRequest(error, videoId, attemptNumber);
+      }
+      throw error;
+    }
+  }
+
+  /**
+   * Handles blocked requests with retry logic based on proxy configuration.
+   *
+   * @private
+   * @param {RequestBlocked} error - The blocked request error
+   * @param {string} videoId - The video ID
+   * @param {number} attemptNumber - Current attempt number
+   * @returns {Promise<CaptionsJson>} The captions JSON data from retry
+   * @throws {RequestBlocked} If retries are exhausted
+   */
+  private async handleBlockedRequest(
+    error: RequestBlocked,
+    videoId: string,
+    attemptNumber: number
+  ): Promise<CaptionsJson> {
+    const maxRetries = this.proxyConfig?.retriesWhenBlocked || 0;
+
+    if (attemptNumber + 1 < maxRetries) {
+      return this.fetchCaptionsJsonWithRetry(videoId, attemptNumber + 1);
+    }
+
+    throw error.withProxyConfig(this.proxyConfig);
+  }
+
+  /**
+   * Fetches the HTML content of a video's watch page, handling consent cookies.
+   *
+   * @private
+   * @param {string} videoId - The video ID
+   * @returns {Promise<string>} The HTML content
+   * @throws {FailedToCreateConsentCookie} If consent cookie creation fails
+   */
+  private async fetchVideoPageHtml(videoId: string): Promise<string> {
+    let htmlContent = await this.fetchAndDecodeHtml(videoId);
+
+    // Handle YouTube consent cookie requirement
+    if (this.isConsentPageDetected(htmlContent)) {
+      this.createConsentCookie(htmlContent, videoId);
+      htmlContent = await this.fetchAndDecodeHtml(videoId);
+
+      if (this.isConsentPageDetected(htmlContent)) {
+        throw new FailedToCreateConsentCookie(videoId);
+      }
+    }
+
+    return htmlContent;
+  }
+
+  /**
+   * Checks if the HTML content is a consent page.
+   *
+   * @private
+   * @param {string} htmlContent - The HTML content to check
+   * @returns {boolean} True if consent page detected
+   */
+  private isConsentPageDetected(htmlContent: string): boolean {
+    return htmlContent.includes('action="https://consent.youtube.com/s"');
+  }
+
+  /**
+   * Fetches HTML from YouTube watch page and decodes HTML entities.
+   *
+   * @private
+   * @param {string} videoId - The video ID
+   * @returns {Promise<string>} The decoded HTML content
+   */
+  private async fetchAndDecodeHtml(videoId: string): Promise<string> {
+    const url = this.buildWatchUrl(videoId);
+    const response = await this.httpClient.get(url);
+    handleHttpErrors(response, videoId);
+    const rawHtml = await response.text();
+    return decode(rawHtml);
+  }
+
+  /**
+   * Builds the YouTube watch URL for a video ID.
+   *
+   * @private
+   * @param {string} videoId - The video ID
+   * @returns {string} The watch URL
+   */
+  private buildWatchUrl(videoId: string): string {
+    return YOUTUBE_WATCH_URL.replace('{video_id}', videoId);
+  }
+
+  /**
+   * Creates a consent cookie from the HTML content.
+   * Note: This is a placeholder - cookie handling should be implemented in the HTTP client.
+   *
+   * @private
+   * @param {string} htmlContent - The HTML content containing consent form
+   * @param {string} videoId - The video ID
+   * @throws {FailedToCreateConsentCookie} If consent value cannot be extracted
+   */
+  private createConsentCookie(htmlContent: string, videoId: string): void {
+    const consentValuePattern = /name="v" value="(.*?)"/;
+    const match = htmlContent.match(consentValuePattern);
+
+    if (!match) {
+      throw new FailedToCreateConsentCookie(videoId);
+    }
+
+    // TODO: Cookie handling would need to be implemented in the HTTP client
+    // This is a placeholder for the cookie setting logic
+  }
+
+  /**
+   * Fetches player data from YouTube's InnerTube API.
+   *
+   * @private
+   * @param {string} videoId - The video ID
+   * @param {string} apiKey - The InnerTube API key
+   * @returns {Promise<InnerTubeData>} The InnerTube player data
+   */
+  private async fetchInnertubePlayerData(videoId: string, apiKey: string): Promise<InnerTubeData> {
+    const url = this.buildInnertubeApiUrl(apiKey);
+    const requestBody = this.buildInnertubeRequestBody(videoId);
+
+    const response = await this.httpClient.post(url, {
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(requestBody)
+    });
+
+    handleHttpErrors(response, videoId);
+    return (await response.json()) as InnerTubeData;
+  }
+
+  /**
+   * Builds the InnerTube API URL with the provided API key.
+   *
+   * @private
+   * @param {string} apiKey - The API key
+   * @returns {string} The complete API URL
+   */
+  private buildInnertubeApiUrl(apiKey: string): string {
+    return INNERTUBE_API_URL.replace('{api_key}', apiKey);
+  }
+
+  /**
+   * Builds the request body for InnerTube API calls.
+   *
+   * @private
+   * @param {string} videoId - The video ID
+   * @returns {object} The request body
+   */
+  private buildInnertubeRequestBody(videoId: string): object {
+    return {
+      context: INNERTUBE_CONTEXT,
+      videoId: videoId
+    };
+  }
+}