extract-youtube 1.0.1

package/src/proxies.ts

@@ -0,0 +1,222 @@
+/**
+ * @fileoverview Proxy configuration classes for the YouTube Transcript API
+ */
+
+import { RequestsProxyConfigDict } from './types';
+
+/**
+ * Exception for invalid proxy configurations
+ */
+export class InvalidProxyConfig extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'InvalidProxyConfig';
+    Object.setPrototypeOf(this, InvalidProxyConfig.prototype);
+  }
+}
+
+/**
+ * Base class for all proxy configurations.
+ * Any proxy config can be used as long as it can be converted to a RequestsProxyConfigDict.
+ */
+export abstract class ProxyConfig {
+  /**
+   * Converts this proxy config to a dictionary that can be used by HTTP clients.
+   * @returns {RequestsProxyConfigDict} The proxy configuration dictionary
+   */
+  abstract toRequestsDict(): RequestsProxyConfigDict;
+
+  /**
+   * If you are using rotating proxies, it can be useful to prevent the HTTP
+   * client from keeping TCP connections alive, as your IP won't be rotated on
+   * every request if your connection stays open.
+   * @returns {boolean} Whether to prevent keeping connections alive
+   */
+  get preventKeepingConnectionsAlive(): boolean {
+    return false;
+  }
+
+  /**
+   * Defines how many times we should retry if a request is blocked. When using
+   * rotating residential proxies with a large IP pool it can make sense to retry a
+   * couple of times when a blocked IP is encountered, since a retry will trigger
+   * an IP rotation and the next IP might not be blocked.
+   * @returns {number} Number of retries when blocked
+   */
+  get retriesWhenBlocked(): number {
+    return 0;
+  }
+}
+
+/**
+ * Generic proxy configuration that can be used to set up any HTTP/HTTPS/SOCKS proxy.
+ *
+ * If only an HTTP or an HTTPS proxy is provided, it will be used for both types of
+ * connections. However, you will have to provide at least one of the two.
+ *
+ * @example
+ * ```typescript
+ * const proxyConfig = new GenericProxyConfig({
+ *   httpUrl: 'http://user:pass@proxy.example.com:8080',
+ *   httpsUrl: 'https://user:pass@proxy.example.com:8080'
+ * });
+ * ```
+ */
+export class GenericProxyConfig extends ProxyConfig {
+  protected _httpUrl: string | null;
+  protected _httpsUrl: string | null;
+
+  /**
+   * Creates a generic proxy configuration.
+   * If only an HTTP or an HTTPS proxy is provided, it will be used for both types of
+   * connections. However, you will have to provide at least one of the two.
+   *
+   * @param {Object} options - Proxy configuration options
+   * @param {string} [options.httpUrl] - The proxy URL used for HTTP requests. Defaults to httpsUrl if not provided.
+   * @param {string} [options.httpsUrl] - The proxy URL used for HTTPS requests. Defaults to httpUrl if not provided.
+   * @throws {InvalidProxyConfig} If neither httpUrl nor httpsUrl is provided
+   */
+  constructor(options: { httpUrl?: string; httpsUrl?: string }) {
+    super();
+    const { httpUrl, httpsUrl } = options;
+
+    if (!httpUrl && !httpsUrl) {
+      throw new InvalidProxyConfig(
+        'GenericProxyConfig requires you to define at least one of the two: http or https'
+      );
+    }
+
+    this._httpUrl = httpUrl || null;
+    this._httpsUrl = httpsUrl || null;
+  }
+
+  get httpUrl(): string | null {
+    return this._httpUrl;
+  }
+
+  get httpsUrl(): string | null {
+    return this._httpsUrl;
+  }
+
+  toRequestsDict(): RequestsProxyConfigDict {
+    return {
+      http: this.httpUrl || this.httpsUrl!,
+      https: this.httpsUrl || this.httpUrl!
+    };
+  }
+}
+
+/**
+ * Webshare proxy configuration for rotating residential proxies.
+ *
+ * Webshare is a provider offering rotating residential proxies, which is the
+ * most reliable way to work around being blocked by YouTube.
+ *
+ * If you don't have a Webshare account yet, you will have to create one
+ * at https://www.webshare.io/?referral_code=w0xno53eb50g and purchase a "Residential"
+ * proxy package that suits your workload (make sure NOT to purchase "Proxy Server"
+ * or "Static Residential"!).
+ *
+ * Once you have created an account you only need the "Proxy Username" and
+ * "Proxy Password" that you can find in your Webshare settings to set up this config.
+ *
+ * @example
+ * ```typescript
+ * const proxyConfig = new WebshareProxyConfig({
+ *   proxyUsername: 'your-username',
+ *   proxyPassword: 'your-password',
+ *   filterIpLocations: ['us', 'de']
+ * });
+ * ```
+ */
+export class WebshareProxyConfig extends GenericProxyConfig {
+  public static readonly DEFAULT_DOMAIN_NAME = 'p.webshare.io';
+  public static readonly DEFAULT_PORT = 80;
+
+  public readonly proxyUsername: string;
+  public readonly proxyPassword: string;
+  public readonly domainName: string;
+  public readonly proxyPort: number;
+  private readonly filterIpLocations: string[];
+  private readonly _retriesWhenBlocked: number;
+
+  /**
+   * Creates a Webshare proxy configuration.
+   *
+   * Once you have created a Webshare account and purchased a "Residential" package
+   * (make sure NOT to purchase "Proxy Server" or "Static Residential"!), this config
+   * class allows you to easily use it by defaulting to the most reliable proxy settings
+   * (rotating residential proxies).
+   *
+   * @param {Object} options - Webshare configuration options
+   * @param {string} options.proxyUsername - "Proxy Username" found at https://dashboard.webshare.io/proxy/settings
+   * @param {string} options.proxyPassword - "Proxy Password" found at https://dashboard.webshare.io/proxy/settings
+   * @param {string[]} [options.filterIpLocations] - If you want to limit the pool of IPs to specific countries,
+   *   provide a list of location codes (e.g., ['us', 'de']). This can reduce latency and work around location-based restrictions.
+   * @param {number} [options.retriesWhenBlocked=10] - How many times to retry if a request is blocked
+   * @param {string} [options.domainName] - Custom domain name (defaults to p.webshare.io)
+   * @param {number} [options.proxyPort] - Custom proxy port (defaults to 80)
+   */
+  constructor(options: {
+    proxyUsername: string;
+    proxyPassword: string;
+    filterIpLocations?: string[];
+    retriesWhenBlocked?: number;
+    domainName?: string;
+    proxyPort?: number;
+  }) {
+    super({ httpUrl: 'placeholder' });
+
+    this.proxyUsername = options.proxyUsername;
+    this.proxyPassword = options.proxyPassword;
+    this.domainName = options.domainName || WebshareProxyConfig.DEFAULT_DOMAIN_NAME;
+    this.proxyPort = options.proxyPort || WebshareProxyConfig.DEFAULT_PORT;
+    this.filterIpLocations = options.filterIpLocations || [];
+    this._retriesWhenBlocked = options.retriesWhenBlocked ?? 10;
+  }
+
+  /**
+   * Gets the full proxy URL with location filtering and rotation settings.
+   * @returns {string} The complete proxy URL
+   */
+  get url(): string {
+    const locationCodes = this.filterIpLocations
+      .map(code => `-${code.toUpperCase()}`)
+      .join('');
+
+    let username = this.proxyUsername;
+    const suffix = '-rotate';
+
+    if (username.endsWith(suffix)) {
+      username = username.slice(0, -suffix.length);
+    }
+
+    return (
+      `http://${username}${locationCodes}${suffix}:${this.proxyPassword}` +
+      `@${this.domainName}:${this.proxyPort}/`
+    );
+  }
+
+  get httpUrl(): string {
+    return this.url;
+  }
+
+  get httpsUrl(): string {
+    return this.url;
+  }
+
+  toRequestsDict(): RequestsProxyConfigDict {
+    return {
+      http: this.url,
+      https: this.url
+    };
+  }
+
+  get preventKeepingConnectionsAlive(): boolean {
+    return true;
+  }
+
+  get retriesWhenBlocked(): number {
+    return this._retriesWhenBlocked;
+  }
+}

package/src/types.ts

@@ -0,0 +1,119 @@
+/**
+ * @fileoverview Type definitions for the YouTube Transcript API
+ */
+
+import { Response, RequestInit } from 'node-fetch';
+
+/**
+ * Represents a single snippet/cue of a transcript
+ */
+export interface FetchedTranscriptSnippet {
+  /** The text content of this transcript snippet */
+  text: string;
+  /** The timestamp at which this transcript snippet appears on screen in seconds */
+  start: number;
+  /**
+   * The duration of how long the snippet stays on screen in seconds.
+   * Note: This is not the duration of the transcribed speech, but how long
+   * the snippet stays on screen. Therefore, there can be overlaps between snippets!
+   */
+  duration: number;
+}
+
+/**
+ * Translation language information
+ */
+export interface TranslationLanguage {
+  /** The name of the language */
+  language: string;
+  /** The language code (e.g., 'en', 'de', 'fr') */
+  language_code: string;
+}
+
+/**
+ * Playability status enum values
+ */
+export enum PlayabilityStatus {
+  OK = 'OK',
+  ERROR = 'ERROR',
+  LOGIN_REQUIRED = 'LOGIN_REQUIRED'
+}
+
+/**
+ * Playability failed reason enum values
+ */
+export enum PlayabilityFailedReason {
+  BOT_DETECTED = "Sign in to confirm you're not a bot",
+  AGE_RESTRICTED = 'This video may be inappropriate for some users.',
+  VIDEO_UNAVAILABLE = 'This video is unavailable'
+}
+
+/**
+ * Proxy configuration dictionary for HTTP clients
+ */
+export interface RequestsProxyConfigDict {
+  http: string;
+  https: string;
+}
+
+/**
+ * Caption track data from YouTube API
+ */
+export interface CaptionTrack {
+  baseUrl: string;
+  name: {
+    runs: Array<{ text: string }>;
+  };
+  languageCode: string;
+  kind?: string;
+  isTranslatable?: boolean;
+}
+
+/**
+ * Captions JSON data from YouTube
+ */
+export interface CaptionsJson {
+  captionTracks: CaptionTrack[];
+  translationLanguages?: Array<{
+    languageName: {
+      runs: Array<{ text: string }>;
+    };
+    languageCode: string;
+  }>;
+}
+
+/**
+ * YouTube InnerTube API response
+ */
+export interface InnerTubeData {
+  playabilityStatus?: {
+    status?: string;
+    reason?: string;
+    errorScreen?: {
+      playerErrorMessageRenderer?: {
+        subreason?: {
+          runs?: Array<{ text?: string }>;
+        };
+      };
+    };
+  };
+  captions?: {
+    playerCaptionsTracklistRenderer?: CaptionsJson;
+  };
+}
+
+/**
+ * HTTP client interface with fetch-like API
+ */
+export interface HttpClient {
+  get(url: string, options?: RequestInit): Promise<Response>;
+  post(url: string, options?: RequestInit): Promise<Response>;
+}
+
+/**
+ * Cookie storage interface
+ */
+export interface CookieJar {
+  set(name: string, value: string, domain: string): void;
+  get(name: string): string | undefined;
+}

package/src/utils/http-error-handler.ts

@@ -0,0 +1,32 @@
+/**
+ * @fileoverview HTTP error handling utilities for YouTube API requests
+ */
+
+import { Response } from 'node-fetch';
+import { IpBlocked, YouTubeRequestFailed } from '../errors';
+
+/**
+ * Checks an HTTP response and throws appropriate errors if the request failed.
+ *
+ * @param {Response} response - The HTTP response to check
+ * @param {string} videoId - The video ID associated with the request
+ * @throws {IpBlocked} If status is 429 (Too Many Requests)
+ * @throws {YouTubeRequestFailed} If status indicates an error
+ *
+ * @example
+ * ```typescript
+ * const response = await httpClient.get(url);
+ * handleHttpErrors(response, videoId);
+ * // If we reach here, the request was successful
+ * ```
+ */
+export function handleHttpErrors(response: Response, videoId: string): void {
+  if (response.status === 429) {
+    throw new IpBlocked(videoId);
+  }
+
+  if (!response.ok) {
+    const errorMessage = `HTTP ${response.status}: ${response.statusText}`;
+    throw new YouTubeRequestFailed(videoId, new Error(errorMessage));
+  }
+}

package/src/youtube-transcript-api.ts

@@ -0,0 +1,254 @@
+/**
+ * @fileoverview Main API class for the YouTube Transcript API
+ */
+
+import { ProxyConfig } from './proxies';
+import { FetchedTranscript, TranscriptList } from './models';
+import { TranscriptListFetcher } from './fetchers/transcript-list-fetcher';
+import { HttpClient } from './types';
+import { FetchHttpClient } from './http/fetch-http-client';
+
+/**
+ * Main API class for retrieving YouTube transcripts.
+ * This is the primary entry point for the library.
+ *
+ * @example
+ * ```typescript
+ * // Basic usage
+ * const api = new YouTubeTranscriptApi();
+ * const transcript = await api.fetchTranscript('video_id');
+ *
+ * // With language preference
+ * const transcript = await api.fetchTranscript('video_id', { languages: ['de', 'en'] });
+ *
+ * // With proxy
+ * import { GenericProxyConfig } from 'youtube-transcript-api';
+ * const api = new YouTubeTranscriptApi({
+ *   proxyConfig: new GenericProxyConfig({
+ *     httpUrl: 'http://user:pass@proxy.example.com:8080'
+ *   })
+ * });
+ * ```
+ */
+export class YouTubeTranscriptApi {
+  private transcriptListFetcher: TranscriptListFetcher;
+
+  /**
+   * Creates a new YouTubeTranscriptApi instance.
+   *
+   * Note on thread-safety: As this class initializes an HTTP client,
+   * it is not thread-safe. Make sure to initialize an instance per
+   * thread if used in a multi-threading scenario!
+   *
+   * @param {Object} [options] - Configuration options
+   * @param {ProxyConfig} [options.proxyConfig] - An optional ProxyConfig object defining proxies
+   *   used for all network requests. This can be used to work around your IP being blocked
+   *   by YouTube, as described in the "Working around IP bans" section of the README.
+   * @param {HttpClient} [options.httpClient] - You can optionally pass in a custom HTTP client
+   *   if you want to share state between different instances of YouTubeTranscriptApi or
+   *   customize request behavior.
+   *
+   * @example
+   * ```typescript
+   * // Basic usage
+   * const api = new YouTubeTranscriptApi();
+   *
+   * // With Webshare proxy
+   * import { WebshareProxyConfig } from 'youtube-transcript-api';
+   * const api = new YouTubeTranscriptApi({
+   *   proxyConfig: new WebshareProxyConfig({
+   *     proxyUsername: 'your-username',
+   *     proxyPassword: 'your-password',
+   *     filterIpLocations: ['us', 'de']
+   *   })
+   * });
+   *
+   * // With generic proxy
+   * import { GenericProxyConfig } from 'youtube-transcript-api';
+   * const api = new YouTubeTranscriptApi({
+   *   proxyConfig: new GenericProxyConfig({
+   *     httpUrl: 'http://user:pass@proxy.example.com:8080',
+   *     httpsUrl: 'https://user:pass@proxy.example.com:8080'
+   *   })
+   * });
+   * ```
+   */
+  constructor(options?: { proxyConfig?: ProxyConfig; httpClient?: HttpClient }) {
+    const httpClient = this.initializeHttpClient(options);
+    this.configureHttpClientWithProxy(httpClient, options?.proxyConfig);
+    this.transcriptListFetcher = new TranscriptListFetcher(httpClient, options?.proxyConfig || null);
+  }
+
+  /**
+   * Initializes the HTTP client from options or creates a default one.
+   *
+   * @private
+   * @param {Object} [options] - Configuration options
+   * @returns {HttpClient} The initialized HTTP client
+   */
+  private initializeHttpClient(options?: { httpClient?: HttpClient }): HttpClient {
+    return options?.httpClient || new FetchHttpClient();
+  }
+
+  /**
+   * Configures the HTTP client with proxy settings if provided.
+   *
+   * @private
+   * @param {HttpClient} httpClient - The HTTP client to configure
+   * @param {ProxyConfig} [proxyConfig] - Optional proxy configuration
+   */
+  private configureHttpClientWithProxy(httpClient: HttpClient, proxyConfig?: ProxyConfig): void {
+    if (!proxyConfig || !(httpClient instanceof FetchHttpClient)) {
+      return;
+    }
+
+    const proxyDictionary = proxyConfig.toRequestsDict();
+    httpClient.setProxies(proxyDictionary);
+
+    if (proxyConfig.preventKeepingConnectionsAlive) {
+      httpClient.setHeader('Connection', 'close');
+    }
+
+    // Note: Retry logic for 429 status codes with configured retries
+    // would need to be implemented in the HTTP client or fetcher
+  }
+
+  /**
+   * Retrieves the transcript for a single video.
+   * This is a shortcut for calling:
+   * `api.listTranscripts(videoId).then(list => list.findTranscript(languages)).then(t => t.fetch())`
+   *
+   * @param {string} videoId - The ID of the video you want to retrieve the transcript for.
+   *   Make sure that this is the actual ID, NOT the full URL to the video!
+   * @param {Object} [options] - Fetch options
+   * @param {string[]} [options.languages=['en']] - A list of language codes in descending priority.
+   *   For example, if this is set to ['de', 'en'] it will first try to fetch the german
+   *   transcript (de) and then fetch the english transcript (en) if it fails to do so.
+   * @param {boolean} [options.preserveFormatting=false] - Whether to keep select HTML text formatting
+   * @returns {Promise<FetchedTranscript>} The fetched transcript
+   *
+   * @example
+   * ```typescript
+   * const api = new YouTubeTranscriptApi();
+   *
+   * // Fetch English transcript
+   * const transcript = await api.fetchTranscript('dQw4w9WgXcQ');
+   *
+   * // Fetch with language preference
+   * const transcript = await api.fetchTranscript('dQw4w9WgXcQ', {
+   *   languages: ['de', 'en']
+   * });
+   *
+   * // Preserve HTML formatting
+   * const transcript = await api.fetchTranscript('dQw4w9WgXcQ', {
+   *   preserveFormatting: true
+   * });
+   *
+   * // Iterate over snippets
+   * for (const snippet of transcript) {
+   *   console.log(`${snippet.start}s: ${snippet.text}`);
+   * }
+   * ```
+   */
+  async fetchTranscript(
+    videoId: string,
+    options?: {
+      languages?: string[];
+      preserveFormatting?: boolean;
+    }
+  ): Promise<FetchedTranscript> {
+    const preferredLanguages = options?.languages || ['en'];
+    const preserveFormatting = options?.preserveFormatting || false;
+
+    const transcriptList = await this.listTranscripts(videoId);
+    const transcript = transcriptList.findTranscript(preferredLanguages);
+    return transcript.fetch(preserveFormatting);
+  }
+
+  /**
+   * Retrieves the list of transcripts which are available for a given video.
+   * It returns a TranscriptList object which is iterable and provides methods to
+   * filter the list of transcripts for specific languages.
+   *
+   * While iterating over the TranscriptList, the individual transcripts are
+   * represented by Transcript objects, which provide metadata and can either
+   * be fetched by calling transcript.fetch() or translated by calling
+   * transcript.translate('en').
+   *
+   * @param {string} videoId - The ID of the video you want to retrieve the transcript for.
+   *   Make sure that this is the actual ID, NOT the full URL to the video!
+   * @returns {Promise<TranscriptList>} A list of available transcripts
+   *
+   * @example
+   * ```typescript
+   * const api = new YouTubeTranscriptApi();
+   *
+   * // Retrieve available transcripts
+   * const transcriptList = await api.listTranscripts('video_id');
+   *
+   * // Iterate over all available transcripts
+   * for (const transcript of transcriptList) {
+   *   console.log(
+   *     transcript.videoId,
+   *     transcript.language,
+   *     transcript.languageCode,
+   *     transcript.isGenerated,
+   *     transcript.isTranslatable,
+   *     transcript.translationLanguages
+   *   );
+   *
+   *   // Fetch the actual transcript data
+   *   const fetched = await transcript.fetch();
+   *   console.log(fetched);
+   *
+   *   // Translate the transcript
+   *   if (transcript.isTranslatable) {
+   *     const translated = transcript.translate('de');
+   *     const fetchedTranslated = await translated.fetch();
+   *     console.log(fetchedTranslated);
+   *   }
+   * }
+   *
+   * // Filter for specific language
+   * const transcript = transcriptList.findTranscript(['de', 'en']);
+   *
+   * // Filter for manually created transcripts
+   * const manual = transcriptList.findManuallyCreatedTranscript(['de', 'en']);
+   *
+   * // Filter for automatically generated transcripts
+   * const generated = transcriptList.findGeneratedTranscript(['de', 'en']);
+   * ```
+   */
+  async listTranscripts(videoId: string): Promise<TranscriptList> {
+    return this.transcriptListFetcher.fetchTranscriptList(videoId);
+  }
+
+  /**
+   * Alias for fetchTranscript() for backward compatibility.
+   *
+   * @deprecated Use fetchTranscript() instead
+   * @param {string} videoId - The video ID
+   * @param {Object} [options] - Fetch options
+   * @returns {Promise<FetchedTranscript>} The fetched transcript
+   */
+  async fetch(
+    videoId: string,
+    options?: {
+      languages?: string[];
+      preserveFormatting?: boolean;
+    }
+  ): Promise<FetchedTranscript> {
+    return this.fetchTranscript(videoId, options);
+  }
+
+  /**
+   * Alias for listTranscripts() for backward compatibility.
+   *
+   * @deprecated Use listTranscripts() instead
+   * @param {string} videoId - The video ID
+   * @returns {Promise<TranscriptList>} A list of available transcripts
+   */
+  async list(videoId: string): Promise<TranscriptList> {
+    return this.listTranscripts(videoId);
+  }
+}

package/tsconfig.json

@@ -0,0 +1,20 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "module": "commonjs",
+    "lib": ["ES2020"],
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true,
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "moduleResolution": "node",
+    "resolveJsonModule": true
+  },
+  "include": ["src/**/*", "__tests__"],
+  "exclude": ["node_modules", "dist", "src/**/*.test.ts", "__tests__"]
+}