youtube-transcript-rb 0.1.0 → 0.2.3

data/lib/youtube_rb/transcript/api.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+require "faraday"
+require "faraday/follow_redirects"
+
+module YoutubeRb
+  module Transcript
+    # Main entry point for fetching YouTube transcripts.
+    # This class provides a simple API for retrieving transcripts from YouTube videos.
+    #
+    # @example Basic usage
+    #   api = YoutubeRb::Transcript::YouTubeTranscriptApi.new
+    #   transcript = api.fetch("dQw4w9WgXcQ")
+    #   transcript.each { |snippet| puts snippet.text }
+    #
+    # @example With language preference
+    #   api = YoutubeRb::Transcript::YouTubeTranscriptApi.new
+    #   transcript = api.fetch("dQw4w9WgXcQ", languages: ["es", "en"])
+    #
+    # @example Listing available transcripts
+    #   api = YoutubeRb::Transcript::YouTubeTranscriptApi.new
+    #   transcript_list = api.list("dQw4w9WgXcQ")
+    #   transcript_list.each { |t| puts t }
+    #
+    class YouTubeTranscriptApi
+      # Default timeout for HTTP requests in seconds
+      DEFAULT_TIMEOUT = 30
+
+      # @param http_client [Faraday::Connection, nil] Custom HTTP client (optional)
+      # @param proxy_config [Object, nil] Proxy configuration (optional)
+      def initialize(http_client: nil, proxy_config: nil)
+        @http_client = http_client || build_default_http_client
+        @proxy_config = proxy_config
+        @fetcher = TranscriptListFetcher.new(
+          http_client: @http_client,
+          proxy_config: @proxy_config
+        )
+      end
+
+      # Fetch a transcript for a video.
+      # This is a convenience method that combines `list` and `find_transcript`.
+      #
+      # @param video_id [String] The YouTube video ID
+      # @param languages [Array<String>] Language codes in order of preference (default: ["en"])
+      # @param preserve_formatting [Boolean] Whether to preserve HTML formatting (default: false)
+      # @return [FetchedTranscript] The fetched transcript
+      # @raise [NoTranscriptFound] If no transcript matches the requested languages
+      # @raise [TranscriptsDisabled] If transcripts are disabled for the video
+      # @raise [VideoUnavailable] If the video is not available
+      #
+      # @example
+      #   api = YouTubeTranscriptApi.new
+      #   transcript = api.fetch("dQw4w9WgXcQ", languages: ["en", "es"])
+      #   puts transcript.first.text
+      #
+      def fetch(video_id, languages: ["en"], preserve_formatting: false)
+        list(video_id)
+          .find_transcript(languages)
+          .fetch(preserve_formatting: preserve_formatting)
+      end
+
+      # List all available transcripts for a video.
+      #
+      # @param video_id [String] The YouTube video ID
+      # @return [TranscriptList] A list of available transcripts
+      # @raise [TranscriptsDisabled] If transcripts are disabled for the video
+      # @raise [VideoUnavailable] If the video is not available
+      #
+      # @example
+      #   api = YouTubeTranscriptApi.new
+      #   transcript_list = api.list("dQw4w9WgXcQ")
+      #
+      #   # Find a specific transcript
+      #   transcript = transcript_list.find_transcript(["en"])
+      #
+      #   # Or iterate over all available transcripts
+      #   transcript_list.each do |transcript|
+      #     puts "#{transcript.language_code}: #{transcript.language}"
+      #   end
+      #
+      def list(video_id)
+        @fetcher.fetch(video_id)
+      end
+
+      # Fetch transcripts for multiple videos.
+      #
+      # @param video_ids [Array<String>] Array of YouTube video IDs
+      # @param languages [Array<String>] Language codes in order of preference (default: ["en"])
+      # @param preserve_formatting [Boolean] Whether to preserve HTML formatting (default: false)
+      # @param continue_on_error [Boolean] Whether to continue if a video fails (default: false)
+      # @yield [video_id, result] Block called for each video with either transcript or error
+      # @yieldparam video_id [String] The video ID being processed
+      # @yieldparam result [FetchedTranscript, StandardError] The transcript or error
+      # @return [Hash<String, FetchedTranscript>] Hash mapping video IDs to transcripts
+      # @raise [CouldNotRetrieveTranscript] If any video fails and continue_on_error is false
+      #
+      # @example Fetch multiple videos
+      #   api = YouTubeTranscriptApi.new
+      #   transcripts = api.fetch_all(["video1", "video2", "video3"])
+      #   transcripts.each { |id, t| puts "#{id}: #{t.length} snippets" }
+      #
+      # @example With error handling
+      #   api = YouTubeTranscriptApi.new
+      #   api.fetch_all(["video1", "video2"], continue_on_error: true) do |video_id, result|
+      #     if result.is_a?(StandardError)
+      #       puts "Error for #{video_id}: #{result.message}"
+      #     else
+      #       puts "Got #{result.length} snippets for #{video_id}"
+      #     end
+      #   end
+      #
+      def fetch_all(video_ids, languages: ["en"], preserve_formatting: false, continue_on_error: false)
+        results = {}
+
+        video_ids.each do |video_id|
+          transcript = fetch(video_id, languages: languages, preserve_formatting: preserve_formatting)
+          results[video_id] = transcript
+          yield(video_id, transcript) if block_given?
+        rescue CouldNotRetrieveTranscript => e
+          raise unless continue_on_error
+
+          yield(video_id, e) if block_given?
+        end
+
+        results
+      end
+
+      private
+
+      # Build the default Faraday HTTP client
+      #
+      # @return [Faraday::Connection] The configured HTTP client
+      def build_default_http_client
+        Faraday.new do |conn|
+          conn.options.timeout = DEFAULT_TIMEOUT
+          conn.options.open_timeout = DEFAULT_TIMEOUT
+          conn.request :url_encoded
+          conn.response :follow_redirects
+          conn.adapter Faraday.default_adapter
+        end
+      end
+    end
+  end
+end

data/lib/youtube_rb/transcript/errors.rb

@@ -0,0 +1,215 @@
+# frozen_string_literal: true
+
+module YoutubeRb
+  module Transcript
+    # Base error class for all YouTube Transcript errors
+    class Error < StandardError; end
+
+    # Raised when a transcript could not be retrieved
+    class CouldNotRetrieveTranscript < Error
+      WATCH_URL = "https://www.youtube.com/watch?v=%<video_id>s"
+
+      # @return [String] the video ID that caused the error
+      attr_reader :video_id
+
+      # @param video_id [String] the YouTube video ID
+      def initialize(video_id)
+        @video_id = video_id
+        super(build_error_message)
+      end
+
+      # @return [String] the cause of the error
+      def cause_message
+        self.class::CAUSE_MESSAGE
+      end
+
+      private
+
+      def build_error_message
+        video_url = format(WATCH_URL, video_id: @video_id)
+        message = "\nCould not retrieve a transcript for the video #{video_url}!"
+
+        if cause_message && !cause_message.empty?
+          message += " This is most likely caused by:\n\n#{cause_message}"
+          message += github_referral
+        end
+
+        message
+      end
+
+      def github_referral
+        "\n\nIf you are sure that the described cause is not responsible for this error " \
+          "and that a transcript should be retrievable, please create an issue at " \
+          "https://github.com/stadia/youtube-transcript-rb/issues. " \
+          "Please add which version of youtube_transcript_api you are using " \
+          "and provide the information needed to replicate the error. " \
+          "Also make sure that there are no open issues which already describe your problem!"
+      end
+    end
+
+    # Raised when YouTube data cannot be parsed
+    class YouTubeDataUnparsable < CouldNotRetrieveTranscript
+      CAUSE_MESSAGE = "The data required to fetch the transcript is not parsable. This should " \
+                      "not happen, please open an issue (make sure to include the video ID)!"
+    end
+
+    # Raised when a request to YouTube fails
+    class YouTubeRequestFailed < CouldNotRetrieveTranscript
+      CAUSE_MESSAGE = "Request to YouTube failed: %<reason>s"
+
+      # @return [String] the reason for the failure
+      attr_reader :reason
+
+      # @param video_id [String] the YouTube video ID
+      # @param http_error [StandardError] the HTTP error that occurred
+      def initialize(video_id, http_error)
+        @reason = http_error.to_s
+        super(video_id)
+      end
+
+      def cause_message
+        format(CAUSE_MESSAGE, reason: @reason)
+      end
+    end
+
+    # Raised when a video is unplayable
+    class VideoUnplayable < CouldNotRetrieveTranscript
+      CAUSE_MESSAGE = "The video is unplayable for the following reason: %<reason>s"
+
+      # @return [String, nil] the reason the video is unplayable
+      attr_reader :reason
+
+      # @return [Array<String>] additional sub-reasons
+      attr_reader :sub_reasons
+
+      # @param video_id [String] the YouTube video ID
+      # @param reason [String, nil] the reason the video is unplayable
+      # @param sub_reasons [Array<String>] additional details
+      def initialize(video_id, reason = nil, sub_reasons = [])
+        @reason = reason
+        @sub_reasons = sub_reasons
+        super(video_id)
+      end
+
+      def cause_message
+        reason_text = @reason || "No reason specified!"
+
+        if @sub_reasons.any?
+          sub_reasons_text = @sub_reasons.map { |r| " - #{r}" }.join("\n")
+          reason_text = "#{reason_text}\n\nAdditional Details:\n#{sub_reasons_text}"
+        end
+
+        format(CAUSE_MESSAGE, reason: reason_text)
+      end
+    end
+
+    # Raised when a video is unavailable
+    class VideoUnavailable < CouldNotRetrieveTranscript
+      CAUSE_MESSAGE = "The video is no longer available"
+    end
+
+    # Raised when an invalid video ID is provided
+    class InvalidVideoId < CouldNotRetrieveTranscript
+      CAUSE_MESSAGE = "You provided an invalid video id. Make sure you are using the video id and NOT the url!\n\n" \
+                      'Do NOT run: `YoutubeRb::Transcript.fetch("https://www.youtube.com/watch?v=1234")`' \
+                      "\n" \
+                      'Instead run: `YoutubeRb::Transcript.fetch("1234")`'
+    end
+
+    # Raised when YouTube blocks the request
+    class RequestBlocked < CouldNotRetrieveTranscript
+      BASE_CAUSE_MESSAGE = "YouTube is blocking requests from your IP. This usually is due to one of the " \
+                           "following reasons:\n" \
+                           "- You have done too many requests and your IP has been blocked by YouTube\n" \
+                           "- You are doing requests from an IP belonging to a cloud provider (like AWS, " \
+                           "Google Cloud Platform, Azure, etc.). Unfortunately, most IPs from cloud " \
+                           "providers are blocked by YouTube.\n\n"
+
+      CAUSE_MESSAGE = "#{BASE_CAUSE_MESSAGE}" \
+                      "There are two things you can do to work around this:\n" \
+                      "1. Use proxies to hide your IP address.\n" \
+                      "2. (NOT RECOMMENDED) If you authenticate your requests using cookies, you " \
+                      "will be able to continue doing requests for a while. However, YouTube will " \
+                      "eventually permanently ban the account that you have used to authenticate " \
+                      "with! So only do this if you don't mind your account being banned!".freeze
+    end
+
+    # Raised when YouTube blocks the IP specifically
+    class IpBlocked < RequestBlocked
+      CAUSE_MESSAGE = "#{RequestBlocked::BASE_CAUSE_MESSAGE}" \
+                      "Ways to work around this are using proxies or rotating residential IPs.".freeze
+    end
+
+    # Raised when too many requests are made (HTTP 429)
+    class TooManyRequests < CouldNotRetrieveTranscript
+      CAUSE_MESSAGE = "YouTube is rate limiting your requests. Please wait before making more requests."
+    end
+
+    # Raised when transcripts are disabled for a video
+    class TranscriptsDisabled < CouldNotRetrieveTranscript
+      CAUSE_MESSAGE = "Subtitles are disabled for this video"
+    end
+
+    # Raised when a video is age restricted
+    class AgeRestricted < CouldNotRetrieveTranscript
+      CAUSE_MESSAGE = "This video is age-restricted. Therefore, you are unable to retrieve " \
+                      "transcripts for it without authenticating yourself.\n\n" \
+                      "Unfortunately, Cookie Authentication is temporarily unsupported, " \
+                      "as recent changes in YouTube's API broke the previous implementation."
+    end
+
+    # Raised when a transcript is not translatable
+    class NotTranslatable < CouldNotRetrieveTranscript
+      CAUSE_MESSAGE = "The requested language is not translatable"
+    end
+
+    # Raised when the requested translation language is not available
+    class TranslationLanguageNotAvailable < CouldNotRetrieveTranscript
+      CAUSE_MESSAGE = "The requested translation language is not available"
+    end
+
+    # Raised when consent cookie creation fails
+    class FailedToCreateConsentCookie < CouldNotRetrieveTranscript
+      CAUSE_MESSAGE = "Failed to automatically give consent to saving cookies"
+    end
+
+    # Raised when no transcript is found for the requested languages
+    class NoTranscriptFound < CouldNotRetrieveTranscript
+      CAUSE_MESSAGE = "No transcripts were found for any of the requested language codes: %<requested_language_codes>s\n\n%<transcript_data>s"
+
+      # @return [Array<String>] the requested language codes
+      attr_reader :requested_language_codes
+
+      # @return [Object] the transcript data (TranscriptList)
+      attr_reader :transcript_data
+
+      # @param video_id [String] the YouTube video ID
+      # @param requested_language_codes [Array<String>] the language codes that were requested
+      # @param transcript_data [Object] the TranscriptList object with available transcripts
+      def initialize(video_id, requested_language_codes, transcript_data)
+        @requested_language_codes = requested_language_codes
+        @transcript_data = transcript_data
+        super(video_id)
+      end
+
+      def cause_message
+        format(
+          CAUSE_MESSAGE,
+          requested_language_codes: @requested_language_codes.inspect,
+          transcript_data: @transcript_data.to_s
+        )
+      end
+    end
+
+    # Raised when no transcripts are available for a video
+    class NoTranscriptAvailable < CouldNotRetrieveTranscript
+      CAUSE_MESSAGE = "No transcripts are available for this video"
+    end
+
+    # Raised when a PO token is required to fetch the transcript
+    class PoTokenRequired < CouldNotRetrieveTranscript
+      CAUSE_MESSAGE = "The requested video cannot be retrieved without a PO Token. " \
+                      "If this happens, please open a GitHub issue!"
+    end
+  end
+end

data/lib/youtube_rb/transcript/settings.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module YoutubeRb
+  module Transcript
+    # YouTube watch URL template
+    # @example
+    #   format(WATCH_URL, video_id: "abc123")
+    #   # => "https://www.youtube.com/watch?v=abc123"
+    WATCH_URL = "https://www.youtube.com/watch?v=%<video_id>s"
+
+    # YouTube Innertube API URL template
+    # @example
+    #   format(INNERTUBE_API_URL, api_key: "key123")
+    #   # => "https://www.youtube.com/youtubei/v1/player?key=key123"
+    INNERTUBE_API_URL = "https://www.youtube.com/youtubei/v1/player?key=%<api_key>s"
+
+    # Innertube API context for Android client
+    # Used in POST requests to the Innertube API
+    INNERTUBE_CONTEXT = {
+      "client" => {
+        "clientName" => "ANDROID",
+        "clientVersion" => "20.10.38"
+      }
+    }.freeze
+  end
+end

data/lib/youtube_rb/transcript/transcript.rb

@@ -0,0 +1,237 @@
+# frozen_string_literal: true
+
+module YoutubeRb
+  module Transcript
+    # Represents a language available for translation
+    class TranslationLanguage
+      # @return [String] the language name (e.g., "Spanish")
+      attr_reader :language
+
+      # @return [String] the language code (e.g., "es")
+      attr_reader :language_code
+
+      # @param language [String] the language name
+      # @param language_code [String] the language code
+      def initialize(language:, language_code:)
+        @language = language
+        @language_code = language_code
+      end
+    end
+
+    # Represents a single transcript snippet/segment
+    class TranscriptSnippet
+      # @return [String] the text content of the snippet
+      attr_reader :text
+
+      # @return [Float] the start time in seconds
+      attr_reader :start
+
+      # @return [Float] the duration in seconds
+      attr_reader :duration
+
+      # @param text [String] the text content
+      # @param start [Float] the start time in seconds
+      # @param duration [Float] the duration in seconds
+      def initialize(text:, start:, duration:)
+        @text = text
+        @start = start.to_f
+        @duration = duration.to_f
+      end
+
+      # Convert to hash representation
+      # @return [Hash] hash with text, start, and duration keys
+      def to_h
+        {
+          "text" => @text,
+          "start" => @start,
+          "duration" => @duration
+        }
+      end
+    end
+
+    # Represents a fetched transcript containing multiple snippets
+    # This class is Enumerable, allowing iteration over snippets
+    class FetchedTranscript
+      include Enumerable
+
+      # @return [String] the video ID
+      attr_reader :video_id
+
+      # @return [String] the language name (e.g., "English")
+      attr_reader :language
+
+      # @return [String] the language code (e.g., "en")
+      attr_reader :language_code
+
+      # @return [Boolean] whether the transcript was auto-generated
+      attr_reader :is_generated
+
+      # @return [Array<TranscriptSnippet>] the transcript snippets
+      attr_reader :snippets
+
+      # @param video_id [String] the YouTube video ID
+      # @param language [String] the language name
+      # @param language_code [String] the language code
+      # @param is_generated [Boolean] whether auto-generated
+      # @param snippets [Array<TranscriptSnippet>] the snippets (optional)
+      def initialize(video_id:, language:, language_code:, is_generated:, snippets: [])
+        @video_id = video_id
+        @language = language
+        @language_code = language_code
+        @is_generated = is_generated
+        @snippets = snippets
+      end
+
+      # Add a snippet to the transcript
+      # @param snippet [TranscriptSnippet] the snippet to add
+      # @return [self]
+      def add_snippet(snippet)
+        @snippets << snippet
+        self
+      end
+
+      # Iterate over each snippet
+      # @yield [TranscriptSnippet] each snippet in the transcript
+      def each(&)
+        @snippets.each(&)
+      end
+
+      # Get a snippet by index
+      # @param index [Integer] the index
+      # @return [TranscriptSnippet] the snippet at the given index
+      def [](index)
+        @snippets[index]
+      end
+
+      # Get the number of snippets
+      # @return [Integer] the count of snippets
+      def length
+        @snippets.length
+      end
+      alias size length
+
+      # Convert to raw data (array of hashes)
+      # @return [Array<Hash>] array of snippet hashes
+      def to_raw_data
+        @snippets.map(&:to_h)
+      end
+
+      # Check if transcript was auto-generated
+      # @return [Boolean]
+      def generated?
+        @is_generated
+      end
+    end
+
+    # Represents transcript metadata and provides fetch/translate capabilities
+    class TranscriptMetadata
+      # @return [String] the video ID
+      attr_reader :video_id
+
+      # @return [String] the language name
+      attr_reader :language
+
+      # @return [String] the language code
+      attr_reader :language_code
+
+      # @return [Boolean] whether auto-generated
+      attr_reader :is_generated
+
+      # @return [Array<TranslationLanguage>] available translation languages
+      attr_reader :translation_languages
+
+      # @param http_client [Faraday::Connection] the HTTP client
+      # @param video_id [String] the YouTube video ID
+      # @param url [String] the transcript URL
+      # @param language [String] the language name
+      # @param language_code [String] the language code
+      # @param is_generated [Boolean] whether auto-generated
+      # @param translation_languages [Array<TranslationLanguage>] available translations
+      def initialize(http_client:, video_id:, url:, language:, language_code:, is_generated:, translation_languages:)
+        @http_client = http_client
+        @video_id = video_id
+        @url = url
+        @language = language
+        @language_code = language_code
+        @is_generated = is_generated
+        @translation_languages = translation_languages
+        @translation_languages_dict = translation_languages.each_with_object({}) do |tl, hash|
+          hash[tl.language_code] = tl.language
+        end
+      end
+
+      # Fetch the actual transcript data
+      # @param preserve_formatting [Boolean] whether to preserve HTML formatting
+      # @return [FetchedTranscript] the fetched transcript
+      # @raise [PoTokenRequired] if a PO token is required
+      def fetch(preserve_formatting: false)
+        raise PoTokenRequired, @video_id if @url.include?("&exp=xpe")
+
+        response = @http_client.get(@url)
+        raise_http_errors(response)
+
+        parser = TranscriptParser.new(preserve_formatting: preserve_formatting)
+        snippets = parser.parse(response.body)
+
+        FetchedTranscript.new(
+          video_id: @video_id,
+          language: @language,
+          language_code: @language_code,
+          is_generated: @is_generated,
+          snippets: snippets
+        )
+      end
+
+      # Check if this transcript can be translated
+      # @return [Boolean]
+      def translatable?
+        !@translation_languages.empty?
+      end
+      alias is_translatable translatable?
+
+      # Translate this transcript to another language
+      # @param language_code [String] the target language code
+      # @return [TranscriptMetadata] a new TranscriptMetadata object for the translated version
+      # @raise [NotTranslatable] if the transcript cannot be translated
+      # @raise [TranslationLanguageNotAvailable] if the language is not available
+      def translate(language_code)
+        raise NotTranslatable, @video_id unless translatable?
+        raise TranslationLanguageNotAvailable, @video_id unless @translation_languages_dict.key?(language_code)
+
+        TranscriptMetadata.new(
+          http_client: @http_client,
+          video_id: @video_id,
+          url: "#{@url}&tlang=#{language_code}",
+          language: @translation_languages_dict[language_code],
+          language_code: language_code,
+          is_generated: true,
+          translation_languages: []
+        )
+      end
+
+      # Check if transcript was auto-generated
+      # @return [Boolean]
+      def generated?
+        @is_generated
+      end
+
+      # String representation of the transcript
+      # @return [String]
+      def to_s
+        translation_desc = translatable? ? "[TRANSLATABLE]" : ""
+        "#{@language_code} (\"#{@language}\")#{translation_desc}"
+      end
+
+      private
+
+      def raise_http_errors(response)
+        case response.status
+        when 429
+          raise IpBlocked, @video_id
+        when 400..599
+          raise YouTubeRequestFailed.new(@video_id, StandardError.new("HTTP #{response.status}"))
+        end
+      end
+    end
+  end
+end