youtube-transcript-rb 0.1.0 → 0.2.0

data/lib/youtube/transcript/rb/transcript_list.rb

@@ -1,170 +0,0 @@
-# frozen_string_literal: true
-
-module Youtube
-  module Transcript
-    module Rb
-      # Represents a list of available transcripts for a YouTube video.
-      # This class is Enumerable, allowing iteration over all available transcripts.
-      # It provides functionality to search for transcripts in specific languages.
-      class TranscriptList
-        include Enumerable
-
-        # @return [String] the video ID this TranscriptList is for
-        attr_reader :video_id
-
-        # Build a TranscriptList from captions JSON data
-        #
-        # @param http_client [Faraday::Connection] the HTTP client for fetching transcripts
-        # @param video_id [String] the YouTube video ID
-        # @param captions_json [Hash] the captions JSON parsed from YouTube
-        # @return [TranscriptList] the created TranscriptList
-        def self.build(http_client:, video_id:, captions_json:)
-          translation_languages = (captions_json["translationLanguages"] || []).map do |tl|
-            TranslationLanguage.new(
-              language: tl.dig("languageName", "runs", 0, "text") || "",
-              language_code: tl["languageCode"]
-            )
-          end
-
-          manually_created_transcripts = {}
-          generated_transcripts = {}
-
-          (captions_json["captionTracks"] || []).each do |caption|
-            is_generated = caption.fetch("kind", "") == "asr"
-            target_dict = is_generated ? generated_transcripts : manually_created_transcripts
-
-            language_code = caption["languageCode"]
-            transcript_translation_languages = caption.fetch("isTranslatable", false) ? translation_languages : []
-
-            target_dict[language_code] = Transcript.new(
-              http_client: http_client,
-              video_id: video_id,
-              url: caption["baseUrl"].to_s.gsub("&fmt=srv3", ""),
-              language: caption.dig("name", "runs", 0, "text") || "",
-              language_code: language_code,
-              is_generated: is_generated,
-              translation_languages: transcript_translation_languages
-            )
-          end
-
-          new(
-            video_id: video_id,
-            manually_created_transcripts: manually_created_transcripts,
-            generated_transcripts: generated_transcripts,
-            translation_languages: translation_languages
-          )
-        end
-
-        # @param video_id [String] the YouTube video ID
-        # @param manually_created_transcripts [Hash<String, Transcript>] manually created transcripts by language code
-        # @param generated_transcripts [Hash<String, Transcript>] auto-generated transcripts by language code
-        # @param translation_languages [Array<TranslationLanguage>] available translation languages
-        def initialize(video_id:, manually_created_transcripts:, generated_transcripts:, translation_languages:)
-          @video_id = video_id
-          @manually_created_transcripts = manually_created_transcripts
-          @generated_transcripts = generated_transcripts
-          @translation_languages = translation_languages
-        end
-
-        # Iterate over all transcripts (manually created first, then generated)
-        #
-        # @yield [Transcript] each available transcript
-        # @return [Enumerator] if no block given
-        def each(&block)
-          return to_enum(:each) unless block_given?
-
-          @manually_created_transcripts.each_value(&block)
-          @generated_transcripts.each_value(&block)
-        end
-
-        # Find a transcript for the given language codes.
-        # Manually created transcripts are preferred over generated ones.
-        #
-        # @param language_codes [Array<String>] language codes in descending priority
-        # @return [Transcript] the found transcript
-        # @raise [NoTranscriptFound] if no transcript matches the requested languages
-        def find_transcript(language_codes)
-          find_transcript_in(
-            language_codes,
-            [@manually_created_transcripts, @generated_transcripts]
-          )
-        end
-
-        # Find an automatically generated transcript for the given language codes.
-        #
-        # @param language_codes [Array<String>] language codes in descending priority
-        # @return [Transcript] the found transcript
-        # @raise [NoTranscriptFound] if no generated transcript matches
-        def find_generated_transcript(language_codes)
-          find_transcript_in(language_codes, [@generated_transcripts])
-        end
-
-        # Find a manually created transcript for the given language codes.
-        #
-        # @param language_codes [Array<String>] language codes in descending priority
-        # @return [Transcript] the found transcript
-        # @raise [NoTranscriptFound] if no manually created transcript matches
-        def find_manually_created_transcript(language_codes)
-          find_transcript_in(language_codes, [@manually_created_transcripts])
-        end
-
-        # String representation of the transcript list
-        #
-        # @return [String] human-readable description of available transcripts
-        def to_s
-          <<~DESC
-            For this video (#{@video_id}) transcripts are available in the following languages:
-
-            (MANUALLY CREATED)
-            #{format_language_list(@manually_created_transcripts.values)}
-
-            (GENERATED)
-            #{format_language_list(@generated_transcripts.values)}
-
-            (TRANSLATION LANGUAGES)
-            #{format_translation_languages}
-          DESC
-        end
-
-        private
-
-        # Find a transcript from the given dictionaries
-        #
-        # @param language_codes [Array<String>] language codes to search for
-        # @param transcript_dicts [Array<Hash>] transcript dictionaries to search
-        # @return [Transcript] the found transcript
-        # @raise [NoTranscriptFound] if no transcript matches
-        def find_transcript_in(language_codes, transcript_dicts)
-          language_codes.each do |language_code|
-            transcript_dicts.each do |dict|
-              return dict[language_code] if dict.key?(language_code)
-            end
-          end
-
-          raise NoTranscriptFound.new(@video_id, language_codes, self)
-        end
-
-        # Format a list of transcripts for display
-        #
-        # @param transcripts [Array<Transcript>] transcripts to format
-        # @return [String] formatted list or "None"
-        def format_language_list(transcripts)
-          return "None" if transcripts.empty?
-
-          transcripts.map { |t| " - #{t}" }.join("\n")
-        end
-
-        # Format translation languages for display
-        #
-        # @return [String] formatted list or "None"
-        def format_translation_languages
-          return "None" if @translation_languages.empty?
-
-          @translation_languages.map do |tl|
-            " - #{tl.language_code} (\"#{tl.language}\")"
-          end.join("\n")
-        end
-      end
-    end
-  end
-end

data/lib/youtube/transcript/rb/transcript_list_fetcher.rb

@@ -1,225 +0,0 @@
-# frozen_string_literal: true
-
-require "cgi"
-require "json"
-
-module Youtube
-  module Transcript
-    module Rb
-      # Playability status values returned by YouTube
-      module PlayabilityStatus
-        OK = "OK"
-        ERROR = "ERROR"
-        LOGIN_REQUIRED = "LOGIN_REQUIRED"
-      end
-
-      # Reason messages for playability failures
-      module PlayabilityFailedReason
-        BOT_DETECTED = "Sign in to confirm you're not a bot"
-        AGE_RESTRICTED = "This video may be inappropriate for some users."
-        VIDEO_UNAVAILABLE = "This video is unavailable"
-      end
-
-      # Fetches transcript lists from YouTube videos.
-      # This class handles all the HTTP communication with YouTube,
-      # including consent cookie handling and error detection.
-      class TranscriptListFetcher
-        # @param http_client [Faraday::Connection] the HTTP client to use
-        # @param proxy_config [Object, nil] optional proxy configuration
-        def initialize(http_client:, proxy_config: nil)
-          @http_client = http_client
-          @proxy_config = proxy_config
-        end
-
-        # Fetch the transcript list for a video
-        #
-        # @param video_id [String] the YouTube video ID
-        # @return [TranscriptList] the list of available transcripts
-        # @raise [CouldNotRetrieveTranscript] if transcripts cannot be retrieved
-        def fetch(video_id)
-          TranscriptList.build(
-            http_client: @http_client,
-            video_id: video_id,
-            captions_json: fetch_captions_json(video_id)
-          )
-        end
-
-        private
-
-        # Fetch captions JSON with retry support
-        #
-        # @param video_id [String] the YouTube video ID
-        # @param try_number [Integer] current retry attempt
-        # @return [Hash] the captions JSON
-        def fetch_captions_json(video_id, try_number: 0)
-          html = fetch_video_html(video_id)
-          api_key = extract_innertube_api_key(html, video_id)
-          innertube_data = fetch_innertube_data(video_id, api_key)
-          extract_captions_json(innertube_data, video_id)
-        rescue RequestBlocked => e
-          retries = @proxy_config.nil? ? 0 : (@proxy_config.respond_to?(:retries_when_blocked) ? @proxy_config.retries_when_blocked : 0)
-          if try_number + 1 < retries
-            return fetch_captions_json(video_id, try_number: try_number + 1)
-          end
-          raise e
-        end
-
-        # Extract the INNERTUBE_API_KEY from the video page HTML
-        #
-        # @param html [String] the HTML content
-        # @param video_id [String] the video ID (for error messages)
-        # @return [String] the API key
-        # @raise [IpBlocked] if a CAPTCHA is detected
-        # @raise [YouTubeDataUnparsable] if the key cannot be found
-        def extract_innertube_api_key(html, video_id)
-          match = html.match(/"INNERTUBE_API_KEY":\s*"([a-zA-Z0-9_-]+)"/)
-          if match && match[1]
-            return match[1]
-          end
-
-          raise IpBlocked, video_id if html.include?('class="g-recaptcha"')
-          raise YouTubeDataUnparsable, video_id
-        end
-
-        # Extract captions JSON from innertube data
-        #
-        # @param innertube_data [Hash] the innertube API response
-        # @param video_id [String] the video ID
-        # @return [Hash] the captions JSON
-        # @raise [TranscriptsDisabled] if no captions are available
-        def extract_captions_json(innertube_data, video_id)
-          assert_playability(innertube_data["playabilityStatus"], video_id)
-
-          captions_json = innertube_data.dig("captions", "playerCaptionsTracklistRenderer")
-          if captions_json.nil? || !captions_json.key?("captionTracks")
-            raise TranscriptsDisabled, video_id
-          end
-
-          captions_json
-        end
-
-        # Assert that the video is playable
-        #
-        # @param playability_status_data [Hash, nil] the playability status from API
-        # @param video_id [String] the video ID
-        # @raise [Various] depending on the playability status
-        def assert_playability(playability_status_data, video_id)
-          return if playability_status_data.nil?
-
-          status = playability_status_data["status"]
-          return if status == PlayabilityStatus::OK || status.nil?
-
-          reason = playability_status_data["reason"]
-
-          if status == PlayabilityStatus::LOGIN_REQUIRED
-            if reason == PlayabilityFailedReason::BOT_DETECTED
-              raise RequestBlocked, video_id
-            elsif reason == PlayabilityFailedReason::AGE_RESTRICTED
-              raise AgeRestricted, video_id
-            end
-          end
-
-          if status == PlayabilityStatus::ERROR && reason == PlayabilityFailedReason::VIDEO_UNAVAILABLE
-            if video_id.start_with?("http://") || video_id.start_with?("https://")
-              raise InvalidVideoId, video_id
-            end
-            raise VideoUnavailable, video_id
-          end
-
-          # Extract subreasons for more detailed error messages
-          subreasons = playability_status_data.dig("errorScreen", "playerErrorMessageRenderer", "subreason", "runs") || []
-          subreason_texts = subreasons.map { |run| run["text"] || "" }
-
-          raise VideoUnplayable.new(video_id, reason, subreason_texts)
-        end
-
-        # Create a consent cookie from the HTML
-        #
-        # @param html [String] the HTML content
-        # @param video_id [String] the video ID
-        # @raise [FailedToCreateConsentCookie] if the cookie cannot be created
-        def create_consent_cookie(html, video_id)
-          match = html.match(/name="v" value="(.*?)"/)
-          raise FailedToCreateConsentCookie, video_id if match.nil?
-
-          # Set the consent cookie
-          # Note: Faraday doesn't have built-in cookie management like requests.Session
-          # We'll need to handle this via headers or middleware
-          @consent_value = "YES+#{match[1]}"
-        end
-
-        # Fetch the video HTML page
-        #
-        # @param video_id [String] the video ID
-        # @return [String] the HTML content
-        def fetch_video_html(video_id)
-          html = fetch_html(video_id)
-
-          if html.include?('action="https://consent.youtube.com/s"')
-            create_consent_cookie(html, video_id)
-            html = fetch_html(video_id)
-            if html.include?('action="https://consent.youtube.com/s"')
-              raise FailedToCreateConsentCookie, video_id
-            end
-          end
-
-          html
-        end
-
-        # Fetch raw HTML from YouTube
-        #
-        # @param video_id [String] the video ID
-        # @return [String] the HTML content (unescaped)
-        def fetch_html(video_id)
-          url = format(WATCH_URL, video_id: video_id)
-          headers = { "Accept-Language" => "en-US" }
-
-          # Add consent cookie if we have one
-          headers["Cookie"] = "CONSENT=#{@consent_value}" if @consent_value
-
-          response = @http_client.get(url) do |req|
-            headers.each { |k, v| req.headers[k] = v }
-          end
-
-          raise_http_errors(response, video_id)
-          CGI.unescapeHTML(response.body)
-        end
-
-        # Fetch data from the Innertube API
-        #
-        # @param video_id [String] the video ID
-        # @param api_key [String] the API key
-        # @return [Hash] the API response
-        def fetch_innertube_data(video_id, api_key)
-          url = format(INNERTUBE_API_URL, api_key: api_key)
-
-          response = @http_client.post(url) do |req|
-            req.headers["Content-Type"] = "application/json"
-            req.body = JSON.generate({
-              "context" => INNERTUBE_CONTEXT,
-              "videoId" => video_id
-            })
-          end
-
-          raise_http_errors(response, video_id)
-          JSON.parse(response.body)
-        end
-
-        # Raise appropriate errors for HTTP responses
-        #
-        # @param response [Faraday::Response] the HTTP response
-        # @param video_id [String] the video ID
-        # @raise [IpBlocked] for 429 responses
-        # @raise [YouTubeRequestFailed] for other error responses
-        def raise_http_errors(response, video_id)
-          case response.status
-          when 429
-            raise IpBlocked, video_id
-          when 400..599
-            raise YouTubeRequestFailed.new(video_id, StandardError.new("HTTP #{response.status}"))
-          end
-        end
-      end
-    end
-  end
-end

data/lib/youtube/transcript/rb/transcript_parser.rb

@@ -1,83 +0,0 @@
-# frozen_string_literal: true
-
-require "nokogiri"
-require "cgi"
-
-module Youtube
-  module Transcript
-    module Rb
-      # Parses XML transcript data from YouTube
-      class TranscriptParser
-        # HTML formatting tags to preserve when preserve_formatting is enabled
-        FORMATTING_TAGS = %w[
-          strong
-          em
-          b
-          i
-          mark
-          small
-          del
-          ins
-          sub
-          sup
-        ].freeze
-
-        # @param preserve_formatting [Boolean] whether to preserve HTML formatting tags
-        def initialize(preserve_formatting: false)
-          @preserve_formatting = preserve_formatting
-          @html_regex = build_html_regex
-        end
-
-        # Parse XML transcript data into TranscriptSnippet objects
-        # @param raw_data [String] the raw XML data from YouTube
-        # @return [Array<TranscriptSnippet>] parsed transcript snippets
-        def parse(raw_data)
-          doc = Nokogiri::XML(raw_data)
-          snippets = []
-
-          doc.xpath("//text").each do |element|
-            text_content = element.text
-            next if text_content.nil? || text_content.empty?
-
-            # Unescape HTML entities and remove unwanted HTML tags
-            text = process_text(text_content)
-
-            snippets << TranscriptSnippet.new(
-              text: text,
-              start: element["start"].to_f,
-              duration: (element["dur"] || "0.0").to_f
-            )
-          end
-
-          snippets
-        end
-
-        private
-
-        # Build regex for removing HTML tags
-        # @return [Regexp]
-        def build_html_regex
-          if @preserve_formatting
-            # Remove all tags except formatting tags
-            formats_pattern = FORMATTING_TAGS.join("|")
-            # Match tags that are NOT the formatting tags
-            Regexp.new("</?(?!/?(?:#{formats_pattern})\\b)[^>]*>", Regexp::IGNORECASE)
-          else
-            # Remove all HTML tags
-            Regexp.new("<[^>]*>", Regexp::IGNORECASE)
-          end
-        end
-
-        # Process text by unescaping HTML entities and removing unwanted tags
-        # @param text [String] the raw text
-        # @return [String] processed text
-        def process_text(text)
-          # Unescape HTML entities
-          unescaped = CGI.unescapeHTML(text)
-          # Remove unwanted HTML tags
-          unescaped.gsub(@html_regex, "")
-        end
-      end
-    end
-  end
-end

data/lib/youtube/transcript/rb.rb

@@ -1,37 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "rb/version"
-require_relative "rb/settings"
-require_relative "rb/errors"
-require_relative "rb/transcript_parser"
-require_relative "rb/transcript"
-require_relative "rb/transcript_list"
-require_relative "rb/transcript_list_fetcher"
-require_relative "rb/api"
-require_relative "rb/formatters"
-
-module Youtube
-  module Transcript
-    module Rb
-      class << self
-        # Convenience method to fetch a transcript
-        # @param video_id [String] YouTube video ID
-        # @param languages [Array<String>] Language codes in order of preference
-        # @param preserve_formatting [Boolean] Whether to preserve HTML formatting
-        # @return [FetchedTranscript] The fetched transcript
-        def fetch(video_id, languages: ["en"], preserve_formatting: false)
-          api = YouTubeTranscriptApi.new
-          api.fetch(video_id, languages: languages, preserve_formatting: preserve_formatting)
-        end
-
-        # Convenience method to list available transcripts
-        # @param video_id [String] YouTube video ID
-        # @return [TranscriptList] List of available transcripts
-        def list(video_id)
-          api = YouTubeTranscriptApi.new
-          api.list(video_id)
-        end
-      end
-    end
-  end
-end

data/sig/youtube/transcript/rb.rbs

@@ -1,8 +0,0 @@
-module Youtube
-  module Transcript
-    module Rb
-      VERSION: String
-      # See the writing guide of rbs: https://github.com/ruby/rbs#guides
-    end
-  end
-end