rubytube 0.4.0 → 1.2.0

data/lib/rubytube/extractor.rb

@@ -1,177 +0,0 @@
-module RubyTube
-  class Extractor
-    class << self
-      def playability_status(watch_html)
-        player_response = initial_player_response(watch_html)
-        player_response = JSON.parse(player_response)
-        status_obj = player_response['playabilityStatus'] || {}
-      
-        if status_obj.has_key?('liveStreamability')
-          return ['LIVE_STREAM', 'Video is a live stream.']
-        end
-      
-        if status_obj.has_key?('status')
-          if status_obj.has_key?('reason')
-            return [status_obj['status'], [status_obj['reason']]]
-          end
-      
-          if status_obj.has_key?('messages')
-            return [status_obj['status'], status_obj['messages']]
-          end
-        end
-      
-        [nil, [nil]]
-      end
-
-      def video_id(url)
-        return Utils.regex_search(/(?:v=|\/)([0-9A-Za-z_-]{11}).*/, url, 1)
-      end
-
-      def js_url(html)
-        begin
-          base_js = get_ytplayer_config(html)['assets']['js']
-        rescue RegexMatchError, NoMethodError
-          base_js = get_ytplayer_js(html)
-        end
-
-        "https://youtube.com#{base_js}"
-      end
-
-      def mime_type_codec(mime_type_codec)
-        pattern = %r{(\w+\/\w+)\;\scodecs=\"([a-zA-Z\-0-9.,\s]*)\"}
-        results = mime_type_codec.match(pattern)
-      
-        raise RegexMatchError.new("mime_type_codec, pattern=#{pattern}") if results.nil?
-      
-        mime_type, codecs = results.captures
-        [mime_type, codecs.split(",").map(&:strip)]
-      end
-
-      def get_ytplayer_js(html)
-        js_url_patterns = [
-          %r{(/s/player/[\w\d]+/[\w\d_/.]+/base\.js)},
-        ]
-
-        js_url_patterns.each do |pattern|
-          function_match = html.match(pattern)
-          if function_match
-            return function_match[1]
-          end
-        end
-
-        raise RegexMatchError.new('get_ytplayer_js', 'js_url_patterns')
-      end
-
-      def get_ytplayer_config(html)
-        config_patterns = [
-          /ytplayer\.config\s*=\s*/,
-          /ytInitialPlayerResponse\s*=\s*/
-        ]
-
-        config_patterns.each do |pattern|
-          begin
-            return Parser.parse_for_object(html, pattern)
-          rescue HTMLParseError => e
-            next
-          end
-        end
-
-        setconfig_patterns = [
-          /yt\.setConfig\(.*['\"]PLAYER_CONFIG['\"]:\s*/
-        ]
-
-        setconfig_patterns.each do |pattern|
-          begin
-            return Parser.parse_for_object(html, pattern)
-          rescue HTMLParseError => e
-            next
-          end
-        end
-
-        raise RegexMatchError.new('get_ytplayer_config', 'config_patterns, setconfig_patterns')
-      end
-
-      def apply_signature(stream_manifest, vid_info, js)
-        cipher = Cipher.new(js)
-
-        stream_manifest.each_with_index do |stream, i|
-          begin
-            url = stream['url']
-          rescue NoMethodError
-            live_stream = vid_info.fetch('playabilityStatus', {})['liveStreamability']
-            if live_stream
-              raise LiveStreamError.new('UNKNOWN')
-            end
-          end
-
-          if url.include?("signature") || 
-            (!stream.key?("s") && (url.include?("&sig=") || url.include?("&lsig=")))
-            # For certain videos, YouTube will just provide them pre-signed, in
-            # which case there's no real magic to download them and we can skip
-            # the whole signature descrambling entirely.
-            next
-          end
-
-          signature = cipher.get_signature(stream['s'])
-
-          parsed_url = URI.parse(url)
-
-          query_params = CGI.parse(parsed_url.query)
-          query_params.transform_values!(&:first)
-          query_params['sig'] = signature
-          unless query_params.key?('ratebypass')
-            initial_n = query_params['n'].split('')
-            new_n = cipher.calculate_n(initial_n)
-            query_params['n'] = new_n
-          end
-
-          url = "#{parsed_url.scheme}://#{parsed_url.host}#{parsed_url.path}?#{URI.encode_www_form(query_params)}"
-
-          stream_manifest[i]["url"] = url
-        end
-      end
-
-      def apply_descrambler(stream_data)
-        return if stream_data.has_key?('url')
-
-        # Merge formats and adaptiveFormats into a single array
-        formats = []
-        formats += stream_data['formats'] if stream_data.has_key?('formats')
-        formats += stream_data['adaptiveFormats'] if stream_data.has_key?('adaptiveFormats')
-
-        # Extract url and s from signatureCiphers as necessary
-        formats.each do |data|
-          unless data.has_key?('url')
-            if data.has_key?('signatureCipher')
-              cipher_url = URI.decode_www_form(data['signatureCipher']).to_h
-              data['url'] = cipher_url['url']
-              data['s'] = cipher_url['s']
-            end
-          end
-          data['is_otf'] = data['type'] == 'FORMAT_STREAM_TYPE_OTF'
-        end
-
-        formats
-      end
-
-      private
-
-      def initial_player_response(watch_html)
-        patterns = [
-          "window\\[['\"]ytInitialPlayerResponse['\"]\\]\\s*=\\s*",
-          "ytInitialPlayerResponse\\s*=\\s*"
-        ]
-      
-        patterns.each do |pattern|
-          begin
-            return Parser.parse_for_object(watch_html, pattern)
-          rescue HTMLParseError
-            next
-          end
-        end
-      
-        raise RegexMatchError.new('initial_player_response', 'initial_player_response_pattern')
-      end
-    end
-  end
-end

data/lib/rubytube/monostate.rb

@@ -1,5 +0,0 @@
-module RubyTube
-  class Monostate
-    attr_accessor :title, :duration
-  end
-end

data/lib/rubytube/parser.rb

@@ -1,164 +0,0 @@
-module RubyTube
-  module Parser
-    module_function
-
-    def parse_for_object(html, preceding_regex)
-      regex = Regexp.new(preceding_regex)
-      result = regex.match(html)
-
-      if result.nil?
-        raise HTMLParseError, "No matches for regex #{preceding_regex}"
-      end
-      start_index = result.end(0)
-
-      return parse_for_object_from_startpoint(html, start_index)
-    end
-
-    def find_object_from_startpoint(html, start_point)
-      html = html[start_point..-1]
-      unless ['{', '['].include?(html[0])
-        raise HTMLParseError, "Invalid start point. Start of HTML:\n#{html[0..19]}"
-      end
-
-      last_char = '{'
-      curr_char = nil
-      stack = [html[0]]
-      i = 1
-
-      context_closers = {
-        '{' => '}',
-        '[' => ']',
-        '"' => '"',
-        '/' => '/',
-      }
-
-      while i < html.length
-        break if stack.empty?
-
-        last_char = curr_char unless [' ', '\n'].include?(curr_char)
-        curr_char = html[i]
-        curr_context = stack.last
-
-        if curr_char == context_closers[curr_context]
-          stack.pop
-          i += 1
-          next
-        end
-
-        if ['"', '/'].include?(curr_context)
-          if curr_char == '\\'
-            i += 2
-            next
-          end
-        else
-          if context_closers.keys.include?(curr_char)
-            unless curr_char == '/' && !['(', ',', '=', ':', '[', '!', '&', '|', '?', '{', '}', ';'].include?(last_char)
-              stack.push(curr_char)
-            end
-          end
-        end
-
-        i += 1
-      end
-
-      full_obj = html[0...i]
-      full_obj
-    end
-
-    def parse_for_object_from_startpoint(html, start_point)
-      html = html[start_point..-1]
-
-      unless ['{', '['].include?(html[0])
-        raise HTMLParseError, "Invalid start point. Start of HTML:\n#{html[0..19]}"
-      end
-
-      # First letter MUST be an open brace, so we put that in the stack,
-      # and skip the first character.
-      last_char = '{'
-      curr_char = nil
-      stack = [html[0]]
-      i = 1
-
-      context_closers = {
-        '{' => '}',
-        '[' => ']',
-        '"' => '"',
-        '/' => '/' # JavaScript regex
-      }
-
-      while i < html.length
-        break if stack.empty?
-    
-        last_char = curr_char unless [' ', '\n'].include?(curr_char)
-        curr_char = html[i]
-        curr_context = stack.last
-    
-        # If we've reached a context closer, we can remove an element off the stack
-        if curr_char == context_closers[curr_context]
-          stack.pop
-          i += 1
-          next
-        end
-        # Strings and regex expressions require special context handling because they can contain
-        # context openers *and* closers
-        if ['"', '/'].include?(curr_context)
-          # If there's a backslash in a string or regex expression, we skip a character
-          if curr_char == '\\'
-            i += 2
-            next
-          end
-        else
-          # Non-string contexts are when we need to look for context openers.
-          if context_closers.keys.include?(curr_char)
-            # Slash starts a regular expression depending on context
-            unless curr_char == '/' && ['(', ',', '=', ':', '[', '!', '&', '|', '?', '{', '}', ';'].include?(last_char)
-              stack << curr_char
-            end
-          end
-        end
-
-        i += 1
-      end
-
-      full_obj = html[0..(i - 1)]
-      full_obj
-    end
-
-    def throttling_array_split(js_array)
-      results = []
-      curr_substring = js_array[1..-1]
-    
-      comma_regex = /,/
-      func_regex = /function\([^)]*\)/
-    
-      until curr_substring.empty?
-        if curr_substring.start_with?('function')
-          match = func_regex.match(curr_substring)
-          match_start = match.begin(0)
-          match_end = match.end(0)
-    
-          function_text = find_object_from_startpoint(curr_substring, match_end)
-          full_function_def = curr_substring[0, match_end + function_text.length]
-          results << full_function_def
-          curr_substring = curr_substring[full_function_def.length + 1..-1]
-        else
-          match = comma_regex.match(curr_substring)
-
-          begin
-            match_start = match.begin(0)
-            match_end = match.end(0)
-          rescue NoMethodError
-            match_start = curr_substring.length - 1
-            match_end = match_start + 1
-          end
-    
-          curr_el = curr_substring[0, match_start]
-          results << curr_el
-          curr_substring = curr_substring[match_end..-1]
-        end
-      end
-    
-      results
-    end
-  end
-end

data/lib/rubytube/request.rb

@@ -1,75 +0,0 @@
-module RubyTube
-  module Request
-    module_function
-
-    DEFAULT_RANGE_SIZE = 9437184
-
-    def get(url, options = {})
-      send(:get, url, options).body
-    end
-
-    def post(url, options = {})
-      send(:post, url, options).body
-    end
-
-    def head(url, options = {})
-      send(:head, url, options).headers
-    end
-
-    def stream(url, timeout: 60, max_retries: 0)
-      file_size = DEFAULT_RANGE_SIZE
-      downloaded = 0
-
-      while downloaded < file_size
-        stop_pos = [downloaded + DEFAULT_RANGE_SIZE, file_size].min - 1
-        range_header = "bytes=#{downloaded}-#{stop_pos}"
-        tries = 0
-    
-        while true
-          begin
-            if tries >= 1 + max_retries
-              raise MaxRetriesExceeded
-            end
-            response = send(:get, "#{url}&range=#{downloaded}-#{stop_pos}")
-            break
-          rescue Faraday::TimeoutError
-          rescue Faraday::ClientError => e
-            raise e
-          end
-          tries += 1
-        end
-
-        if file_size == DEFAULT_RANGE_SIZE
-          begin
-            resp = send(:get, "#{url}&range=0-99999999999")
-            content_range = resp.headers["Content-Length"]
-            file_size = content_range.to_i
-          rescue KeyError, IndexError, StandardError => e
-          end
-        end
-
-        response.body.each_char do |chunk|
-          downloaded += chunk.length
-          yield chunk
-        end
-      end
-    end
-
-    def send(method, url, options = {})
-      headers = { 'Content-Type': 'text/html' }
-      options[:headers] && headers.merge!(options[:headers])
-
-      connection = Faraday.new(url: url) do |faraday|
-        faraday.response :follow_redirects
-        faraday.adapter Faraday.default_adapter
-      end
-      response = connection.send(method) do |req|
-        req.headers = headers
-        options[:query] && req.params = options[:query] 
-        options[:data] && req.body = JSON.dump(options[:data])
-      end
-
-      response
-    end
-  end
-end

data/lib/rubytube/stream_format.rb

@@ -1,152 +0,0 @@
-module RubyTube
-  class StreamFormat
-    PROGRESSIVE_VIDEO = {
-      5 => ["240p", "64kbps"],
-      6 => ["270p", "64kbps"],
-      13 => ["144p", nil],
-      17 => ["144p", "24kbps"],
-      18 => ["360p", "96kbps"],
-      22 => ["720p", "192kbps"],
-      34 => ["360p", "128kbps"],
-      35 => ["480p", "128kbps"],
-      36 => ["240p", nil],
-      37 => ["1080p", "192kbps"],
-      38 => ["3072p", "192kbps"],
-      43 => ["360p", "128kbps"],
-      44 => ["480p", "128kbps"],
-      45 => ["720p", "192kbps"],
-      46 => ["1080p", "192kbps"],
-      59 => ["480p", "128kbps"],
-      78 => ["480p", "128kbps"],
-      82 => ["360p", "128kbps"],
-      83 => ["480p", "128kbps"],
-      84 => ["720p", "192kbps"],
-      85 => ["1080p", "192kbps"],
-      91 => ["144p", "48kbps"],
-      92 => ["240p", "48kbps"],
-      93 => ["360p", "128kbps"],
-      94 => ["480p", "128kbps"],
-      95 => ["720p", "256kbps"],
-      96 => ["1080p", "256kbps"],
-      100 => ["360p", "128kbps"],
-      101 => ["480p", "192kbps"],
-      102 => ["720p", "192kbps"],
-      132 => ["240p", "48kbps"],
-      151 => ["720p", "24kbps"],
-      300 => ["720p", "128kbps"],
-      301 => ["1080p", "128kbps"],
-    }
-
-    DASH_VIDEO = {
-      133 => ["240p", nil],
-      134 => ["360p", nil],
-      135 => ["480p", nil],
-      136 => ["720p", nil],
-      137 => ["1080p", nil],
-      138 => ["2160p", nil],
-      160 => ["144p", nil],
-      167 => ["360p", nil],
-      168 => ["480p", nil],
-      169 => ["720p", nil],
-      170 => ["1080p", nil],
-      212 => ["480p", nil],
-      218 => ["480p", nil],
-      219 => ["480p", nil],
-      242 => ["240p", nil],
-      243 => ["360p", nil],
-      244 => ["480p", nil],
-      245 => ["480p", nil],
-      246 => ["480p", nil],
-      247 => ["720p", nil],
-      248 => ["1080p", nil],
-      264 => ["1440p", nil],
-      266 => ["2160p", nil],
-      271 => ["1440p", nil],
-      272 => ["4320p", nil],
-      278 => ["144p", nil],
-      298 => ["720p", nil],
-      299 => ["1080p", nil],
-      302 => ["720p", nil],
-      303 => ["1080p", nil],
-      308 => ["1440p", nil],
-      313 => ["2160p", nil],
-      315 => ["2160p", nil],
-      330 => ["144p", nil],
-      331 => ["240p", nil],
-      332 => ["360p", nil],
-      333 => ["480p", nil],
-      334 => ["720p", nil],
-      335 => ["1080p", nil],
-      336 => ["1440p", nil],
-      337 => ["2160p", nil],
-      394 => ["144p", nil],
-      395 => ["240p", nil],
-      396 => ["360p", nil],
-      397 => ["480p", nil],
-      398 => ["720p", nil],
-      399 => ["1080p", nil],
-      400 => ["1440p", nil],
-      401 => ["2160p", nil],
-      402 => ["4320p", nil],
-      571 => ["4320p", nil],
-      694 => ["144p", nil],
-      695 => ["240p", nil],
-      696 => ["360p", nil],
-      697 => ["480p", nil],
-      698 => ["720p", nil],
-      699 => ["1080p", nil],
-      700 => ["1440p", nil],
-      701 => ["2160p", nil],
-      702 => ["4320p", nil]
-    }
-
-    DASH_AUDIO = {
-      139 => [nil, "48kbps"],
-      140 => [nil, "128kbps"],
-      141 => [nil, "256kbps"],
-      171 => [nil, "128kbps"],
-      172 => [nil, "256kbps"],
-      249 => [nil, "50kbps"],
-      250 => [nil, "70kbps"],
-      251 => [nil, "160kbps"],
-      256 => [nil, "192kbps"],
-      258 => [nil, "384kbps"],
-      325 => [nil, nil],
-      328 => [nil, nil],
-    }
-
-    ITAGS = {
-      **PROGRESSIVE_VIDEO,
-      **DASH_VIDEO,
-      **DASH_AUDIO
-    }
-
-    HDR = [330, 331, 332, 333, 334, 335, 336, 337]
-    FORMAT_3D = [82, 83, 84, 85, 100, 101, 102]
-    LIVE = [91, 92, 93, 94, 95, 96, 132, 151]
-
-    attr_reader :itag, :resolution, :abr
-
-    def initialize(_itag)
-      @itag = _itag
-
-      @resolution, @abr = ITAGS.fetch(itag)
-    end
-
-    def is_live?
-      !!LIVE[itag]
-    end
-
-    def is_3d?
-      !!FORMAT_3D[itag]
-    end
-
-    def is_hdr?
-      !!HDR[itag]
-    end
-
-    def is_dash?
-      !!DASH_AUDIO[itag] || !!DASH_VIDEO[itag]
-    end
-  end
-end

data/lib/rubytube/utils.rb

@@ -1,24 +0,0 @@
-module RubyTube
-  module Utils
-    module_function
-
-    def regex_search(pattern, string, group)
-      match = string.match(pattern)
-      if match
-        return match[group]
-      end
-      nil
-    end
-
-    def target_directory(output_path = nil)
-      if output_path
-        result = File.join(Dir.pwd, output_path) unless File.absolute_path?(output_path)
-      else
-        result = Dir.pwd
-      end
-
-      FileUtils.mkdir_p(result) unless Dir.exist?(result)
-      result
-    end
-  end
-end

data/lib/rubytube/version.rb

@@ -1,5 +0,0 @@
-# frozen_string_literal: true
-
-module RubyTube
-  VERSION = "0.4.0"
-end

data/sig/rubytube.rbs

@@ -1,4 +0,0 @@
-module RubyTube
-  VERSION: String
-  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
-end