viddl-rb 0.95 → 0.96

checksums.yaml

@@ -1,7 +1,7 @@
 --- 
-SHA1: 
-  metadata.gz: 6b2cb672ebfa65f7f7231fdd615a77287757a529
-  data.tar.gz: 03441807280152b0f821e648bec6289862aa7a60
 SHA512: 
-  metadata.gz: 369b0e52e1d62e9616b31f1477a4fc9cd7c274cd994404e9ce104ebcad2ff82c9168cb37c531362f65462c801ddf0e84cf8205c6d908a447baa45949778d73db
-  data.tar.gz: 2df7a2726f09cd7c74a5e39386e4fe22a411314e93f936a4b84873f75a008aa69c5fd0ba252352af66a230e28eca3625ff394ac072dd696d834597b5df11564a
+  data.tar.gz: ff67e34025fae7f2799db8f7fcde11485942e5a7971d8f915bff66ec9ed49d20a0d8013f14268765dade6adc24c6c80320306fef363b78c876076942071d11eb
+  metadata.gz: d77571b6da60ad4fdb6c235232c19c528ce7e3f03f290c580d89953ee4dc24d04d77fc02d9e5422007c4902a8263fa0c5fccf47e5329ca37c9970fa82ccbd462
+SHA1: 
+  data.tar.gz: fafe08cbf9074f38e92dcb632f397e45f86d2c00
+  metadata.gz: 2019cc2be56479dd8827a2979641b8b14bc6cd08

data/README.md

@@ -23,7 +23,9 @@ Viddl-rb supports the following command line options:
 -f, --filter REGEX               Filters a video playlist according to the regex (Youtube only right now)
 -s, --save-dir DIRECTORY         Specifies the directory where videos should be saved
 -d, --downloader TOOL            Specifies the tool to download with. Supports 'wget', 'curl' and 'net-http'
--q, --quality QUALITY            Specifies the video format and resolution in the following way => resolution:extension (e.g. 720:mp4). Currently only supported by the Youtube plugin.
+-q, --quality QUALITY            Specifies the video format and resolution in the following way: width:height:res (e.g. 1280:720:mp4)
+                                 The width, height and resolution may be omitted with a *.
+                                 For example, to match any quality with a width of 720 pixels in any format specify --quality *:720:*
 -h, --help                       Displays the help screen
 ```
 

data/Rakefile

@@ -2,21 +2,31 @@ require 'rubygems'
 require 'bundler/setup'
 require 'rake/testtask'
 
-task :default => [:test]
+ALL_INTEGRATION = FileList["spec/integration/*.rb"]
+ALL_UNIT 		= FileList["spec/unit/*/*.rb"]
 
-Rake::TestTask.new(:test) do |t|
-  #t.pattern = "spec/*_spec.rb"
-  t.test_files = ["spec/lib_spec.rb", "spec/url_extraction_spec.rb",  "spec/integration_spec.rb"]
+task :default => [:all]
+
+Rake::TestTask.new(:all) do |t|
+  t.test_files =  ALL_INTEGRATION + ALL_UNIT
+end
+
+Rake::TestTask.new(:test_unit) do |t|
+  t.test_files = ALL_UNIT
+end
+
+Rake::TestTask.new(:test_integration) do |t|
+  t.test_files = ALL_INTEGRATION
 end
 
 Rake::TestTask.new(:test_lib) do |t|
-  t.test_files = FileList["spec/lib_spec.rb"]
+  t.test_files = FileList["spec/integration/lib_spec.rb"]
 end
 
 Rake::TestTask.new(:test_extract) do |t|
-  t.test_files = FileList["spec/url_extraction_spec.rb"]
+  t.test_files = FileList["spec/integration/url_extraction_spec.rb"]
 end
 
-Rake::TestTask.new(:test_integration) do |t|
-  t.test_files = FileList["spec/integration_spec.rb"]
+Rake::TestTask.new(:test_download) do |t|
+  t.test_files = FileList["spec/integration/download_spec.rb"]
 end

data/bin/helper/driver.rb

@@ -36,7 +36,7 @@ class Driver
       plugin.get_urls_and_filenames(url, @params)
       
     rescue ViddlRb::PluginBase::CouldNotDownloadVideoError => e
-      raise "ERROR: The video could not be downloaded.\n" +
+      raise "CouldNotDownloadVideoError.\n" +
             "Reason: #{e.message}"
     rescue StandardError => e
       raise "Error while running the #{plugin.name.inspect} plugin. Maybe it has to be updated?\n" +

data/bin/helper/parameter-parser.rb

@@ -34,6 +34,10 @@ class ParameterParser
     optparse = OptionParser.new do |opts|
       opts.banner = "Usage: viddl-rb URL [options]"
 
+      opts.on('-h', '--help', 'Display this screen') do
+        print_help_and_exit(opts)
+      end
+
       opts.on("-e", "--extract-audio", "Save video audio to file") do
         if ViddlRb::UtilityHelper.os_has?("ffmpeg")
           options[:extract_audio] = true
@@ -75,21 +79,18 @@ class ParameterParser
       end
 
       opts.on("-q", "--quality QUALITY",
-              "Specifies the video format and resolution in the following way => resolution:extension (e.g. 720:mp4)") do |quality|
-        if match = quality.match(/(\d+):(.*)/)
-          res = match[1]
-          ext = match[2]
-        elsif match = quality.match(/\d+/)
-          res = match[0]
-          ext = nil
-        else
-          raise OptionParse.InvalidArgument.new("#{quality} is not a valid argument.")
-        end
-        options[:quality] = {:extension => ext, :resolution => res}
-      end
-
-      opts.on_tail('-h', '--help', 'Display this screen') do
-        print_help_and_exit(opts)
+              "Specifies the video format and resolution in the following way: width:height:res (e.g. 1280:720:mp4). " +
+              "The width, height and resolution may be omitted with a *. For example, to match any quality with a " +
+              "width of 720 pixels in any format specify --quality *:720:*") do |quality|
+
+        tokens = quality.split(":")
+        raise OptionParser::InvalidArgument.new("#{quality} is not a valid argument.") unless tokens.size == 3
+        width, height, ext = tokens
+        validate_quality_options!(width, height, ext, quality)
+
+        options[:quality] = {width: width == "*" ? nil : width.to_i,
+                             height: height == "*" ? nil : height.to_i,
+                             extension: ext == "*" ? nil : ext}
       end
     end
 
@@ -109,7 +110,25 @@ class ParameterParser
   def self.validate_url!(url)
     unless url =~ /^http/
       raise OptionParser::InvalidArgument.new(
-                                              "please include 'http' with your URL e.g. http://www.youtube.com/watch?v=QH2-TGUlwu4")
+          "please include 'http' with your URL e.g. http://www.youtube.com/watch?v=QH2-TGUlwu4")
+    end
+  end
+
+  def self.validate_quality_options!(width, height, extension, quality)
+    if !width =~ /(\*|\d+)/ || !height =~ /(\*|\d+)/ || !extension =~ /(\*|\w+)/
+      raise OptionParser::InvalidArgument.new("#{quality} is not a valid argument.")
     end
   end
 end
+
+
+
+
+
+
+
+
+
+
+
+

data/helper/plugin-helper.rb

@@ -55,6 +55,4 @@ module ViddlRb
       nil
     end
   end
-
 end
-

data/helper/utility-helper.rb

@@ -3,11 +3,29 @@
 module ViddlRb
 
   class UtilityHelper
-    #loads all plugins in the plugin directory.
-    #the plugin classes are dynamically added to the ViddlRb module.
+
+    # Loads all plugins in the plugin directory.
+    # The plugin classes are dynamically added to the ViddlRb module.
+    # A plugin can have helper classes. These classes must exist in a in directory under the
+    # plugins directory that has the same name as the plugin filename wihouth the .rb extension.
+    # All classes found in such a directory will dynamically added as inner classes of the
+    # plugin class.
     def self.load_plugins
-      Dir[File.join(File.dirname(__FILE__), "../plugins/*.rb")].each do |plugin|
-        ViddlRb.class_eval(File.read(plugin))
+      plugins_dir  = File.join(File.dirname(__FILE__), "../plugins")
+      plugin_paths = Dir[File.join(plugins_dir, "*.rb")]
+
+      plugin_paths.each do |path|
+        filename = File.basename(path, File.extname(path))
+        plugin_code = File.read(path)
+        class_name = plugin_code[/class (\w+) < PluginBase/, 1]
+        components = Dir[File.join(plugins_dir, filename, "*.rb")]
+
+        ViddlRb.class_eval(plugin_code)
+
+        components.each do |component|
+          code = File.read(component)
+          ViddlRb.const_get(class_name).class_eval(code)
+        end
       end
     end
 

data/plugins/youtube.rb

@@ -1,231 +1,26 @@
-# -*- coding: utf-8 -*-
 
 class Youtube < PluginBase
 
-  # see http://en.wikipedia.org/wiki/YouTube#Quality_and_codecs
-  # TODO: we don't have all the formats from the wiki article here
-  VIDEO_FORMATS = {
-     "38" => {:extension => "mp4",  :name => "MP4 Highest Quality 4096x3027 (H.264, AAC)"},
-     "37" => {:extension => "mp4",  :name => "MP4 Highest Quality 1920x1080 (H.264, AAC)"},
-     "22" => {:extension => "mp4",  :name => "MP4 1280x720 (H.264, AAC)"},
-     "46" => {:extension => "webm", :name => "WebM 1920x1080 (VP8, Vorbis)"},
-     "45" => {:extension => "webm", :name => "WebM 1280x720 (VP8, Vorbis)"},
-     "44" => {:extension => "webm", :name => "WebM 854x480 (VP8, Vorbis)"},
-     "43" => {:extension => "webm", :name => "WebM 480x360 (VP8, Vorbis)"},
-     "18" => {:extension => "mp4",  :name => "MP4 640x360 (H.264, AAC)"},
-     "35" => {:extension => "flv",  :name => "FLV 854x480 (H.264, AAC)"},
-     "34" => {:extension => "flv",  :name => "FLV 640x360 (H.264, AAC)"},
-     "6"  => {:extension => "flv",  :name => "FLV 640x360 (Soerenson H.263)"},
-     "5"  => {:extension => "flv",  :name => "FLV 400x240 (Soerenson H.263)"},
-     "36" => {:extension => "3gp",  :name => "3gp Medium Quality - 320x240 (MPEG-4 Visual, AAC)"},
-     "17" => {:extension => "3gp",  :name => "3gp Medium Quality - 176x144 (MPEG-4 Visual, AAC)"},
-     "13" => {:extension => "3gp",  :name => "3gp Low Quality - 176x144 (MPEG-4 Visual, AAC)"},
-     "82" => {:extension => "mp4",  :name => "MP4 360p (H.264 AAC)"}, 
-     "83" => {:extension => "mp4",  :name => "MP4 240p (H.264 AAC)"}, 
-     "84" => {:extension => "mp4",  :name => "MP4 720p (H.264 AAC)"}, 
-     "85" => {:extension => "mp4",  :name => "MP4 520p (H.264 AAC)"}, 
-    "100" => {:extension => "webm", :name => "WebM 360p (VP8 Vorbis)"}, 
-    "101" => {:extension => "webm", :name => "WebM 360p (VP8 Vorbis)"}, 
-    "102" => {:extension => "webm", :name => "WebM 720p (VP8 Vorbis)"}, 
-    "120" => {:extension => "flv",  :name => "FLV 720p (H.264 AAC)"}, 
-    "133" => {:extension => "mp4",  :name => "MP4 240p (H.264)"}, 
-    "134" => {:extension => "mp4",  :name => "MP4 360p (H.264)"}, 
-    "135" => {:extension => "mp4",  :name => "MP4 480p (H.264)"}, 
-    "136" => {:extension => "mp4",  :name => "MP4 720p (H.264)"}, 
-    "137" => {:extension => "mp4",  :name => "MP4 1080p  (H.264)"}, 
-    "139" => {:extension => "mp4",  :name => "MP4 (AAC)"}, 
-    "140" => {:extension => "mp4",  :name => "MP4 (AAC"}, 
-    "141" => {:extension => "mp4",  :name => "MP4 (AAC)"}, 
-    "160" => {:extension => "mp4",  :name => "MP4 (H.264)"}, 
-    "171" => {:extension => "webm", :name => "WebM (Vorbis)"}, 
-    "172" => {:extension => "webm", :name => "WebM (Vorbis)"}
-  }
-
-  DEFAULT_FORMAT_ORDER = %w[38 37 22 46 45 44 43 18 35 34 6 5 36 17 13 82 83 84 85 100 101 102 120 133 134 135 136 137 139 140 141 160 171 172]
-  VIDEO_INFO_URL       = "http://www.youtube.com/get_video_info?video_id="
-  VIDEO_INFO_PARMS     = "&ps=default&eurl=&gl=US&hl=en"
-
   # this will be called by the main app to check whether this plugin is responsible for the url passed
   def self.matches_provider?(url)
     url.include?("youtube.com") || url.include?("youtu.be")
   end
 
   def self.get_urls_and_filenames(url, options = {})
-    @quality    = options[:quality]
-    filter      = options[:playlist_filter]
-    parser      = PlaylistParser.new
-    return_vals = []
 
-    if playlist_urls = parser.get_playlist_urls(url, filter)
-      playlist_urls.each { |url| return_vals << grab_single_url_filename(url, options) }
-    else
-      return_vals << grab_single_url_filename(url, options)
-    end
-
-    clean_return_values(return_vals)
-  end
+    @url_resolver   = UrlResolver.new
+    @video_resolver = VideoResolver.new(Decipherer.new)
+    @format_picker  = FormatPicker.new(options)
 
-  def self.clean_return_values(return_values)
-    cleaned = return_values.reject { |val| val == :no_embed }
+    urls = @url_resolver.get_all_urls(url, options[:filter])
+    videos = get_videos(urls)
 
-    if cleaned.empty?
-      download_error("No videos could be downloaded.")
-    else
-      cleaned
-    end
-  end
-
-  def self.grab_single_url_filename(url, options)
-    UrlGrabber.new(url, self, options).process
-  end
-
-  class UrlGrabber
-    attr_accessor :url, :options, :plugin, :quality
-
-    def initialize(url, plugin, options)
-      @url     = url
-      @plugin  = plugin
-      @options = options
-      @quality = options[:quality]
-    end
-
-    def process
-      grab_url_embeddable(url) || grab_url_non_embeddable(url)
-    end
-
-    # VEVO video: http://www.youtube.com/watch?v=A_J7kEhY9sM
-    # Non-VEVO video: http://www.youtube.com/watch?v=WkkC9cK8Hz0
-
-    def grab_url_embeddable(url)
-      video_info   = get_video_info(url)
-      video_params = extract_video_parameters(video_info)
-
-      unless video_params[:embeddable]
-        Youtube.notify("VIDEO IS NOT EMBEDDABLE")
-        return false
-      end
-
-      urls_formats    = extract_urls_formats(video_info)
-      selected_format = choose_format(urls_formats)
-      title           = video_params[:title]
-      file_name       = PluginBase.make_filename_safe(title) + "." + VIDEO_FORMATS[selected_format][:extension]
-
-      {:url => urls_formats[selected_format], :name => file_name}
-    end
-
-    def grab_url_non_embeddable(url)
-      video_info      = open(url).read
-      stream_map      = video_info[/url_encoded_fmt_stream_map\" *: *\"([^\"]+)\"/,1]
-
-      # Video has been deleted!
-      if stream_map.nil? or !stream_map.index("been+removed").nil?
-        Youtube.notify("VIDEO IS REMOVED")
-        return false
-      end
-
-      urls_formats    = parse_stream_map(url_decode(stream_map))
-      selected_format = choose_format(urls_formats)
-      title           = video_info[/<meta name="title" content="([^"]*)">/, 1]
-      file_name       = PluginBase.make_filename_safe(title) + "." + VIDEO_FORMATS[selected_format][:extension]
-
-      # cleaning
-      clean_url = urls_formats[selected_format].gsub(/\\u0026[^&]*/, "").split(',type=video').first
-      {:url => clean_url, :name => file_name}
-    end
-
-    def get_video_info(url)
-      id = extract_video_id(url)
-      request_url = VIDEO_INFO_URL + id + VIDEO_INFO_PARMS
-      open(request_url).read
-    end
-
-    def extract_video_parameters(video_info)
-      video_params = CGI.parse(url_decode(video_info))
-
-      {
-        :title      => video_params["title"].first,
-        :length_sec => video_params["length_seconds"].first,
-        :author     => video_params["author"].first,
-        :embeddable => (video_params["status"].first != "fail")
-      }
-    end
-
-    def extract_video_id(url)
-      # the youtube video ID looks like this: [...]v=abc5a5_afe5agae6g&[...], we only want the ID (the \w in the brackets)
-      # addition: might also look like this /v/abc5-a5afe5agae6g
-      # alternative:  video_id = url[/v[\/=]([\w-]*)&?/, 1]
-      url = open(url).base_uri.to_s if url.include?("youtu.be")
-      video_id = url[/(v|embed)[=\/]([^\/\?\&]*)/, 2]
-
-      if video_id
-        Youtube.notify("ID FOUND: #{video_id}")
-        video_id
-      else
-        Youtube.download_error("No video id found.")
-      end
-    end
-
-    def extract_urls_formats(video_info)
-      stream_map = video_info[/url_encoded_fmt_stream_map=(.+?)(?:&|$)/, 1]
-      parse_stream_map(stream_map)
-    end
-
-    def choose_format(urls_formats)
-      available_formats = urls_formats.keys
-
-      if @quality                        #if the user specified a format
-        ext = @quality[:extension]
-        res = @quality[:resolution] || ""
-        #gets a nested array with all the formats of the same res as the user wanted
-        requested = VIDEO_FORMATS.select { |id, format| available_formats.include?(id) && format[:name].include?(res) }.to_a
-
-        if requested.empty?
-          Youtube.notify "Requested format \"#{res}:#{ext}\" not found. Downloading default format."
-          get_default_format(available_formats)
-        else
-          pick = requested.find { |format| format[1][:extension] == ext }             # get requsted extension if possible
-          pick ? pick.first : get_default_format(requested.map { |req| req.first })   # else return the default format
-        end
-      else
-        get_default_format(available_formats)
-      end
-    end
-
-    def parse_stream_map(stream_map)
-      urls = extract_download_urls(stream_map)
-      formats_urls = {}
-
-      urls.each do |url|
-        format = url[/itag=(\d+)/, 1]
-        formats_urls[format] = url
-      end
-
-      formats_urls
-    end
-
-    def extract_download_urls(stream_map)
-      entries = stream_map.split("%2C")
-      decoded = entries.map { |entry| url_decode(entry) }
-
-      decoded.map do |entry|
-        url = entry[/url=(.*?itag=.+?)(?:itag=|;|$)/, 1]
-        sig = entry[/sig=(.+?)(?:&|$)/, 1]
-
-        url + "&signature=#{sig}"
-      end
-    end
-
-    def get_default_format(available)
-      DEFAULT_FORMAT_ORDER.find { |default| available.include?(default) }
-    end
-
-    def url_decode(text)
-      while text != (decoded = CGI::unescape(text)) do
-        text = decoded
-      end
-      text
+    return_value = videos.map do |video|
+      format = @format_picker.pick_format(video)
+      make_url_filname_hash(video, format)
     end
 
+    return_value.empty? ? download_error("No videos could be downloaded.") : return_value
   end
 
   def self.notify(message)
@@ -236,83 +31,23 @@ class Youtube < PluginBase
     raise CouldNotDownloadVideoError, message
   end
 
-  #
-  # class PlaylistParser
-  #_____________________
-
-  class PlaylistParser
-
-    PLAYLIST_FEED = "http://gdata.youtube.com/feeds/api/playlists/%s?&max-results=50&v=2"
-    USER_FEED     = "http://gdata.youtube.com/feeds/api/users/%s/uploads?&max-results=50&v=2"
-
-    def get_playlist_urls(url, filter = nil)
-      @filter = filter
-
-      if url.include?("view_play_list") || url.include?("playlist?list=")     # if playlist URL
-        parse_playlist(url)
-      elsif username = url[/\/(?:user|channel)\/([\w\d]+)(?:\/|$)/, 1]        # if user/channel URL
-        parse_user(username)
-      else                                                                    # if neither return nil
-        nil
-      end
-    end
-
-    def parse_playlist(url)
-      #http://www.youtube.com/view_play_list?p=F96B063007B44E1E&search_query=welt+auf+schwäbisch
-      #http://www.youtube.com/watch?v=9WEP5nCxkEY&videos=jKY836_WMhE&playnext_from=TL&playnext=1
-      #http://www.youtube.com/watch?v=Tk78sr5JMIU&videos=jKY836_WMhE
-
-      playlist_ID = url[/(?:list=PL|p=)(.+?)(?:&|\/|$)/, 1]
-      Youtube.notify "Playlist ID: #{playlist_ID}"
-      feed_url = PLAYLIST_FEED % playlist_ID
-      url_array = get_video_urls(feed_url)
-      Youtube.notify "#{url_array.size} links found!"
-      url_array
-    end
-
-    def parse_user(username)
-      Youtube.notify "User: #{username}"
-      feed_url = USER_FEED % username
-      url_array = get_video_urls(feed_url)
-      Youtube.notify "#{url_array.size} links found!"
-      url_array
-    end
-
-    #get all videos and return their urls in an array
-    def get_video_urls(feed_url)
-      Youtube.notify "Retrieving videos..."
-      urls_titles = {}
-      result_feed = Nokogiri::XML(open(feed_url))
-      urls_titles.merge!(grab_urls_and_titles(result_feed))
-
-      #as long as the feed has a next link we follow it and add the resulting video urls
-      loop do
-        next_link = result_feed.search("//feed/link[@rel='next']").first
-        break if next_link.nil?
-        result_feed = Nokogiri::HTML(open(next_link["href"]))
-        urls_titles.merge!(grab_urls_and_titles(result_feed))
+  def self.get_videos(urls)
+    videos = urls.map do |url|
+      begin
+        @video_resolver.get_video(url)
+      rescue VideoResolver::VideoRemovedError
+        notify "The video #{url} has been removed."
+      rescue => e
+        notify "Error getting the video: #{e.message}"
       end
-
-      filter_urls(urls_titles)
     end
 
-    #extract all video urls and their titles from a feed and return in a hash
-    def grab_urls_and_titles(feed)
-      feed.remove_namespaces!  #so that we can get to the titles easily
-      urls   = feed.search("//entry/link[@rel='alternate']").map { |link| link["href"] }
-      titles = feed.search("//entry/group/title").map { |title| title.text }
-      Hash[urls.zip(titles)]    #hash like this: url => title
-    end
+    videos.reject(&:nil?)
+  end
 
-    #returns only the urls that match the --filter argument regex (if present)
-    def filter_urls(url_hash)
-      if @filter
-        Youtube.notify "Using filter: #{@filter}"
-        filtered = url_hash.select { |url, title| title =~ @filter }
-        filtered.keys
-      else
-        url_hash.keys
-      end
-    end
+  def self.make_url_filname_hash(video, format)
+    url = video.get_download_url(format.itag)
+    name = PluginBase.make_filename_safe(video.title) + ".#{format.extension}"
+    {url: url, name: name}
   end
 end

data/plugins/youtube/decipherer.rb

@@ -0,0 +1,149 @@
+
+class Decipherer
+
+  class UnknownCipherVersionError < StandardError; end 
+  class UnknownCipherOperationError < StandardError; end 
+
+  CIPHERS = {
+    'vflNzKG7n' => 's3 r s2 r s1 r w67',              # 30 Jan 2013, untested
+    'vfllMCQWM' => 's2 w46 r w27 s2 w43 s2 r',        # 15 Feb 2013, untested
+    'vflJv8FA8' => 's1 w51 w52 r',                    # 12 Mar 2013, untested
+    'vflR_cX32' => 's2 w64 s3',                       # 11 Apr 2013, untested
+    'vflveGye9' => 'w21 w3 s1 r w44 w36 r w41 s1',    # 02 May 2013, untested
+    'vflj7Fxxt' => 'r s3 w3 r w17 r w41 r s2',        # 14 May 2013, untested
+    'vfltM3odl' => 'w60 s1 w49 r s1 w7 r s2 r',       # 23 May 2013
+    'vflDG7-a-' => 'w52 r s3 w21 r s3 r',             # 06 Jun 2013
+    'vfl39KBj1' => 'w52 r s3 w21 r s3 r',             # 12 Jun 2013
+    'vflmOfVEX' => 'w52 r s3 w21 r s3 r',             # 21 Jun 2013
+    'vflJwJuHJ' => 'r s3 w19 r s2',                   # 25 Jun 2013
+    'vfl_ymO4Z' => 'r s3 w19 r s2',                   # 26 Jun 2013
+    'vfl26ng3K' => 'r s2 r',                          # 08 Jul 2013
+    'vflcaqGO8' => 'w24 w53 s2 w31 w4',               # 11 Jul 2013
+    'vflQw-fB4' => 's2 r s3 w9 s3 w43 s3 r w23',      # 16 Jul 2013
+    'vflSAFCP9' => 'r s2 w17 w61 r s1 w7 s1',         # 18 Jul 2013
+    'vflART1Nf' => 's3 r w63 s2 r s1',                # 22 Jul 2013
+    'vflLC8JvQ' => 'w34 w29 w9 r w39 w24',            # 25 Jul 2013
+    'vflm_D8eE' => 's2 r w39 w55 w49 s3 w56 w2',      # 30 Jul 2013
+    'vflTWC9KW' => 'r s2 w65 r',                      # 31 Jul 2013
+    'vflRFcHMl' => 's3 w24 r',                        # 04 Aug 2013
+    'vflM2EmfJ' => 'w10 r s1 w45 s2 r s3 w50 r',      # 06 Aug 2013
+    'vflz8giW0' => 's2 w18 s3',                       # 07 Aug 2013
+    'vfl_wGgYV' => 'w60 s1 r s1 w9 s3 r s3 r',        # 08 Aug 2013
+    'vfl1HXdPb' => 'w52 r w18 r s1 w44 w51 r s1',     # 12 Aug 2013
+    'vflkn6DAl' => 'w39 s2 w57 s2 w23 w35 s2',        # 15 Aug 2013
+    'vfl2LOvBh' => 'w34 w19 r s1 r s3 w24 r',         # 16 Aug 2013
+    'vfl-bxy_m' => 'w48 s3 w37 s2',                   # 20 Aug 2013
+    'vflZK4ZYR' => 'w19 w68 s1',                      # 21 Aug 2013
+    'vflh9ybst' => 'w48 s3 w37 s2',                   # 21 Aug 2013
+    'vflapUV9V' => 's2 w53 r w59 r s2 w41 s3',        # 27 Aug 2013
+    'vflg0g8PQ' => 'w36 s3 r s2',                     # 28 Aug 2013
+    'vflHOr_nV' => 'w58 r w50 s1 r s1 r w11 s3',      # 30 Aug 2013
+    'vfluy6kdb' => 'r w12 w32 r w34 s3 w35 w42 s2',   # 05 Sep 2013
+    'vflkuzxcs' => 'w22 w43 s3 r s1 w43',             # 10 Sep 2013
+    'vflGNjMhJ' => 'w43 w2 w54 r w8 s1',              # 12 Sep 2013
+    'vfldJ8xgI' => 'w11 r w29 s1 r s3',               # 17 Sep 2013
+    'vfl79wBKW' => 's3 r s1 r s3 r s3 w59 s2',        # 19 Sep 2013
+    'vflg3FZfr' => 'r s3 w66 w10 w43 s2',             # 24 Sep 2013
+    'vflUKrNpT' => 'r s2 r w63 r',                    # 25 Sep 2013
+    'vfldWnjUz' => 'r s1 w68',                        # 30 Sep 2013
+    'vflP7iCEe' => 'w7 w37 r s1',                     # 03 Oct 2013
+    'vflzVne63' => 'w59 s2 r',                        # 07 Oct 2013
+    'vflO-N-9M' => 'w9 s1 w67 r s3',                  # 09 Oct 2013
+    'vflZ4JlpT' => 's3 r s1 r w28 s1',                # 11 Oct 2013
+    'vflDgXSDS' => 's3 r s1 r w28 s1',                # 15 Oct 2013
+    'vflW444Sr' => 'r w9 r s1 w51 w27 r s1 r',        # 17 Oct 2013
+    'vflK7RoTQ' => 'w44 r w36 r w45',                 # 21 Oct 2013
+    'vflKOCFq2' => 's1 r w41 r w41 s1 w15',           # 23 Oct 2013
+    'vflcLL31E' => 's1 r w41 r w41 s1 w15',           # 28 Oct 2013
+    'vflz9bT3N' => 's1 r w41 r w41 s1 w15',           # 31 Oct 2013
+    'vfliZsE79' => 'r s3 w49 s3 r w58 s2 r s2',       # 05 Nov 2013
+    'vfljOFtAt' => 'r s3 r s1 r w69 r',               # 07 Nov 2013
+    'vflqSl9GX' => 'w32 r s2 w65 w26 w45 w24 w40 s2', # 14 Nov 2013
+    'vflFrKymJ' => 'w32 r s2 w65 w26 w45 w24 w40 s2', # 15 Nov 2013
+    'vflKz4WoM' => 'w50 w17 r w7 w65',                # 19 Nov 2013
+    'vflhdWW8S' => 's2 w55 w10 s3 w57 r w25 w41',     # 21 Nov 2013
+    'vfl66X2C5' => 'r s2 w34 s2 w39',                 # 26 Nov 2013
+    'vflCXG8Sm' => 'r s2 w34 s2 w39',                 # 02 Dec 2013
+    'vfl_3Uag6' => 'w3 w7 r s2 w27 s2 w42 r',         # 04 Dec 2013
+    'vflQdXVwM' => 's1 r w66 s2 r w12',               # 10 Dec 2013
+    'vflCtc3aO' => 's2 r w11 r s3 w28',               # 12 Dec 2013
+    'vflCt6YZX' => 's2 r w11 r s3 w28',               # 17 Dec 2013
+    'vflG49soT' => 'w32 r s3 r s1 r w19 w24 s3',      # 18 Dec 2013
+    'vfl4cHApe' => 'w25 s1 r s1 w27 w21 s1 w39',      # 06 Jan 2014
+    'vflwMrwdI' => 'w3 r w39 r w51 s1 w36 w14',       # 06 Jan 2014
+    'vfl4AMHqP' => 'r s1 w1 r w43 r s1 r',            # 09 Jan 2014
+    'vfln8xPyM' => 'w36 w14 s1 r s1 w54',             # 10 Jan 2014
+    'vflVSLmnY' => 's3 w56 w10 r s2 r w28 w35',       # 13 Jan 2014
+    'vflkLvpg7' => 'w4 s3 w53 s2',                    # 15 Jan 2014
+    'vflbxes4n' => 'w4 s3 w53 s2',                    # 15 Jan 2014
+    'vflmXMtFI' => 'w57 s3 w62 w41 s3 r w60 r',       # 23 Jan 2014
+    'vflYDqEW1' => 'w24 s1 r s2 w31 w4 w11 r',        # 24 Jan 2014
+    'vflapGX6Q' => 's3 w2 w59 s2 w68 r s3 r s1',      # 28 Jan 2014
+    'vflLCYwkM' => 's3 w2 w59 s2 w68 r s3 r s1',      # 29 Jan 2014
+    'vflcY_8N0' => 's2 w36 s1 r w18 r w19 r',         # 30 Jan 2014
+    'vfl9qWoOL' => 'w68 w64 w28 r',                   # 03 Feb 2014
+    'vfle-mVwz' => 's3 w7 r s3 r w14 w59 s3 r',       # 04 Feb 2014
+    'vfltdb6U3' => 'w61 w5 r s2 w69 s2 r',            # 05 Feb 2014
+    'vflLjFx3B' => 'w40 w62 r s2 w21 s3 r w7 s3',     # 10 Feb 2014
+    'vfliqjKfF' => 'w40 w62 r s2 w21 s3 r w7 s3',     # 13 Feb 2014
+    'ima-vflxBu-5R' => 'w40 w62 r s2 w21 s3 r w7 s3', # 13 Feb 2014
+    'ima-vflrGwWV9' => 'w36 w45 r s2 r'               # 20 Feb 2014
+  }
+
+  def decipher_with_version(cipher, cipher_version)
+    operations = CIPHERS[cipher_version] 
+    raise UnknownCipherVersionError.new("Unknown cipher version: #{cipher_version}") unless operations
+
+    decipher_with_operations(cipher, operations.split)
+  end
+
+  def decipher_with_operations(cipher, operations)
+    cipher = cipher.dup
+
+    operations.each do |op|
+      cipher = apply_operation(cipher, op)
+    end
+    cipher
+  end
+
+  private
+
+  def apply_operation(cipher, op)
+    op = check_operation(op)
+
+    case op[0].downcase
+    when "r"
+      cipher.reverse
+    when "w"
+      index = get_op_index(op)
+      swap_first_char(cipher, index)
+    when "s"
+      index = get_op_index(op)
+      cipher[index, cipher.length - 1] # slice from index to the end
+    else
+      raise_unknown_op_error(op)
+    end
+  end
+
+  def check_operation(op)
+    raise_unknown_op_error(op) if op.nil? || !op.respond_to?(:to_s)
+    op.to_s
+  end
+
+  def swap_first_char(string, index)
+    temp = string[0]
+    string[0] = string[index]
+    string[index] = temp
+    string
+  end
+
+  def get_op_index(op)
+    index = op[/.(\d+)/, 1]
+    raise_unknown_op_error(op) unless index
+    index.to_i
+  end
+
+  def raise_unknown_op_error(op)
+    raise UnknownCipherOperationError.new("Unkown operation: #{op}")
+  end
+end

data/plugins/youtube/format_picker.rb

@@ -0,0 +1,116 @@
+
+class FormatPicker
+
+  Format = Struct.new(:itag, :extension, :resolution, :name)
+  Resolution = Struct.new(:width, :height)
+
+  # see http://en.wikipedia.org/wiki/YouTube#Quality_and_codecs
+  # TODO: we don't have all the formats from the wiki article here
+  # :u means the resolution is unknown.
+  FORMATS = [
+    Format.new("38",  "mp4",   Resolution.new(4096, 3027),  "MP4 Highest Quality 4096x3027 (H.264, AAC)"),
+    Format.new("37",  "mp4",   Resolution.new(1920, 1080),  "MP4 Highest Quality 1920x1080 (H.264, AAC)"),
+    Format.new("22",  "mp4",   Resolution.new(1280, 720),   "MP4 1280x720 (H.264, AAC)"),
+    Format.new("46",  "webm",  Resolution.new(1920, 1080),  "WebM 1920x1080 (VP8, Vorbis)"),
+    Format.new("45",  "webm",  Resolution.new(1280, 720),   "WebM 1280x720 (VP8, Vorbis)"),
+    Format.new("44",  "webm",  Resolution.new(854, 480),    "WebM 854x480 (VP8, Vorbis)"),
+    Format.new("43",  "webm",  Resolution.new(480, 360),    "WebM 480x360 (VP8, Vorbis)"),
+    Format.new("18",  "mp4",   Resolution.new(640, 360),    "MP4 640x360 (H.264, AAC)"),
+    Format.new("35",  "flv",   Resolution.new(854, 480),    "FLV 854x480 (H.264, AAC)"),
+    Format.new("34",  "flv",   Resolution.new(640, 360),    "FLV 640x360 (H.264, AAC)"),
+    Format.new("6",   "flv",   Resolution.new(640, 360),    "FLV 640x360 (Soerenson H.263)"),
+    Format.new("5",   "flv",   Resolution.new(400, 240),    "FLV 400x240 (Soerenson H.263)"),
+    Format.new("36",  "3gp",   Resolution.new(320, 240),    "3gp Medium Quality - 320x240 (MPEG-4 Visual, AAC)"),
+    Format.new("17",  "3gp",   Resolution.new(174, 144),    "3gp Medium Quality - 176x144 (MPEG-4 Visual, AAC)"),
+    Format.new("13",  "3gp",   Resolution.new(176, 144),    "3gp Low Quality - 176x144 (MPEG-4 Visual, AAC)"),
+    Format.new("82",  "mp4",   Resolution.new(480, 360),    "MP4 360p (H.264 AAC)"),
+    Format.new("83",  "mp4",   Resolution.new(320, 240),    "MP4 240p (H.264 AAC)"),
+    Format.new("84",  "mp4",   Resolution.new(1280, 720),   "MP4 720p (H.264 AAC)"),
+    Format.new("85",  "mp4",   Resolution.new(960, 520),    "MP4 520p (H.264 AAC)"),
+    Format.new("100", "webm",  Resolution.new(480, 360),    "WebM 360p (VP8 Vorbis)"),
+    Format.new("101", "webm",  Resolution.new(480, 360),    "WebM 360p (VP8 Vorbis)"),
+    Format.new("102", "webm",  Resolution.new(1280, 720),   "WebM 720p (VP8 Vorbis)"),
+    Format.new("120", "flv",   Resolution.new(1280, 720),   "FLV 720p (H.264 AAC)"),
+    Format.new("133", "mp4",   Resolution.new(320, 240),    "MP4 240p (H.264)"),
+    Format.new("134", "mp4",   Resolution.new(480, 360),    "MP4 360p (H.264)"),
+    Format.new("135", "mp4",   Resolution.new(640, 480),    "MP4 480p (H.264)"),
+    Format.new("136", "mp4",   Resolution.new(1280, 720),   "MP4 720p (H.264)"),
+    Format.new("137", "mp4",   Resolution.new(1920, 1080),  "MP4 1080p (H.264)"),
+    Format.new("139", "mp4",   Resolution.new(:u, :u),      "MP4 (AAC)"),
+    Format.new("140", "mp4",   Resolution.new(:u, :u),      "MP4 (AAC"),
+    Format.new("141", "mp4",   Resolution.new(:u, :u),      "MP4 (AAC)"),
+    Format.new("160", "mp4",   Resolution.new(:u, :u),      "MP4 (H.264)"),
+    Format.new("171", "webm",  Resolution.new(:u, :u),      "WebM (Vorbis)"),
+    Format.new("172", "webm",  Resolution.new(:u, :u),      "WebM (Vorbis)")
+  ]
+
+  DEFAULT_FORMAT_ORDER = %w[38 37 22 46 45 44 43 18 35 34 6 5 36 17 13 82 83 84 85 100 101 102 120 133 134 135 136 137 139 140 141 160 171 172]
+
+  def initialize(options)
+    @options = options
+  end
+
+  def pick_format(video)
+    if quality = @options[:quality]
+      get_quality_format(video, quality)
+    else
+      get_default_format_for_video(video)
+    end
+  end
+
+  private
+
+  def get_default_format_for_video(video)
+    available = get_available_formats_for_video(video)
+    get_default_format(available)
+  end
+
+  def get_available_formats_for_video(video)
+    video.available_itags.map { |itag| get_format_by_itag(itag) }
+  end
+
+  def get_format_by_itag(itag)
+    FORMATS.find { |format| format.itag == itag }
+  end
+
+  def get_default_format(formats)
+    DEFAULT_FORMAT_ORDER.each do |itag|
+      default_format = formats.find { |format| format.itag == itag }
+      return default_format if default_format
+    end
+    nil
+  end
+
+  def get_quality_format(video, quality)
+    available = get_available_formats_for_video(video) 
+
+    matches = available.select do |format|
+      matches_extension?(format, quality) && matches_resolution?(format, quality)
+    end
+
+    select_format(video, matches)
+  end
+
+  def matches_extension?(format, quality)
+    return false if quality[:extension] && quality[:extension] != format.extension
+    true
+  end
+
+  def matches_resolution?(format, quality)
+    return false if quality[:width] && quality[:width] != format.resolution.width
+    return false if quality[:height] && quality[:height] != format.resolution.height
+    true
+  end
+
+  def select_format(video, formats)
+    case formats.length
+    when 0
+      Youtube.notify "Requested format not found. Downloading default format."
+      get_default_format_for_video(video)
+    when 1
+      formats.first
+    else
+      get_default_format(matches_resolution)
+    end
+  end
+end

data/plugins/youtube/url_resolver.rb

@@ -0,0 +1,77 @@
+class UrlResolver
+
+  PLAYLIST_FEED = "http://gdata.youtube.com/feeds/api/playlists/%s?&max-results=50&v=2"
+  USER_FEED     = "http://gdata.youtube.com/feeds/api/users/%s/uploads?&max-results=50&v=2"
+
+  def get_all_urls(url, filter = nil)
+    @filter = filter
+
+    if url.include?("view_play_list") || url.include?("playlist?list=")     # if playlist URL
+      parse_playlist(url)
+    elsif username = url[/\/(?:user|channel)\/([\w\d]+)(?:\/|$)/, 1]        # if user/channel URL
+      parse_user(username)
+    else                                                                    # if neither return nil
+      [url]
+    end
+  end
+
+  private
+
+  def parse_playlist(url)
+    #http://www.youtube.com/view_play_list?p=F96B063007B44E1E&search_query=welt+auf+schwäbisch
+    #http://www.youtube.com/watch?v=9WEP5nCxkEY&videos=jKY836_WMhE&playnext_from=TL&playnext=1
+    #http://www.youtube.com/watch?v=Tk78sr5JMIU&videos=jKY836_WMhE
+
+    playlist_ID = url[/(?:list=PL|p=)(.+?)(?:&|\/|$)/, 1]
+    Youtube.notify "Playlist ID: #{playlist_ID}"
+    feed_url = PLAYLIST_FEED % playlist_ID
+    url_array = get_video_urls(feed_url)
+    Youtube.notify "#{url_array.size} links found!"
+    url_array
+  end
+
+  def parse_user(username)
+    Youtube.notify "User: #{username}"
+    feed_url = USER_FEED % username
+    url_array = get_video_urls(feed_url)
+    Youtube.notify "#{url_array.size} links found!"
+    url_array
+  end
+
+  #get all videos and return their urls in an array
+  def get_video_urls(feed_url)
+    Youtube.notify "Retrieving videos..."
+    urls_titles = {}
+    result_feed = Nokogiri::XML(open(feed_url))
+    urls_titles.merge!(grab_urls_and_titles(result_feed))
+
+    #as long as the feed has a next link we follow it and add the resulting video urls
+    loop do
+      next_link = result_feed.search("//feed/link[@rel='next']").first
+      break if next_link.nil?
+      result_feed = Nokogiri::HTML(open(next_link["href"]))
+      urls_titles.merge!(grab_urls_and_titles(result_feed))
+    end
+
+    filter_urls(urls_titles)
+  end
+
+  #extract all video urls and their titles from a feed and return in a hash
+  def grab_urls_and_titles(feed)
+    feed.remove_namespaces!  #so that we can get to the titles easily
+    urls   = feed.search("//entry/link[@rel='alternate']").map { |link| link["href"] }
+    titles = feed.search("//entry/group/title").map { |title| title.text }
+    Hash[urls.zip(titles)]    #hash like this: url => title
+  end
+
+  #returns only the urls that match the --filter argument regex (if present)
+  def filter_urls(url_hash)
+    if @filter
+      Youtube.notify "Using filter: #{@filter}"
+      filtered = url_hash.select { |url, title| title =~ @filter }
+      filtered.keys
+    else
+      url_hash.keys
+    end
+  end
+end

data/plugins/youtube/video_resolver.rb

@@ -0,0 +1,111 @@
+
+class VideoResolver
+
+  class VideoRemovedError < StandardError; end
+
+  CORRECT_SIGNATURE_LENGTH = 81
+  SIGNATURE_URL_PARAMETER = "signature"
+
+  def initialize(decipherer)
+    @decipherer = decipherer
+  end
+
+  def get_video(url)
+    @json = load_json(url)
+    Video.new(get_title, parse_stream_map(get_stream_map))
+  end
+
+  private
+
+  def load_json(url)
+    html = open(url).read
+    json_data = html[/ytplayer\.config\s*=\s*(\{.+?\});/m, 1] 
+    MultiJson.load(json_data)
+  end
+
+  def get_stream_map
+    stream_map = @json["args"]["url_encoded_fmt_stream_map"]
+    raise VideoRemovedError.new if stream_map.nil? || stream_map.include?("been+removed")
+    stream_map
+  end
+
+  def get_html5player_version
+    @json["assets"]["js"][/html5player-(.+?)\.js/, 1]
+  end
+
+  def get_title
+    @json["args"]["title"]
+  end
+
+  #
+  # Returns a an array of hashes in the following format:
+  # [
+  #  {format: format_id, url: download_url},
+  #  {format: format_id, url: download_url}
+  #  ...
+  # ]
+  #
+  def parse_stream_map(stream_map)
+    entries = stream_map.split(",")
+
+    parsed = entries.map { |entry| parse_stream_map_entry(entry) }
+    parsed.each { |entry| apply_signature!(entry) if entry[:sig] }
+    parsed
+  end
+
+  def parse_stream_map_entry(entry)
+    # Note: CGI.parse puts each value in an array.
+    params = CGI.parse((entry))
+
+    {
+      itag: params["itag"].first,
+      sig:  fetch_signature(params),
+      url:  url_decode(params["url"].first)
+    }
+  end
+
+  # The signature key can be either "sig" or "s".
+  # Very rarely there is no "s" or "sig" paramater. In this case the signature is already
+  # applied and the the video can be downloaded directly.
+  def fetch_signature(params)
+    sig = params.fetch("sig", nil) || params.fetch("s", nil)
+    sig && sig.first
+  end
+
+  def url_decode(text)
+    while text != (decoded = CGI::unescape(text)) do
+      text = decoded
+    end
+    text
+  end
+
+  def apply_signature!(entry)
+    sig = get_deciphered_sig(entry[:sig])
+    entry[:url] << "&#{SIGNATURE_URL_PARAMETER}=#{sig}"
+    entry.delete(:sig)
+  end
+
+  def get_deciphered_sig(sig)
+    return sig if sig.length == CORRECT_SIGNATURE_LENGTH
+    #crequire 'pry'; binding.pry; exit
+    @decipherer.decipher_with_version(sig, get_html5player_version)
+  end
+
+  class Video
+    attr_reader :title
+
+    def initialize(title, itags_urls)
+      @title = title
+      @itags_urls = itags_urls
+    end
+
+    def available_itags
+      @itags_urls.map { |iu| iu[:itag] }
+    end
+
+    def get_download_url(itag)
+      itag_url = @itags_urls.find { |iu| iu[:itag] == itag }
+      itag_url[:url] if itag_url
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: viddl-rb
 version: !ruby/object:Gem::Version 
-  version: "0.95"
+  version: "0.96"
 platform: ruby
 authors: 
 - Marc Seeger
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2014-02-16 00:00:00 Z
+date: 2014-02-25 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: mime-types
@@ -95,6 +95,10 @@ files:
 - plugins/soundcloud.rb
 - plugins/veoh.rb
 - plugins/vimeo.rb
+- plugins/youtube/decipherer.rb
+- plugins/youtube/format_picker.rb
+- plugins/youtube/url_resolver.rb
+- plugins/youtube/video_resolver.rb
 - plugins/youtube.rb
 - Gemfile
 - Gemfile.lock
@@ -123,7 +127,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: viddl-rb
-rubygems_version: 2.0.14
+rubygems_version: 2.1.11
 signing_key: 
 specification_version: 4
 summary: An extendable commandline video downloader for flash video sites.