viddl-rb 0.75 → 0.76

data/Gemfile.lock

@@ -23,7 +23,7 @@ GEM
     mime-types (1.19)
     minitest (4.1.0)
     net-http-digest_auth (1.2.1)
-    net-http-persistent (2.7)
+    net-http-persistent (2.8)
     nokogiri (1.5.5)
     nokogiri (1.5.5-java)
     ntlm-http (0.1.1)

data/README.md

@@ -20,6 +20,7 @@ Viddl-rb supports the following command line options:
 -f, --filter REGEX               Filters a video playlist according to the regex (Youtube only right now)
 -s, --save-dir DIRECTORY         Specifies the directory where videos should be saved
 -d, --downloader TOOL            Specifies the tool to download with. Supports 'wget', 'curl' and 'net-http'
+-q, --quality QUALITY            Specifies the video format and resolution in the following way => resolution:extension (e.g. 720:mp4). Currently only supported by the Youtube plugin.
 -h, --help                       Displays the help screen
 ```
 
@@ -109,7 +110,6 @@ __Requirements:__
 * curl/wget or the [progress bar](http://github.com/nex3/ruby-progressbar/) gem  
 * [Nokogiri](http://nokogiri.org/)
 * [Mechanize](http://mechanize.rubyforge.org/)
-* [Rest-Client](https://github.com/archiloque/rest-client)
 * ffmpeg if you want to extract audio tracks from the videos
 
 __Co Maintainer:__

data/bin/helper/parameter-parser.rb

@@ -14,6 +14,8 @@ class ParameterParser
   # :title_only       => do not download, only print the titles to stdout
   # :playlist_filter  => a regular expression used to filter playlists
   # :save_dir         => the directory where the videos are saved
+  # :tool             => the download tool (wget, curl, net/http) to use
+  # :quality          => the resolution and format to download
   def self.parse_app_parameters(args)
 
     # Default option values are set here
@@ -23,7 +25,8 @@ class ParameterParser
       :title_only       => false,
       :playlist_filter  => nil,
       :save_dir         => DEFAULT_SAVE_DIR,
-      :tool             => nil
+      :tool             => nil,
+      :quality          => nil
     }
 
     optparse = OptionParser.new do |opts|
@@ -65,6 +68,20 @@ class ParameterParser
         end
       end
 
+      opts.on("-q", "--quality QUALITY", 
+          "Specifies the video format and resolution in the following way => resolution:extension (e.g. 720:mp4)") do |quality|
+        if match = quality.match(/(\d+):(.*)/)
+          res = match[1]
+          ext = match[2]
+        elsif match = quality.match(/\d+/)
+          res = match[0]
+          ext = nil
+        else
+          raise OptionParse.InvalidArgument.new("#{quality} is not a valid argument.")
+        end
+        options[:quality] = {:extension => ext, :resolution => res}
+      end
+
       opts.on_tail('-h', '--help', 'Display this screen') do
         print_help_and_exit(opts)
       end

data/helper/download-helper.rb

@@ -6,15 +6,15 @@ module ViddlRb
 
     #viddl will use the first of these tools it finds on the system to download the video.
     #if the system does not have any of these tools, net/http is used instead.
-    TOOLS_PRIORITY_LIST = [:wget, :curl] 
-    
+    TOOLS_PRIORITY_LIST = [:wget, :curl]
+
     #simple helper that will save a file from the web and save it with a progress bar
     def self.save_file(file_url, file_name, opts = {})
       trap("SIGINT") { puts "goodbye"; exit }
 
       #default options
-      options = {:save_dir => ".", 
-                 :amount_of_retries => 6, 
+      options = {:save_dir => ".",
+                 :amount_of_retries => 6,
                  :tool => get_tool}
 
       opts[:tool] = options[:tool] if opts[:tool].nil?
@@ -37,15 +37,15 @@ module ViddlRb
           require_progressbar
           puts "Using net/http"
           success = download_and_save_file(file_url, file_path)
-        end  
+        end
         #we were successful, we're outta here
         if success
           break
         else
-          puts "Download seems to have failed (retrying, attempt #{i+1}/#{amount_of_retries})"
+          puts "Download seems to have failed (retrying, attempt #{i+1}/#{options[:amount_of_retries]})"
           sleep 2
         end
-      end    
+      end
       success
     end
 
@@ -73,7 +73,7 @@ module ViddlRb
 
       Net::HTTP.start(uri.host, uri.port) do |http|
         http.request_get(uri.request_uri) do |res|
-          file_size = res.read_header["content-length"].to_i 
+          file_size = res.read_header["content-length"].to_i
           bar = ProgressBar.new(File.basename(full_path), file_size)
           bar.file_transfer_mode
           res.read_body do |segment|

data/plugins/soundcloud.rb

@@ -1,4 +1,4 @@
-require 'rest_client'
+require 'open-uri'
 class Soundcloud < PluginBase
   # this will be called by the main app to check whether this plugin is responsible for the url passed
   def self.matches_provider?(url)
@@ -7,7 +7,7 @@ class Soundcloud < PluginBase
 
   # return the url for original video file and title
   def self.get_urls_and_filenames(url, options = {})
-    doc          = Nokogiri::HTML(RestClient.get(url).body)
+    doc          = Nokogiri::HTML(open(url))
     download_filename = doc.at("#main-content-inner img[class=waveform]").attributes["src"].value.to_s.match(/\.com\/(.+)\_/)[1]
     download_url = "http://media.soundcloud.com/stream/#{download_filename}"
     file_name    = transliterate("#{doc.at('//h1/em').text.chomp}") + ".mp3"

data/plugins/youtube.rb

@@ -1,5 +1,24 @@
+require 'open-uri'
 
 class Youtube < PluginBase
+
+  VIDEO_INFO_URL = "http://www.youtube.com/get_video_info?video_id="
+
+  VIDEO_FORMATS = {
+    "38" => {:extension => "mp4", :name => "MP4 Highest Quality 4096x3027 (H.264, AAC)"},            
+    "37" => {:extension => "mp4", :name => "MP4 Highest Quality 1920x1080 (H.264, AAC)"},
+    "22" => {:extension => "mp4", :name => "MP4 1280x720 (H.264, AAC)"},
+    "45" => {:extension => "webm", :name => "WebM 1280x720 (VP8, Vorbis)"},
+    "44" => {:extension => "webm", :name => "WebM 854x480 (VP8, Vorbis)"},
+    "18" => {:extension => "mp4", :name => "MP4 640x360 (H.264, AAC)"},
+    "35" => {:extension => "flv", :name => "FLV 854x480 (H.264, AAC)"},
+    "34" => {:extension => "flv", :name => "FLV 640x360 (H.264, AAC)"},
+    "5"  => {:extension => "flv", :name => "FLV 400x240 (Soerenson H.263)"},
+    "17" => {:extension => "3gp", :name => "3gp"}    
+  }
+
+  DEFAULT_FORMAT_ORDER = %w[38 37 22 45 44 18 35 34 5 7]
+
   #this will be called by the main app to check whether this plugin is responsible for the url passed
   def self.matches_provider?(url)
     url.include?("youtube.com") || url.include?("youtu.be")
@@ -7,17 +26,17 @@ class Youtube < PluginBase
   
   #get all videos and return their urls in an array
   def self.get_video_urls(feed_url)
-    puts "[YOUTUBE] Retrieving videos..."
+    notify "Retrieving videos..."
     urls_titles = Hash.new
     result_feed = Nokogiri::XML(open(feed_url))
-    urls_titles.merge!(grab_ut(result_feed))
+    urls_titles.merge!(grab_urls_and_titles(result_feed))
 
     #as long as the feed has a next link we follow it and add the resulting video urls
     loop do   
       next_link = result_feed.search("//feed/link[@rel='next']").first
       break if next_link.nil?
       result_feed = Nokogiri::HTML(open(next_link["href"]))
-      urls_titles.merge!(grab_ut(result_feed))
+      urls_titles.merge!(grab_urls_and_titles(result_feed))
     end
 
     self.filter_urls(urls_titles)
@@ -26,7 +45,7 @@ class Youtube < PluginBase
   #returns only the urls that match the --filter argument regex (if present)
   def self.filter_urls(url_hash)
     if @filter
-      puts "[YOUTUBE] Using filter: #{@filter}"
+      notify "Using filter: #{@filter}"
       filtered = url_hash.select { |url, title| title =~ @filter }
       filtered.keys
     else
@@ -35,7 +54,7 @@ class Youtube < PluginBase
   end
 
   #extract all video urls and their titles from a feed and return in a hash
-  def self.grab_ut(feed)
+  def self.grab_urls_and_titles(feed)
     feed.remove_namespaces!  #so that we can get to the titles easily
     urls   = feed.search("//entry/link[@rel='alternate']").map { |link| link["href"] }
     titles = feed.search("//entry/group/title").map { |title| title.text } 
@@ -48,44 +67,48 @@ class Youtube < PluginBase
     #http://www.youtube.com/watch?v=Tk78sr5JMIU&videos=jKY836_WMhE
 
     playlist_ID = url[/(?:list=PL|p=)(\w{16})&?/,1]
-    puts "[YOUTUBE] Playlist ID: #{playlist_ID}"
+    notify "Playlist ID: #{playlist_ID}"
     feed_url = "http://gdata.youtube.com/feeds/api/playlists/#{playlist_ID}?&max-results=50&v=2"
     url_array = self.get_video_urls(feed_url)
-    puts "[YOUTUBE] #{url_array.size} links found!"
+    notify "#{url_array.size} links found!"
     url_array
   end
 
   def self.parse_user(username)
-    puts "[YOUTUBE] User: #{username}"
+    notify "User: #{username}"
     feed_url = "http://gdata.youtube.com/feeds/api/users/#{username}/uploads?&max-results=50&v=2"
     url_array = get_video_urls(feed_url)
-    puts "[YOUTUBE] #{url_array.size} links found!"
+    notify "#{url_array.size} links found!"
     url_array
   end
 
   def self.get_urls_and_filenames(url, options = {})
     @filter = options[:playlist_filter]                                    #used to filter a playlist in self.filter_urls
+    @quality = options[:quality]
+
     return_values = []
+
     if url.include?("view_play_list") || url.include?("playlist?list=")    #if playlist
-      puts "[YOUTUBE] playlist found! analyzing..."
-      files = self.parse_playlist(url)
-      puts "[YOUTUBE] Starting playlist download"
+      notify "playlist found! analyzing..."
+      files = parse_playlist(url)
+      notify "Starting playlist download"
       files.each do |file|
-        puts "[YOUTUBE] Downloading next movie on the playlist (#{file})"
-        return_values << self.grab_single_url_filename(file)
+        notify "Downloading next movie on the playlist (#{file})"
+        return_values << grab_single_url_filename(file)
       end  
     elsif match = url.match(/\/user\/([\w\d]+)$/)                          #if user url, e.g. youtube.com/user/woot
       username = match[1]
-      video_urls = self.parse_user(username)
-      puts "[YOUTUBE] Starting user videos download"
+      video_urls = parse_user(username)
+      notify "Starting user videos download"
       video_urls.each do |url|
-        puts "[YOUTUBE] Downloading next user video (#{url})"
-        return_values << self.grab_single_url_filename(url)
+        notify "Downloading next user video (#{url})"
+        return_values << grab_single_url_filename(url)
       end
     else                                                                   #if single video
-      return_values << self.grab_single_url_filename(url)
+      return_values << grab_single_url_filename(url)
     end 
-    return_values.reject! { |value| value == :no_embed }   #remove results that can not be downloaded
+
+    return_values.reject! { |value| value == :no_embed }                   #remove results that can not be downloaded
 
     if return_values.empty?
       raise CouldNotDownloadVideoError, "No videos could be downloaded - embedding disabled."
@@ -99,18 +122,14 @@ class Youtube < PluginBase
     #addition: might also look like this /v/abc5-a5afe5agae6g
     # alternative:  video_id = url[/v[\/=]([\w-]*)&?/, 1]
     # First get the redirect
-    if url.include?("youtu.be")
-      url = open(url).base_uri.to_s
-    end
+
+    url = open(url).base_uri.to_s if url.include?("youtu.be")
     video_id = url[/(v|embed)[=\/]([^\/\?\&]*)/,2]
-    if video_id.nil?
-      raise CouldNotDownloadVideoError, "No video id found."
-    else
-      puts "[YOUTUBE] ID FOUND: #{video_id}"
-    end
+    video_id ? notify("ID FOUND: #{video_id}") : download_error("No video id found.")
+
     #let's get some infos about the video. data is urlencoded
-    yt_url = "http://www.youtube.com/get_video_info?video_id=#{video_id}"
-    video_info = RestClient.get(yt_url).body
+    video_info = open(VIDEO_INFO_URL + video_id).read
+
     #converting the huge infostring into a hash. simply by splitting it at the & and then splitting it into key and value arround the =
     #[...]blabla=blubb&narf=poit&marc=awesome[...]
     video_info_hash = Hash[*video_info.split("&").collect { |v| 
@@ -150,42 +169,22 @@ class Youtube < PluginBase
       [key, value]
     }.flatten]
     
-    if video_info_hash["status"] == "fail"
-      return :no_embed
-    end
-    
+    return :no_embed if video_info_hash["status"] == "fail"
+      
     title = video_info_hash["title"]
     length_s = video_info_hash["length_seconds"]
     token = video_info_hash["token"]
 
+    notify "Title: #{title}"
+    notify "Length: #{length_s} s"
+    notify "t-parameter: #{token}"
+
     #for the formats, see: http://en.wikipedia.org/wiki/YouTube#Quality_and_codecs
     fmt_list = video_info_hash["fmt_list"].split(",")
-    available_formats = fmt_list.map{|format| format.split("/").first}
-    
-    format_ext = {}
-    format_ext["38"] = {:extension => "mp4", :name => "MP4 Highest Quality 4096x3027 (H.264, AAC)"}            
-    format_ext["37"] = {:extension => "mp4", :name => "MP4 Highest Quality 1920x1080 (H.264, AAC)"}
-    format_ext["22"] = {:extension => "mp4", :name => "MP4 1280x720 (H.264, AAC)"}
-    format_ext["45"] = {:extension => "webm", :name => "WebM 1280x720 (VP8, Vorbis)"}
-    format_ext["44"] = {:extension => "webm", :name => "WebM 854x480 (VP8, Vorbis)"}    
-    format_ext["18"] = {:extension => "mp4", :name => "MP4 640x360 (H.264, AAC)"}
-    format_ext["35"] = {:extension => "flv", :name => "FLV 854x480 (H.264, AAC)"}
-    format_ext["34"] = {:extension => "flv", :name => "FLV 640x360 (H.264, AAC)"}
-    format_ext["5"] = {:extension => "flv", :name => "FLV 400x240 (Soerenson H.263)"}
-    format_ext["17"] = {:extension => "3gp", :name => "3gp"}    
-    
-    #since 1.8 doesn't do ordered hashes
-    prefered_order = ["38","37","22","45","44","18","35","34","5","17"]
-    
-    selected_format = prefered_order.select{|possible_format| available_formats.include?(possible_format)}.first
-    
-    puts "[YOUTUBE] Title: #{title}"
-    puts "[YOUTUBE] Length: #{length_s} s"
-    puts "[YOUTUBE] t-parameter: #{token}"
-    #best quality seems always to be firsts
-    puts "[YOUTUBE] formats available: #{available_formats.inspect} (downloading format #{selected_format} -> #{format_ext[selected_format][:name]})"
 
-    #video_info_hash.keys.sort.each{|key| puts "#{key} : #{video_info_hash[key]}" }
+    selected_format = pick_video_format(fmt_list)
+    puts "(downloading format #{selected_format} -> #{VIDEO_FORMATS[selected_format][:name]})"
+
     download_url = video_info_hash["url_encoded_fmt_stream_map"][selected_format]
 
     #if download url ends with a ';' followed by a codec string remove that part because it stops URI.parse from working
@@ -199,8 +198,40 @@ class Youtube < PluginBase
       download_url.sub!("&sig=", "&signature=")     #else we just have to change sig to signature
     end
 
-    file_name = PluginBase.make_filename_safe(title) + "." + format_ext[selected_format][:extension]
-    puts "downloading to " + file_name
+    file_name = PluginBase.make_filename_safe(title) + "." + VIDEO_FORMATS[selected_format][:extension]
+    puts "downloading to " + file_name + "\n\n"
     {:url => download_url, :name => file_name}
   end
+
+  #returns the format of the video the user picked or the first default format if it does not exist
+  def self.pick_video_format(fmt_list)
+    available_formats = fmt_list.map { |format| format.split("/").first }
+    notify "formats available: #{available_formats.inspect}"
+
+    if @quality                         #if the user specified a format
+      ext = @quality[:extension]
+      res = @quality[:resolution]
+
+      #gets a nested array with all the formats of the same res as the user wanted
+      requested = VIDEO_FORMATS.select { |id, format| format[:name].include?(res) }.to_a
+
+      if requested.empty?
+        notify "Requested format \"#{res}:#{ext}\" not found. Downloading default format."
+        get_default_format(available_formats)
+      else
+        pick = requested.find { |format| format[1][:extension] == ext }             #get requsted extension if possible
+        pick ? pick.first : get_default_format(requested.map { |req| req.first })   #else return the default format
+      end
+    else
+      get_default_format(available_formats)
+    end
+  end
+
+  def self.get_default_format(available)
+    DEFAULT_FORMAT_ORDER.find { |default| available.include?(default) }
+  end
+
+  def self.notify(message)
+    puts "[YOUTUBE] #{message}"
+  end
 end

metadata

@@ -1,12 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: viddl-rb
 version: !ruby/object:Gem::Version 
-  hash: 157
+  hash: 147
   prerelease: 
   segments: 
   - 0
-  - 75
-  version: "0.75"
+  - 76
+  version: "0.76"
 platform: ruby
 authors: 
 - Marc Seeger
@@ -14,8 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-10-16 00:00:00 +02:00
-default_executable: 
+date: 2013-01-30 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: nokogiri
@@ -46,7 +45,7 @@ dependencies:
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: rest-client
+  name: progressbar
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
@@ -60,7 +59,7 @@ dependencies:
   type: :runtime
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
-  name: progressbar
+  name: rake
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
@@ -71,10 +70,10 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  type: :runtime
+  type: :development
   version_requirements: *id004
 - !ruby/object:Gem::Dependency 
-  name: rake
+  name: rest-client
   prerelease: false
   requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
@@ -131,7 +130,6 @@ files:
 - Rakefile
 - README.md
 - TODO.txt
-has_rdoc: false
 homepage: https://github.com/rb2k/viddl-rb
 licenses: []
 
@@ -163,9 +161,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: viddl-rb
-rubygems_version: 1.6.2
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: An extendable commandline video downloader for flash video sites.
 test_files: []
 
+has_rdoc: false