viddl-rb 0.7 → 0.8

data/Gemfile.lock

@@ -1,41 +1,43 @@
 PATH
   remote: .
   specs:
-    viddl-rb (0.68)
-      jruby-openssl
+    viddl-rb (0.79)
       mechanize
-      nokogiri
+      nokogiri (~> 1.5.0)
+      progressbar
 
 GEM
   remote: http://rubygems.org/
   specs:
-    bouncy-castle-java (1.5.0146.1)
-    domain_name (0.5.3)
-      unf (~> 0.0.3)
-    jruby-openssl (0.7.7)
-      bouncy-castle-java (>= 1.5.0146.1)
-    mechanize (2.5.1)
+    domain_name (0.5.12)
+      unf (>= 0.0.5, < 1.0.0)
+    http-cookie (1.0.1)
+      domain_name (~> 0.5)
+    mechanize (2.7.1)
       domain_name (~> 0.5, >= 0.5.1)
+      http-cookie (~> 1.0.0)
       mime-types (~> 1.17, >= 1.17.2)
       net-http-digest_auth (~> 1.1, >= 1.1.1)
       net-http-persistent (~> 2.5, >= 2.5.2)
       nokogiri (~> 1.4)
       ntlm-http (~> 0.1, >= 0.1.1)
-      webrobots (~> 0.0, >= 0.0.9)
-    mime-types (1.19)
-    minitest (3.3.0)
-    net-http-digest_auth (1.2.1)
-    net-http-persistent (2.7)
-    nokogiri (1.5.5-java)
+      webrobots (>= 0.0.9, < 0.2)
+    mime-types (1.23)
+    minitest (5.0.4)
+    net-http-digest_auth (1.3)
+    net-http-persistent (2.8)
+    nokogiri (1.5.10)
     ntlm-http (0.1.1)
-    rake (0.9.2.2)
+    progressbar (0.20.0)
+    rake (10.0.4)
     rest-client (1.6.7)
       mime-types (>= 1.16)
-    unf (0.0.5-java)
-    webrobots (0.0.13)
+    unf (0.1.1)
+      unf_ext
+    unf_ext (0.0.6)
+    webrobots (0.1.1)
 
 PLATFORMS
-  java
   ruby
 
 DEPENDENCIES

data/README.md

@@ -1,41 +1,53 @@
-__viddl-rb:__  
+__viddl-rb:__ 
 Initially created by Marc Seeger (@rb2k)  
 Repo: http://github.com/rb2k/viddl-rb  
-[![Build Status](https://secure.travis-ci.org/rb2k/viddl-rb.png)](http://travis-ci.org/rb2k/viddl-rb) [![Dependency Status](https://gemnasium.com/rb2k/viddl-rb.png)](https://gemnasium.com/rb2k/viddl-rb)
+[![Gem Version](https://badge.fury.io/rb/viddl-rb.png)](http://badge.fury.io/rb/viddl-rb)[![Build Status](https://secure.travis-ci.org/rb2k/viddl-rb.png)](http://travis-ci.org/rb2k/viddl-rb) [![Dependency Status](https://gemnasium.com/rb2k/viddl-rb.png)](https://gemnasium.com/rb2k/viddl-rb)
 
 __Installation:__
 
 gem install viddl-rb
 
-__Usage:__  
+__Usage:__
 
 Download a video:  
-    viddl-rb http://www.youtube.com/watch?v=QH2-TGUlwu4
+    ```viddl-rb http://www.youtube.com/watch?v=QH2-TGUlwu4```
+
+Viddl-rb supports the following command line options:
+```
+-e, --extract-audio              Save video audio to file
+-u, --url-only                   Prints url without downloading
+-t, --title-only                 Prints title without downloading
+-f, --filter REGEX               Filters a video playlist according to the regex (Youtube only right now)
+-s, --save-dir DIRECTORY         Specifies the directory where videos should be saved
+-d, --downloader TOOL            Specifies the tool to download with. Supports 'wget', 'curl' and 'net-http'
+-q, --quality QUALITY            Specifies the video format and resolution in the following way => resolution:extension (e.g. 720:mp4). Currently only supported by the Youtube plugin.
+-h, --help                       Displays the help screen
+```
 
 Download a video and extract the audio:  
-    viddl-rb http://www.youtube.com/watch?v=QH2-TGUlwu4 --extract-audio 
+    ```viddl-rb http://www.youtube.com/watch?v=QH2-TGUlwu4 --extract-audio```
 
 In both cases we'll name the output file according to the video title.
 
 Setting the video save directory:  
-    viddl-rb http://vimeo.com/38372260 --save-dir=C:/myvideos
+    ```viddl-rb http://vimeo.com/38372260 --save-dir C:/myvideos```
 
 The --save-dir option works with both absolute and relative paths (relative based on the directory viddl-rb is run from).
-If you want to save to a folder with spaces in it, you have to quote the path like this: --save-dir="C:/my videos"
+If you want to save to a folder with spaces in it, you have to quote the path like this: --save-dir "C:/my videos"
 
 __Youtube plugin specifics:__  
 
 Download all videos on a playlist:  
-    viddl-rb http://www.youtube.com/playlist?list=PL7E8DA0A515924126
+    ```viddl-rb http://www.youtube.com/playlist?list=PL7E8DA0A515924126```
 
 Download all videos from a user:  
-    viddl-rb http://www.youtube.com/user/tedtalksdirector
+    ```viddl-rb http://www.youtube.com/user/tedtalksdirector```
 
 Filter videos to download from a user/playlist:  
-    viddl-rb http://www.youtube.com/user/tedtalksdirector --filter=internet/i
+    ```viddl-rb http://www.youtube.com/user/tedtalksdirector --filter /internet/i```
 
 The --filter argument accepts a regular expression and will only download videos where the title matches the regex.
-The /i option does a case-insensitive search.
+It uses the same syntax as Ruby regular expression literals do.
 
 __Library Usage:__
 

data/Rakefile

@@ -6,7 +6,7 @@ task :default => [:test]
 
 Rake::TestTask.new(:test) do |t|
   #t.pattern = "spec/*_spec.rb"
-  t.test_files = ["spec/lib_spec.rb", "spec/url_extraction_spec.rb"]
+  t.test_files = ["spec/lib_spec.rb", "spec/url_extraction_spec.rb",  "spec/integration_spec.rb"]
 end
 
 Rake::TestTask.new(:test_lib) do |t|
@@ -16,3 +16,7 @@ end
 Rake::TestTask.new(:test_extract) do |t|
   t.test_files = FileList["spec/url_extraction_spec.rb"]
 end
+
+Rake::TestTask.new(:test_integration) do |t|
+  t.test_files = FileList["spec/integration_spec.rb"]
+end

data/bin/helper/downloader.rb

@@ -8,18 +8,13 @@ class Downloader
       url = url_name[:url]
       name = url_name[:name]
 
-      result = save_file(url, name, params[:save_dir])
+      result = ViddlRb::DownloadHelper.save_file(url, name, :save_dir => params[:save_dir], :tool => params[:tool])
       unless result
         raise DownloadFailedError, "Download for #{name} failed."
       else
         puts "Download for #{name} successful."
-        AudioHelper.extract(name) if params[:extract_audio]
+        ViddlRb::AudioHelper.extract(name, params[:save_dir]) if params[:extract_audio]
       end
     end
   end
-
-  # TODO save_dir is not used yet
-  def save_file(url, name, save_dir)
-    ViddlRb::DownloadHelper.save_file(url, name, save_dir)
-  end
 end

data/bin/helper/parameter-parser.rb

@@ -1,67 +1,109 @@
 
 # ParameterParser parses the program parameters.
 # If the parameters are not valid in some way an exception is raised.
+# The exceptions raised by this class are handeled in the bin program.
+
 class ParameterParser
 
   DEFAULT_SAVE_DIR = "."
 
   #returns a hash with the parameters in it:
-  # :url            => the video url
-  # :extract_audio  => should attempt to extract audio? (true/false)
-  # :url_only       => do not download, only print the urls to stdout
-  # :title_only     => do not download, only print the titles to stdout
-  # :youtube_filer  => a regular expression used ot fileter youtube playlists
-  # :save_dir       => the directory where the videos are saved
-  def self.parse_app_parameters
-    check_valid_parameters!
-
-    params = {}
-    params[:url] = ARGV.first
-    params[:extract_audio] = ARGV.include?("--extract-audio")
-    params[:url_only] = ARGV.include?("--url-only")
-    params[:title_only] = ARGV.include?("--title-only")
-    params[:playlist_filter] = get_youtube_filter
-    params[:save_dir] = get_save_dir  
-    params
-  end
+  # :url              => the video url
+  # :extract_audio    => should attempt to extract audio? (true/false)
+  # :url_only         => do not download, only print the urls to stdout
+  # :title_only       => do not download, only print the titles to stdout
+  # :playlist_filter  => a regular expression used to filter playlists
+  # :save_dir         => the directory where the videos are saved
+  # :tool             => the download tool (wget, curl, net/http) to use
+  # :quality          => the resolution and format to download
+  def self.parse_app_parameters(args)
 
-  #check if parameters are valid.
-  #the exceptions raised by this method are caught by the viddl-rb bin utility.
-  def self.check_valid_parameters!
-    if ARGV.empty?
-      raise "Usage: viddl-rb URL [--extract-audio]"
-    elsif !ARGV.first.match(/^http/)
-      raise "ERROR: Please include 'http' with your URL e.g. http://www.youtube.com/watch?v=QH2-TGUlwu4"
-    elsif ARGV.include?("--extract-audio") && !DownloadHelper.os_has?("ffmpeg")
-      raise "ERROR: To extract audio you need to have ffmpeg on your system"
-    end
-  end
+    # Default option values are set here
+    options = {
+      :extract_audio    => false,
+      :url_only         => false,
+      :title_only       => false,
+      :playlist_filter  => nil,
+      :save_dir         => DEFAULT_SAVE_DIR,
+      :tool             => nil,
+      :quality          => nil
+    }
+
+    optparse = OptionParser.new do |opts|
+      opts.banner = "Usage: viddl-rb URL [options]"
+
+      opts.on("-e", "--extract-audio", "Save video audio to file") do
+        if ViddlRb::UtilityHelper.os_has?("ffmpeg")
+          options[:extract_audio] = true
+        else
+          raise OptionParser::ParseError.new("to extract audio you need to have ffmpeg on your PATH")
+        end
+      end
+
+      opts.on("-u", "--url-only", "Prints url without downloading") do
+        options[:url_only] = true
+      end
+
+      opts.on("-t", "--title-only", "Prints title without downloading") do
+        options[:title_only] = true
+      end
 
-  #gets the regular expression used to filter youtube playlists.
-  def self.get_youtube_filter
-    filter = ARGV.find { |arg| arg =~ /--filter=./ }  # --filter= and at least one more character
-    return nil unless filter
+      opts.on("-f", "--filter REGEX", Regexp, "Filters a video playlist according to the regex") do |regex|
+        options[:filter] = regex
+      end
 
-    ignore_case = filter.include?("/i")
-    regex = filter[/--filter=(.*?)(?:\/|$)/, 1]   # everything up to the first / (could be an empty string)
-    raise "ERROR: '#{regex}' is not a valid regular expression" unless is_valid_regex?(regex)
-    Regexp.new(regex, ignore_case)
+      opts.on("-s", "--save-dir DIRECTORY", "Specifies the directory where videos should be saved") do |dir|
+        if File.directory?(dir)
+          options[:save_dir] = dir
+        else
+          raise OptionParser::InvalidArgument.new("'#{dir}' is not a valid directory")
+        end
+      end
+
+      opts.on("-d", "--downloader TOOL", "Specifies the tool to download with. Supports 'wget', 'curl' and 'net-http'") do |tool|
+        if tool =~ /(^wget$)|(^curl$)|(^net-http$)/
+          options[:tool] = tool
+        else
+          raise OptionParser::InvalidArgument.new("'#{tool}' is not a valid tool.")
+        end
+      end
+
+      opts.on("-q", "--quality QUALITY", 
+          "Specifies the video format and resolution in the following way => resolution:extension (e.g. 720:mp4)") do |quality|
+        if match = quality.match(/(\d+):(.*)/)
+          res = match[1]
+          ext = match[2]
+        elsif match = quality.match(/\d+/)
+          res = match[0]
+          ext = nil
+        else
+          raise OptionParse.InvalidArgument.new("#{quality} is not a valid argument.")
+        end
+        options[:quality] = {:extension => ext, :resolution => res}
+      end
+
+      opts.on_tail('-h', '--help', 'Display this screen') do
+        print_help_and_exit(opts)
+      end
+    end
+
+    optparse.parse!(args)                           # removes all options from args
+    print_help_and_exit(optparse) if args.empty?    # exit if no video url
+    url = args.first                                # the url is the only element left
+    validate_url!(url)                              # raise exception if invalid url
+    options[:url] = url                     
+    options
   end
 
-  #checks if the string is a valid regex (for example "*****" is not)
-  def self.is_valid_regex?(regex)
-    Regexp.compile(regex)
-  rescue RegexpError
-    false
+  def self.print_help_and_exit(opts)
+    puts opts
+    exit(0)
   end
-  
-  #gets the directory used for saving videos in.
-  def self.get_save_dir
-    save_dir = ARGV.find { |arg| arg =~ /--save-dir=./ }
-    return DEFAULT_SAVE_DIR unless save_dir
-
-    dir = save_dir[/--save-dir=(.+)/, 1]
-    raise "ERROR: '#{dir}' is not a valid directory or does not exist" unless File.directory?(dir)
-    dir
+
+  def self.validate_url!(url)
+    unless url =~ /^http/
+      raise OptionParser::InvalidArgument.new(
+        "please include 'http' with your URL e.g. http://www.youtube.com/watch?v=QH2-TGUlwu4")
+    end
   end
 end

data/bin/viddl-rb

@@ -8,6 +8,7 @@ require "mechanize"
 require "cgi"
 require "open-uri"
 require "open3"
+require "optparse"
 
 require "driver.rb"
 require "downloader.rb"
@@ -20,11 +21,11 @@ require "utility-helper.rb"
 begin
   # params is a hash with keys for each of the parameters passed in.
   # see helper/parameter-parser.rb for what those keys are.
-  params = ParameterParser.parse_app_parameters
+  params = ParameterParser.parse_app_parameters(ARGV)
 
   puts "Loading Plugins"
   ViddlRb::UtilityHelper.load_plugins
-  "Plugins loaded: #{ViddlRb::PluginBase.registered_plugins.inspect}"
+  puts "Plugins loaded: #{ViddlRb::PluginBase.registered_plugins.inspect}"
 
   puts "Will try to extract audio: #{params[:extract_audio] == true}."
   puts "Analyzing URL: #{params[:url]}"
@@ -32,8 +33,13 @@ begin
   app = Driver.new(params)   
   app.start                  # starts the download process
 
+rescue OptionParser::ParseError, ViddlRb::RequirementError => e
+  puts "Error: #{e.message}"
+  exit(1)
+
 rescue StandardError => e
-  puts "#{e.message}"
+  puts "Error: #{e.message}"
+  puts "\nBacktrace:"
+  puts e.backtrace
   exit(1)
 end
-

data/helper/audio-helper.rb

@@ -3,7 +3,7 @@ module ViddlRb
   # This class is responsible for extracting audio from video files using ffmpeg.
   class AudioHelper
 
-    def self.extract(file_path)
+    def self.extract(file_path, save_dir)
       no_ext_filename = file_path.split('.')[0..-1][0]
       #capture stderr because ffmpeg expects an output param and will error out
       puts "Gathering information about the downloaded file."
@@ -32,7 +32,7 @@ module ViddlRb
           puts "Unknown audio format: #{audio_format}, using name as extension: '.#{audio_format}'."
           output_extension = audio_format
         end
-        output_filename = "#{no_ext_filename}.#{output_extension}"
+        output_filename = File.join(save_dir, "#{no_ext_filename}.#{output_extension}")
         if File.exist?(output_filename)
           puts "Audio file seems to exist already, removing it before extraction."
           File.delete(output_filename)
@@ -46,4 +46,3 @@ module ViddlRb
   end
 
 end
-

data/helper/download-helper.rb

@@ -1,86 +1,94 @@
 module ViddlRb
 
+  class RequirementError < StandardError; end
+
   class DownloadHelper
-    #usually not called directly
-    def self.fetch_file(uri)
-      begin
-        require "progressbar" #http://github.com/nex3/ruby-progressbar
-      rescue LoadError
-        puts "ERROR: You don't seem to have curl or wget on your system. In this case you'll need to install the 'progressbar' gem."
-        exit
-      end
-      progress_bar = nil 
-      open(uri, :proxy => nil,
-        :content_length_proc => lambda { |length|
-          if length && 0 < length
-            progress_bar = ProgressBar.new(uri.to_s, length)
-            progress_bar.file_transfer_mode   #to show download speed and file size
-          end 
-        },
-        :progress_proc => lambda { |progress|
-          progress_bar.set(progress) if progress_bar
-        }) {|file| return file.read}        
-    end
-    
+
+    #viddl will use the first of these tools it finds on the system to download the video.
+    #if the system does not have any of these tools, net/http is used instead.
+    TOOLS_PRIORITY_LIST = [:wget, :curl]
+
     #simple helper that will save a file from the web and save it with a progress bar
-    def self.save_file(file_uri, file_name, save_dir = ".", amount_of_retries = 6)
+    def self.save_file(file_url, file_name, opts = {})
       trap("SIGINT") { puts "goodbye"; exit }
 
-      file_path = File.absolute_path(File.join(save_dir, file_name))
+      #default options
+      options = {:save_dir => ".",
+                 :amount_of_retries => 6,
+                 :tool => get_tool}
+
+      opts[:tool] = options[:tool] if opts[:tool].nil?
+      options.merge!(opts)
+
+      file_path = File.expand_path(File.join(options[:save_dir], file_name))
+      success = false
+
       #Some providers seem to flake out every now end then
-      amount_of_retries.times do |i|
-        if os_has?("wget")
-          puts "using wget"
-          `wget \"#{file_uri}\" -O #{file_path.inspect}`
-        elsif os_has?("curl")
-          puts "using curl"
-          #require "pry"; binding.pry; exit
+      options[:amount_of_retries].times do |i|
+        case options[:tool].to_sym
+        when :wget
+          puts "Using wget"
+          success = system "wget \"#{file_url}\" -O #{file_path.inspect}"
+        when :curl
+          puts "Using curl"
           #-L means: follow redirects, We set an agent because Vimeo seems to want one
-          `curl -A 'Wget/1.8.1' --retry 10 --retry-delay 5 --retry-max-time 4  -L \"#{file_uri}\" -o #{file_path.inspect}`
+          success = system "curl -A 'Wget/1.8.1' --retry 10 --retry-delay 5 --retry-max-time 4  -L \"#{file_url}\" -o #{file_path.inspect}"
         else
-         puts "using net/http"
-          open(file_path, 'wb') { |file|          
-            file.write(fetch_file(file_uri)); puts
-          }
-        end  
+          require_progressbar
+          puts "Using net/http"
+          success = download_and_save_file(file_url, file_path)
+        end
         #we were successful, we're outta here
-        if $? == 0
+        if success
           break
         else
-          puts "Download seems to have failed (retrying, attempt #{i+1}/#{amount_of_retries})"
+          puts "Download seems to have failed (retrying, attempt #{i+1}/#{options[:amount_of_retries]})"
           sleep 2
         end
-      end    
-      $? == 0
+      end
+      success
     end
 
-    #checks to see whether the os has a certain utility like wget or curl
-    #`` returns the standard output of the process
-    #system returns the exit code of the process
-    def self.os_has?(utility)
-      windows = ENV['OS'] =~ /windows/i
+    def self.get_tool
+      tool = TOOLS_PRIORITY_LIST.find { |tool| ViddlRb::UtilityHelper.os_has?(tool) }
+      tool || :net_http
+    end
 
-      unless windows # if os is not Windows
-        `which #{utility}`.include?(utility)
-      else
-        if has_where?
-          system("where /q #{utility}")   #/q is the quiet mode flag
-        else
-          begin   #as a fallback we just run the utility itself
-            system(utility)
-          rescue Errno::ENOENT
-            false
+    def self.require_progressbar
+      begin
+        require "progressbar"
+      rescue LoadError
+        raise RequirementError,
+          "you don't seem to have curl or wget on your system. In this case you'll need to install the 'progressbar' gem."
+      end
+    end
+
+    # downloads and saves a file using the net/http streaming api
+    # return true if the download was successful, else returns false
+    def self.download_and_save_file(download_url, full_path)
+      final_url = UtilityHelper.get_final_location(download_url)    # follow all redirects
+      uri = URI(final_url)
+      file = File.new(full_path, "wb")
+      file_size = 0
+
+      Net::HTTP.start(uri.host, uri.port) do |http|
+        http.request_get(uri.request_uri) do |res|
+          file_size = res.read_header["content-length"].to_i
+          bar = ProgressBar.new(File.basename(full_path), file_size)
+          bar.file_transfer_mode
+          res.read_body do |segment|
+            bar.inc(segment.size)
+            file.write(segment)
           end
         end
       end
+      file.close
+      print "\n"
+      download_successful?(full_path, file_size)   #because Net::HTTP.start does not throw Net exceptions
     end
 
-    #checks if Windows has the where utility (Server 2003 and later)
-    #system only return nil if the command is not found
-    def self.has_where?
-      !system("where /q where").nil?
+    def self.download_successful?(full_file_path, file_size)
+      File.exist?(full_file_path) && File.size(full_file_path) == file_size
     end
   end
-
 end
-