viddl-rb 0.6 → 0.7

data/helper/audio-helper.rb

@@ -0,0 +1,49 @@
+module ViddlRb
+
+  # This class is responsible for extracting audio from video files using ffmpeg.
+  class AudioHelper
+
+    def self.extract(file_path)
+      no_ext_filename = file_path.split('.')[0..-1][0]
+      #capture stderr because ffmpeg expects an output param and will error out
+      puts "Gathering information about the downloaded file."
+      file_info = Open3.popen3("ffmpeg -i #{file_path}") {|stdin, stdout, stderr, wait_thr| stderr.read }
+      puts "Done gathering information about the downloaded file."
+
+      if !file_info.to_s.empty?
+        audio_format_matches = file_info.match(/Audio: (\w*)/)
+        if audio_format_matches
+          audio_format = audio_format_matches[1]
+          puts "detected audio format: #{audio_format}"
+        else
+          raise "ERROR: Couldn't find any audio:\n#{file_info.inspect}"
+        end
+        
+        extension_mapper = {
+        'aac' => 'm4a',
+        'mp3' => 'mp3',
+        'vorbis' => 'ogg'
+        }
+
+        if extension_mapper.key?(audio_format)
+          output_extension = extension_mapper[audio_format]
+        else
+          #lame fallback
+          puts "Unknown audio format: #{audio_format}, using name as extension: '.#{audio_format}'."
+          output_extension = audio_format
+        end
+        output_filename = "#{no_ext_filename}.#{output_extension}"
+        if File.exist?(output_filename)
+          puts "Audio file seems to exist already, removing it before extraction."
+          File.delete(output_filename)
+        end
+        Open3.popen3("ffmpeg -i #{file_path} -vn -acodec copy #{output_filename}") { |stdin, stdout, stderr, wait_thr| stdout.read }
+        puts "Done extracting audio to #{output_filename}"
+      else
+        raise "ERROR: Error while checking audio track of #{file_path}"
+      end
+    end
+  end
+
+end
+

data/helper/download-helper.rb

@@ -1,58 +1,86 @@
-class DownloadHelper
-#usually not called directly
-  def self.fetch_file(uri)
-
-  begin
-  	require "progressbar" #http://github.com/nex3/ruby-progressbar
-  rescue LoadError
-  	puts "ERROR: You don't seem to have curl or wget on your system. In this case you'll need to install the 'progressbar' gem."
-  	exit
-  end
-    progress_bar = nil 
-    open(uri, :proxy => nil,
-      :content_length_proc => lambda { |length|
-        if length && 0 < length
-          progress_bar = ProgressBar.new(uri.to_s, length)
-        end 
-      },
-      :progress_proc => lambda { |progress|
-        progress_bar.set(progress) if progress_bar
-      }) {|file| return file.read}        
-  end
-  
-  #simple helper that will save a file from the web and save it with a progress bar
-  def self.save_file(file_uri, file_name)
-    unescaped_uri = CGI::unescape(file_uri)
-    result = false
-    if os_has?("wget")
-      puts "using wget"
-      result = system("wget \"#{unescaped_uri}\" -O #{file_name}")
-    elsif os_has?("curl")
-      puts "using curl"
-      #-L means: follow redirects, We set an agent because Vimeo seems to want one
-    	result = system("curl -A 'Wget/1.8.1' -L \"#{unescaped_uri}\" -o #{file_name}")
-    else
-    	puts "using net/http"
-      open(file_name, 'wb') { |file|   	      
-        file.write(fetch_file(unescaped_uri)); puts
-      }
-      result = true
-    end         
-    result
-  end
-  
-  #checks to see whether the os has a certain utility like wget or curl
-  def self.os_has?(utility)
-    windows = ENV['OS'] =~ /windows/i
-    unless windows # if os is something else than Windows
-      return `which #{utility}`.include?(utility)
-    else # OS is Windows
-      begin 
-        `#{utility} --version` #if running the command does not throw an error, Windows has it. --version is for prettier console output.
-        return true
-      rescue Errno::ENOENT
-        return false
-      end
-    end
-  end
-end
+module ViddlRb
+
+  class DownloadHelper
+    #usually not called directly
+    def self.fetch_file(uri)
+      begin
+        require "progressbar" #http://github.com/nex3/ruby-progressbar
+      rescue LoadError
+        puts "ERROR: You don't seem to have curl or wget on your system. In this case you'll need to install the 'progressbar' gem."
+        exit
+      end
+      progress_bar = nil 
+      open(uri, :proxy => nil,
+        :content_length_proc => lambda { |length|
+          if length && 0 < length
+            progress_bar = ProgressBar.new(uri.to_s, length)
+            progress_bar.file_transfer_mode   #to show download speed and file size
+          end 
+        },
+        :progress_proc => lambda { |progress|
+          progress_bar.set(progress) if progress_bar
+        }) {|file| return file.read}        
+    end
+    
+    #simple helper that will save a file from the web and save it with a progress bar
+    def self.save_file(file_uri, file_name, save_dir = ".", amount_of_retries = 6)
+      trap("SIGINT") { puts "goodbye"; exit }
+
+      file_path = File.absolute_path(File.join(save_dir, file_name))
+      #Some providers seem to flake out every now end then
+      amount_of_retries.times do |i|
+        if os_has?("wget")
+          puts "using wget"
+          `wget \"#{file_uri}\" -O #{file_path.inspect}`
+        elsif os_has?("curl")
+          puts "using curl"
+          #require "pry"; binding.pry; exit
+          #-L means: follow redirects, We set an agent because Vimeo seems to want one
+          `curl -A 'Wget/1.8.1' --retry 10 --retry-delay 5 --retry-max-time 4  -L \"#{file_uri}\" -o #{file_path.inspect}`
+        else
+         puts "using net/http"
+          open(file_path, 'wb') { |file|          
+            file.write(fetch_file(file_uri)); puts
+          }
+        end  
+        #we were successful, we're outta here
+        if $? == 0
+          break
+        else
+          puts "Download seems to have failed (retrying, attempt #{i+1}/#{amount_of_retries})"
+          sleep 2
+        end
+      end    
+      $? == 0
+    end
+
+    #checks to see whether the os has a certain utility like wget or curl
+    #`` returns the standard output of the process
+    #system returns the exit code of the process
+    def self.os_has?(utility)
+      windows = ENV['OS'] =~ /windows/i
+
+      unless windows # if os is not Windows
+        `which #{utility}`.include?(utility)
+      else
+        if has_where?
+          system("where /q #{utility}")   #/q is the quiet mode flag
+        else
+          begin   #as a fallback we just run the utility itself
+            system(utility)
+          rescue Errno::ENOENT
+            false
+          end
+        end
+      end
+    end
+
+    #checks if Windows has the where utility (Server 2003 and later)
+    #system only return nil if the command is not found
+    def self.has_where?
+      !system("where /q where").nil?
+    end
+  end
+
+end
+

data/helper/plugin-helper.rb

@@ -1,10 +1,60 @@
-class PluginBase
-#some static stuff
-  class << self; attr_reader :registered_plugins end
-    @registered_plugins = []
-
-#if you inherit from this class, the child gets added to the "registered plugins" array
-  def self.inherited(child)
-    PluginBase.registered_plugins << child
-  end
-end
+module ViddlRb
+
+  class PluginBase
+
+    #this exception is raised by the plugins when it was not 
+    #possible to donwload the video for some reason.
+    class CouldNotDownloadVideoError < StandardError; end
+
+    #some static stuff
+    class << self
+      attr_accessor :io
+      attr_reader   :registered_plugins
+    end
+
+    #all calls to #puts, #print and #p from any plugin instance will be redirected to this object 
+    @io = $stdout
+    @registered_plugins = []
+
+    #if you inherit from this class, the child gets added to the "registered plugins" array
+    def self.inherited(child)
+      PluginBase.registered_plugins << child
+    end
+
+    #takes a string a returns a new string that is file name safe
+    #deletes \"' and replaces anything else that is not a digit or letter with _
+    def self.make_filename_safe(string)
+      string.delete("\"'").gsub(/[^\d\w]/, '_')
+    end
+
+    #the following methods redirects the Kernel printing methods (except #p) to the
+    #PluginBase IO object. this is because sometimes we want plugins to
+    #write to something else than $stdout
+
+    def self.puts(*objects)
+      PluginBase.io.puts(*objects)
+      nil
+    end
+
+    def self.print(*objects)
+      PluginBase.io.print(*objects)
+      nil
+    end
+
+    def self.putc(int)
+      PluginBase.io.putc(int)
+      nil
+    end
+
+    def self.printf(string, *objects)
+      if string.is_a?(IO) || string.is_a?(StringIO)
+        super(string, *objects)  # so we don't redirect the printf that prints to a separate IO object
+      else
+        PluginBase.io.printf(string, *objects)
+      end
+      nil
+    end
+  end
+
+end
+

data/helper/utility-helper.rb

@@ -0,0 +1,16 @@
+# This class contains utility methods that are used by both the bin utility and the library.
+
+module ViddlRb
+
+  class UtilityHelper
+    #loads all plugins in the plugin directory.
+    #the plugin classes are dynamically added to the ViddlRb module.
+    def self.load_plugins
+      Dir[File.join(File.dirname(__FILE__), "../plugins/*.rb")].each do |plugin|
+        ViddlRb.class_eval(File.read(plugin))
+      end
+    end
+  end
+
+end
+

data/lib/viddl-rb.rb

@@ -0,0 +1,113 @@
+#!/usr/bin/env ruby
+$LOAD_PATH << File.join(File.dirname(__FILE__), '..', 'helper')
+
+require "rubygems"
+require "net/http"
+require "nokogiri"
+require "mechanize"
+require "cgi"
+require "open-uri"
+require "stringio"
+require "download-helper.rb"
+require "plugin-helper.rb"
+require "utility-helper.rb"
+
+#load all plugins
+ViddlRb::UtilityHelper.load_plugins
+
+module ViddlRb
+  class PluginError < StandardError; end
+  class DownloadError < StandardError; end
+
+  def self.io=(io_object)
+    PluginBase.io = io_object
+  end
+
+  #set the default PluginBase io object to a StringIO instance.
+  #this will suppress any standard output from the plugins.
+  self.io = StringIO.new
+  
+  #returns an array of hashes containing the download url(s) and filenames(s) 
+  #for the specified video url.
+  #if the url does not match any plugin, return nil and if a plugin
+  #throws an error, throw PluginError.
+  #the reason for returning an array is because some urls will give multiple
+  #download urls (for example a Youtube playlist url).
+  def self.get_urls_names(url)
+    plugin = PluginBase.registered_plugins.find { |p| p.matches_provider?(url) }
+
+    if plugin 
+      begin
+        #we'll end up with an array of hashes with they keys :url and :name
+        urls_filenames = plugin.get_urls_and_filenames(url)
+      rescue PluginBase::CouldNotDownloadVideoError => e
+        raise_download_error(e)
+      rescue StandardError => e
+        raise_plugin_error(e, plugin)
+      end
+      follow_all_redirects(urls_filenames)
+    else
+      nil
+    end
+  end
+
+  #returns an array of download urls for the given video url.
+  def self.get_urls(url)
+    urls_filenames = get_urls_names(url)
+    urls_filenames.nil? ? nil : urls_filenames.map { |uf| uf[:url] }
+  end
+
+  #returns an array of filenames for the given video url.
+  def self.get_names(url)
+    urls_filenames = get_urls_names(url)
+    urls_filenames.nil? ? nil : urls_filenames.map { |uf| uf[:name] }
+  end
+
+  #same as get_urls_and_filenames but with the extensions only.
+  def self.get_urls_exts(url)
+    urls_filenames = get_urls_names(url)
+    urls_filenames.map do |uf|
+      ext = File.extname(uf[:name])
+      {:url => uf[:url], :ext => ext}
+    end
+  end
+
+  #<<< helper methods >>>
+
+  #the default error message when a plugin fails in some unexpected way.
+  def self.raise_plugin_error(e, plugin)
+    error = PluginError.new(e.message + " [Plugin: #{plugin.name}]")
+    error.set_backtrace(e.backtrace)
+    raise error
+  end
+  private_class_method :raise_plugin_error
+
+  #the default error message when a plugin fails to download a video for a known reason.
+  def self.raise_download_error(e)
+    error = DownloadError.new(e.message)
+    error.set_backtrace(e.backtrace)
+    raise error
+  end
+  private_class_method :raise_download_error
+
+  #takes a url-filenames array and returns a new array where the
+  #"location" header has been followed all the way to the end for all urls.
+  def self.follow_all_redirects(urls_filenames)
+    urls_filenames.map do |uf|
+      url = uf[:url]
+      final_location = get_final_location(url)
+      {:url => final_location, :name => uf[:name]}
+    end
+  end
+  private_class_method :follow_all_redirects
+
+  #recursively get the final location (after following all redirects) for an url.
+  def self.get_final_location(url)
+    Net::HTTP.get_response(URI(url)) do |res|
+      location = res["location"]
+      return url if location.nil?
+      return get_final_location(location)
+    end
+  end
+  private_class_method :get_final_location
+end

data/plugins/blip.rb

@@ -1,3 +1,4 @@
+
 class Blip < PluginBase
   # this will be called by the main app to check whether this plugin is responsible for the url passed
   def self.matches_provider?(url)
@@ -5,12 +6,12 @@ class Blip < PluginBase
   end
 
   # return the url for original video file and title
-  def self.get_urls_and_filenames(url)
+  def self.get_urls_and_filenames(url, options = {})
     id           = self.to_id(url)
     xml_url      = "http://blip.tv/rss/#{id}"
     doc          = Nokogiri::XML(open(xml_url))
     user         = doc.at("//channel/item/blip:user").inner_text
-    title        = doc.at("//channel/item/title").inner_text.gsub(" ", "_")
+    title        = PluginBase.make_filename_safe(doc.at("//channel/item/title").inner_text)
     download_url = doc.at("//channel/item/media:group/media:content").attributes["url"].value
     extention    = download_url.split(".").last
     file_name    = "#{id}-#{user}-#{title}.#{extention}"

data/plugins/dailymotion.rb

@@ -0,0 +1,44 @@
+
+class Dailymotion < PluginBase
+
+  #the video quality is choosen based on the following priority list:
+  QUALITY_PRIORITY = %w[hd1080 hd720 hq sd ld]
+
+  # this will be called by the main app to check whether this plugin is responsible for the url passed
+  def self.matches_provider?(url)
+    url.include?("dailymotion.com")
+  end
+
+  # return the url for original video file and title
+  def self.get_urls_and_filenames(url, options = {})
+    doc = Nokogiri::HTML(open(url))
+
+    #check to see that the video is hosted on dailymotion.com - if not raise exception
+    unless doc.xpath("//div[@class='dmco_html dmpi_video_partnerplayer']").empty?
+      raise CouldNotDownloadVideoError,
+            "This video is not hosted on dailymotion's own content servers. It can't be downloaded."
+    end
+
+    title     = doc.xpath("//meta[@property='og:title']").attribute("content").value 
+    urls      = get_download_urls(doc)
+    quality   = QUALITY_PRIORITY.find { |q| urls[q] }   #quality is the first quality from the priority list that exists for the video
+    down_url  = urls[quality]
+    extension = down_url[/(\.[\w\d]+)\?/, 1]
+    file_name = PluginBase.make_filename_safe(title) + extension
+
+    [{:url => unescape_url(down_url), :name => file_name}]
+  end
+
+  #returns a hash with the different video qualities mapped to their respective download urls
+  def self.get_download_urls(doc)
+    flashvars = doc.xpath("//div[@class='dmco_html player_box']/script").text   #the flash player script
+    decoded = CGI::unescape(flashvars)
+    url_array = decoded.scan(/(ld|sd|hq|hd720|hd1080)URL":"(.+?)"/).flatten     #group 1 = the quality, group 2 = the url
+    Hash[*url_array]                                                            #hash like this: {"quality" => "url"}
+  end
+
+  #remove backslashes
+  def self.unescape_url(url)
+    url.gsub("\\", "")
+  end
+end