viddl-rb 0.65 → 0.66

data/README.md

@@ -18,16 +18,25 @@ Download a video and extract the audio:
 
 In both cases we'll name the output file according to the video title.
 
-Download all videos on a Youtube playlist:
+__Youtube plugin specifics:__  
+
+Download all videos on a playlist:  
     viddl-rb http://www.youtube.com/playlist?list=PL7E8DA0A515924126
 
-Download all videos from a Youtube user:
+Download all videos from a user:  
     viddl-rb http://www.youtube.com/user/tedtalksdirector
 
+Filter videos to download from a user/playlist:  
+    viddl-rb http://www.youtube.com/user/tedtalksdirector --filter=internet/i
+
+The --filter argument accepts a regular expression and will only download videos where the title matches the regex.
+The /i option does a case-insensitive search.
+
 __Requirements:__
 
 * curl/wget or the [progress bar](http://github.com/nex3/ruby-progressbar/) gem  
 * [Nokogiri](http://nokogiri.org/)
+* [Mechanize](http://mechanize.rubyforge.org/)
 * ffmpeg if you want to extract audio tracks from the videos
 
 

data/bin/viddl-rb

@@ -3,6 +3,7 @@ $LOAD_PATH << File.join(File.dirname(__FILE__), '..', 'helper')
 
 require "rubygems"
 require "nokogiri"
+require "mechanize"
 require "cgi"
 require "open-uri"
 require "open3"

data/plugins/vimeo.rb

@@ -1,25 +1,39 @@
+
 class Vimeo < PluginBase
-	#this will be called by the main app to check whether this plugin is responsible for the url passed
-	def self.matches_provider?(url)
-		url.include?("vimeo.com")
-	end
-	
-	def self.get_urls_and_filenames(url)
-		#the vimeo ID consists of 7 decimal numbers in the URL
-		vimeo_id = url[/\d{7,8}/]
-		doc = Nokogiri::XML(open("http://www.vimeo.com/moogaloop/load/clip:#{vimeo_id}"))
-		title = doc.at("//video/caption").inner_text
-		puts "[VIMEO] Title: #{title}"
-		request_signature = doc.at("//request_signature").inner_text
-		request_signature_expires = doc.at("//request_signature_expires").inner_text
-		
-
-		puts "[VIMEO] Request Signature: #{request_signature} expires: #{request_signature_expires}"
-		
-		download_url = "http://www.vimeo.com/moogaloop/play/clip:#{vimeo_id}/#{request_signature}/#{request_signature_expires}/?q=hd"
-		#todo: put the filename cleaning stuff into a seperate helper
-		file_name = title.delete("\"'").gsub(/[^0-9A-Za-z]/, '_') + ".flv"
-		puts "downloading to " + file_name
-		[{:url => download_url, :name => file_name}]
-	end
-end
+  #this will be called by the main app to check whether this plugin is responsible for the url passed
+  def self.matches_provider?(url)
+    url.include?("vimeo.com")
+  end
+  
+  def self.get_urls_and_filenames(url)
+    #the vimeo ID consists of 7 decimal numbers in the URL
+    vimeo_id = url[/\d{7,8}/]
+
+    agent = Mechanize.new      #use Mechanize for the automatic cookie handeling
+    agent.redirect_ok = false  #don't follow redirects so we do not download the video when we get it's url
+
+    video_page = agent.get("http://vimeo.com/#{vimeo_id}")
+    page_html = video_page.root.inner_html
+
+    title = page_html[/<meta\s+property="og:title"\s+content="(.+?)"/, 1]
+    puts "[VIMEO] Title: #{title}"
+
+    #the timestamp and sig info is in the embedded player javascript in the video page
+    timestamp = page_html[/"timestamp":(\d+),/, 1]
+    signature = page_html[/"signature":"([\d\w]+)",/, 1]
+
+    # The quality and codecs are listed in order of preference in the url. If HD is not availabe SD will be download for example.
+    redirect_url = "http://player.vimeo.com/play_redirect?clip_id=#{vimeo_id}&sig=#{signature}&time=#{timestamp}&quality=hd,sd&codecs=H264,VP8,VP6"
+
+    #the download url is the value of the location (redirect) header
+    download_url = agent.get(redirect_url).header["location"]
+
+    file_name = make_filename(title)
+    
+    [{:url => download_url, :name => file_name}]
+  end
+
+  def self.make_filename(title)
+    title.delete("\"'").gsub(/[^\d\w]/, '_') + ".mp4"
+  end
+end

data/plugins/youtube.rb

@@ -7,23 +7,43 @@ class Youtube < PluginBase
   
   #get all videos and return their urls in an array
   def self.get_video_urls(feed_url)
-    urls = []
+    puts "[YOUTUBE] Retrieving videos..."
+    urls_titles = Hash.new
     result_feed = Nokogiri::HTML(open(feed_url))
-    urls << grab_urls(result_feed) 
+    urls_titles.merge!(grab_ut(result_feed))
 
     #as long as the feed has a next link we follow it and add the resulting video urls
     loop do   
       next_link = result_feed.search("//feed/link[@rel='next']").first
       break if next_link.nil?
       result_feed = Nokogiri::HTML(open(next_link["href"]))
-      urls << grab_urls(result_feed)
+      urls_titles.merge!(grab_ut(result_feed))
+    end
+
+    self.filter_urls(urls_titles)
+  end
+
+  #returns only the urls that match the --filter argument regex (if present)
+  def self.filter_urls(url_hash)
+    #get the --filter arg or "" if it is not present (because nil would break the next line)
+    filter = ARGV.find( proc {""} ) { |arg| arg =~ /--filter=/ }
+    regex = filter[/--filter=(.+?)(?:\/|$)/, 1]
+    if regex
+      puts "[YOUTUBE] Using filter: #{regex}"
+      ignore_case = filter.include?("/i")
+      filtered = url_hash.select { |url, title| title =~ Regexp.new(regex, ignore_case) }
+      filtered.keys
+    else
+      url_hash.keys
     end
-    urls.flatten 
   end
 
-  #extract all video urls form a feed an return in an array
-  def self.grab_urls(feed)
-    feed.search("//entry/link[@rel='alternate']").map { |link| link["href"] }
+  #extract all video urls and their titles from a feed and return in a hash
+  def self.grab_ut(feed)
+    feed.remove_namespaces!  #so that we can get to the titles easily
+    urls   = feed.search("//entry/link[@rel='alternate']").map { |link| link["href"] }
+    titles = feed.search("//entry/group/title").map { |title| title.text } 
+    Hash[urls.zip(titles)]    #hash like this: url => title
   end
 
   def self.parse_playlist(url)

metadata

@@ -1,12 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: viddl-rb
 version: !ruby/object:Gem::Version 
-  hash: 137
+  hash: 143
   prerelease: 
   segments: 
   - 0
-  - 65
-  version: "0.65"
+  - 66
+  version: "0.66"
 platform: ruby
 authors: 
 - Marc Seeger
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-05-02 00:00:00 Z
+date: 2012-06-03 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: nokogiri
@@ -31,7 +31,7 @@ dependencies:
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
-  name: rake
+  name: mechanize
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
@@ -42,10 +42,10 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  type: :development
+  type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: minitest
+  name: rake
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
@@ -59,7 +59,7 @@ dependencies:
   type: :development
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
-  name: rest-client
+  name: minitest
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
@@ -72,6 +72,20 @@ dependencies:
         version: "0"
   type: :development
   version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: rest-client
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id005
 description: An extendable commandline video downloader for flash video sites. Includes plugins for vimeo, youtube and megavideo
 email: mail@marc-seeger.de
 executables: