viddl-rb 0.84 → 0.85

data/Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    viddl-rb (0.82)
+    viddl-rb (0.84)
       mechanize
+      multi_json
       nokogiri (~> 1.5.0)
       progressbar
 
@@ -24,6 +25,7 @@ GEM
       webrobots (>= 0.0.9, < 0.2)
     mime-types (1.24)
     minitest (5.0.6)
+    multi_json (1.8.0)
     net-http-digest_auth (1.4)
     net-http-persistent (2.9)
     nokogiri (1.5.10)

data/README.md

@@ -115,12 +115,12 @@ __Requirements:__
 * [Mechanize](http://mechanize.rubyforge.org/)
 * ffmpeg if you want to extract audio tracks from the videos
 
-__Co Maintainer:__
+__Co Maintainers:__
 * [kl](https://github.com/kl): Windows support (who knew!), bug fixes, veoh plugin, metacafe plugin, refactoring it into a library, ...
+* [farleyknight](https://github.com/farleyknight): Various small fixes and improvements
 
 __Contributors:__
 * [divout](https://github.com/divout) aka Ivan K: blip.tv plugin, bugfixes
 * Sniper: bugfixes
 * [Serabe](https://github.com/Serabe) aka Sergio Arbeo: packaging viddl as a binary
 * [laserlemon](https://github.com/laserlemon): Adding gemnasium images to readme
-* [farleyknight](https://github.com/farleyknight): Various small fixes and improvements

data/lib/viddl-rb.rb

@@ -4,6 +4,7 @@ $LOAD_PATH << File.join(File.dirname(__FILE__), '..', 'helper')
 require "rubygems"
 require "net/http"
 require "nokogiri"
+require "multi_json"
 require "mechanize"
 require "cgi"
 require "open-uri"

data/plugins/arte_plus_seven.rb

@@ -0,0 +1,37 @@
+require 'multi_json'
+
+class ArtePlusSeven < PluginBase
+  # this will be called by the main app to check whether this plugin is responsible for the url passed
+  def self.matches_provider?(url)
+    url.include?("arte.tv")
+  end
+
+  # return the url for original video file and title
+  def self.get_urls_and_filenames(url, options = {})   
+    id           = self.to_id(url)
+    country      = self.extract_country(url)
+    json_url     = "http://arte.tv/papi/tvguide/videos/stream/player/#{country}/#{id}_PLUS7-#{country.upcase}/ALL/ALL.json"
+    doc          = MultiJson.load(open(json_url))['videoJsonPlayer']
+    # This can be improved a lot,
+    # check the results on http://floriancrouzat.net/arte/
+    first_http_key = doc['VSR'].keys.find{|k| k.start_with?('HTTP')}
+    download_url = doc['VSR'][first_http_key]['url']
+    title        = doc['VTI']
+    file_name    = PluginBase.make_filename_safe(title) + ".mp4"
+    [{:url => download_url, :name => file_name}]
+  end
+
+  def self.to_id(url)
+    url[/([\d-]+)/,1]
+  end
+  
+  def self.extract_country(url)
+    url_country = url[/\/guide\/(..)\//,1]
+    mapping = {
+      'de' => 'D',
+      'fr' => 'F'
+    }
+    mapping[url_country]
+    
+  end
+end

data/plugins/youtube.rb

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 
 class Youtube < PluginBase
 
@@ -34,9 +35,9 @@ class Youtube < PluginBase
     return_vals = []
 
     if playlist_urls = parser.get_playlist_urls(url, filter)
-      playlist_urls.each { |url| return_vals << grab_single_url_filename(url) }
+      playlist_urls.each { |url| return_vals << grab_single_url_filename(url, options) }
     else
-      return_vals << grab_single_url_filename(url)
+      return_vals << grab_single_url_filename(url, options)
     end
 
     clean_return_values(return_vals)
@@ -52,128 +53,150 @@ class Youtube < PluginBase
     end
   end
 
-  def self.grab_single_url_filename(url)
-    grab_url_embeddable(url) || grab_url_non_embeddable(url)
+  def self.grab_single_url_filename(url, options)
+    UrlGrabber.new(url, self, options).process
   end
 
-  def self.grab_url_embeddable(url)
-    video_info   = get_video_info(url)
-    video_params = extract_video_parameters(video_info)
-    unless video_params[:embeddable]
-      notify("VIDEO IS NOT EMBEDDABLE")
-      return false
+  class UrlGrabber
+    attr_accessor :url, :options, :plugin, :quality
+
+    def initialize(url, plugin, options)
+      @url     = url
+      @plugin  = plugin
+      @options = options
+      @quality = options[:quality]
     end
 
-    urls_formats    = extract_urls_formats(video_info)
-    selected_format = choose_format(urls_formats)
-    title           = video_params[:title]
-    file_name       = PluginBase.make_filename_safe(title) + "." + VIDEO_FORMATS[selected_format][:extension]
+    def process
+      grab_url_embeddable(url) || grab_url_non_embeddable(url)
+    end
 
-    {:url => urls_formats[selected_format], :name => file_name}
-  end
+    # VEVO video: http://www.youtube.com/watch?v=A_J7kEhY9sM
+    # Non-VEVO video: http://www.youtube.com/watch?v=WkkC9cK8Hz0
 
-  def self.grab_url_non_embeddable(url)
-    video_info = open(url).read
-    stream_map = video_info[/url_encoded_fmt_stream_map\" *: *\"([^\"]+)\"/,1]
-    urls_formats = parse_stream_map(url_decode(stream_map))
-    selected_format = choose_format(urls_formats)
-    title = video_info[/<meta name="title" content="([^"]*)">/, 1]
-    file_name = PluginBase.make_filename_safe(title) + "." + VIDEO_FORMATS[selected_format][:extension]
-
-    # cleaning
-    clean_url = urls_formats[selected_format].gsub(/\\u0026[^&]*/, "").split(',type=video').first
-    {:url => clean_url, :name => file_name}
-  end
+    def grab_url_embeddable(url)
+      video_info   = get_video_info(url)
+      video_params = extract_video_parameters(video_info)
 
-  def self.get_video_info(url)
-    id = extract_video_id(url)
-    request_url = VIDEO_INFO_URL + id + VIDEO_INFO_PARMS
-    open(request_url).read
-  end
+      unless video_params[:embeddable]
+        Youtube.notify("VIDEO IS NOT EMBEDDABLE")
+        return false
+      end
 
-  def self.extract_video_id(url)
-    # the youtube video ID looks like this: [...]v=abc5a5_afe5agae6g&[...], we only want the ID (the \w in the brackets)
-    # addition: might also look like this /v/abc5-a5afe5agae6g
-    # alternative:  video_id = url[/v[\/=]([\w-]*)&?/, 1]
-    url = open(url).base_uri.to_s if url.include?("youtu.be")
-    video_id = url[/(v|embed)[=\/]([^\/\?\&]*)/, 2]
+      urls_formats    = extract_urls_formats(video_info)
+      selected_format = choose_format(urls_formats)
+      title           = video_params[:title]
+      file_name       = PluginBase.make_filename_safe(title) + "." + VIDEO_FORMATS[selected_format][:extension]
 
-    if video_id
-      notify("ID FOUND: #{video_id}")
-      video_id
-    else
-      download_error("No video id found.")
+      {:url => urls_formats[selected_format], :name => file_name}
+    end
+
+    def grab_url_non_embeddable(url)
+      video_info      = open(url).read
+      stream_map      = video_info[/url_encoded_fmt_stream_map\" *: *\"([^\"]+)\"/,1]
+      urls_formats    = parse_stream_map(url_decode(stream_map))
+      selected_format = choose_format(urls_formats)
+      title           = video_info[/<meta name="title" content="([^"]*)">/, 1]
+      file_name       = PluginBase.make_filename_safe(title) + "." + VIDEO_FORMATS[selected_format][:extension]
+
+      # cleaning
+      clean_url = urls_formats[selected_format].gsub(/\\u0026[^&]*/, "").split(',type=video').first
+      {:url => clean_url, :name => file_name}
     end
-  end
 
-  def self.extract_video_parameters(video_info)
-    decoded = url_decode(video_info)
+    def get_video_info(url)
+      id = extract_video_id(url)
+      request_url = VIDEO_INFO_URL + id + VIDEO_INFO_PARMS
+      open(request_url).read
+    end
 
-    {:title      => decoded[/title=(.+?)(?:&|$)/, 1],
-     :length_sec => decoded[/length_seconds=(.+?)(?:&|$)/, 1],
-     :author     => decoded[/author=(.+?)(?:&|$)/, 1],
-     :embeddable => !decoded.include?("status=fail")}
-  end
+    def extract_video_parameters(video_info)
+      video_params = CGI.parse(url_decode(video_info))
 
-  def self.extract_urls_formats(video_info)
-    stream_map = video_info[/url_encoded_fmt_stream_map=(.+?)(?:&|$)/, 1]
-    parse_stream_map(stream_map)
-  end
+      {
+        :title      => video_params["title"].first,
+        :length_sec => video_params["length_seconds"].first,
+        :author     => video_params["author"].first,
+        :embeddable => (video_params["status"].first != "fail")
+      }
+    end
 
-  def self.parse_stream_map(stream_map)
-    urls = extract_download_urls(stream_map)
-    formats_urls = {}
+    def extract_video_id(url)
+      # the youtube video ID looks like this: [...]v=abc5a5_afe5agae6g&[...], we only want the ID (the \w in the brackets)
+      # addition: might also look like this /v/abc5-a5afe5agae6g
+      # alternative:  video_id = url[/v[\/=]([\w-]*)&?/, 1]
+      url = open(url).base_uri.to_s if url.include?("youtu.be")
+      video_id = url[/(v|embed)[=\/]([^\/\?\&]*)/, 2]
 
-    urls.each do |url|
-      format = url[/itag=(\d+)/, 1]
-      formats_urls[format] = url
+      if video_id
+        Youtube.notify("ID FOUND: #{video_id}")
+        video_id
+      else
+        Youtube.download_error("No video id found.")
+      end
     end
 
-    formats_urls
-  end
+    def extract_urls_formats(video_info)
+      stream_map = video_info[/url_encoded_fmt_stream_map=(.+?)(?:&|$)/, 1]
+      parse_stream_map(stream_map)
+    end
 
-  def self.extract_download_urls(stream_map)
-    entries = stream_map.split("%2C")
-    decoded = entries.map { |entry| url_decode(entry) }
+    def choose_format(urls_formats)
+      available_formats = urls_formats.keys
+
+      if @quality                        #if the user specified a format
+        ext = @quality[:extension]
+        res = @quality[:resolution]
+        #gets a nested array with all the formats of the same res as the user wanted
+        requested = VIDEO_FORMATS.select { |id, format| format[:name].include?(res) }.to_a
+
+        if requested.empty?
+          Youtube.notify "Requested format \"#{res}:#{ext}\" not found. Downloading default format."
+          get_default_format(available_formats)
+        else
+          pick = requested.find { |format| format[1][:extension] == ext }             # get requsted extension if possible
+          pick ? pick.first : get_default_format(requested.map { |req| req.first })   # else return the default format
+        end
+      else
+        get_default_format(available_formats)
+      end
+    end
 
-    decoded.map do |entry|
-      url = entry[/url=(.*?itag=.+?)(?:itag=|;|$)/, 1]
-      sig = entry[/sig=(.+?)(?:&|$)/, 1]
+    def parse_stream_map(stream_map)
+      urls = extract_download_urls(stream_map)
+      formats_urls = {}
 
-      url + "&signature=#{sig}"
+      urls.each do |url|
+        format = url[/itag=(\d+)/, 1]
+        formats_urls[format] = url
+      end
+
+      formats_urls
     end
-  end
 
-  def self.choose_format(urls_formats)
-    available_formats = urls_formats.keys
+    def extract_download_urls(stream_map)
+      entries = stream_map.split("%2C")
+      decoded = entries.map { |entry| url_decode(entry) }
 
-    if @quality                        #if the user specified a format
-      ext = @quality[:extension]
-      res = @quality[:resolution]
-      #gets a nested array with all the formats of the same res as the user wanted
-      requested = VIDEO_FORMATS.select { |id, format| format[:name].include?(res) }.to_a
+      decoded.map do |entry|
+        url = entry[/url=(.*?itag=.+?)(?:itag=|;|$)/, 1]
+        sig = entry[/sig=(.+?)(?:&|$)/, 1]
 
-      if requested.empty?
-        notify "Requested format \"#{res}:#{ext}\" not found. Downloading default format."
-        get_default_format(available_formats)
-      else
-        pick = requested.find { |format| format[1][:extension] == ext }             # get requsted extension if possible
-        pick ? pick.first : get_default_format(requested.map { |req| req.first })   # else return the default format
+        url + "&signature=#{sig}"
       end
-    else
-      get_default_format(available_formats)
     end
-  end
 
-  def self.get_default_format(available)
-    DEFAULT_FORMAT_ORDER.find { |default| available.include?(default) }
-  end
+    def get_default_format(available)
+      DEFAULT_FORMAT_ORDER.find { |default| available.include?(default) }
+    end
 
-  def self.url_decode(text)
-    while text != (decoded = CGI::unescape(text)) do
-      text = decoded
+    def url_decode(text)
+      while text != (decoded = CGI::unescape(text)) do
+        text = decoded
+      end
+      text
     end
-    text
+
   end
 
   def self.notify(message)
@@ -211,24 +234,24 @@ class Youtube < PluginBase
       #http://www.youtube.com/watch?v=Tk78sr5JMIU&videos=jKY836_WMhE
 
       playlist_ID = url[/(?:list=PL|p=)(.+?)(?:&|\/|$)/, 1]
-      notify "Playlist ID: #{playlist_ID}"
+      Youtube.notify "Playlist ID: #{playlist_ID}"
       feed_url = PLAYLIST_FEED % playlist_ID
       url_array = get_video_urls(feed_url)
-      notify "#{url_array.size} links found!"
+      Youtube.notify "#{url_array.size} links found!"
       url_array
     end
 
     def parse_user(username)
-      notify "User: #{username}"
+      Youtube.notify "User: #{username}"
       feed_url = USER_FEED % username
       url_array = get_video_urls(feed_url)
-      notify "#{url_array.size} links found!"
+      Youtube.notify "#{url_array.size} links found!"
       url_array
     end
 
     #get all videos and return their urls in an array
     def get_video_urls(feed_url)
-      notify "Retrieving videos..."
+      Youtube.notify "Retrieving videos..."
       urls_titles = {}
       result_feed = Nokogiri::XML(open(feed_url))
       urls_titles.merge!(grab_urls_and_titles(result_feed))
@@ -255,16 +278,12 @@ class Youtube < PluginBase
     #returns only the urls that match the --filter argument regex (if present)
     def filter_urls(url_hash)
       if @filter
-        notify "Using filter: #{@filter}"
+        Youtube.notify "Using filter: #{@filter}"
         filtered = url_hash.select { |url, title| title =~ @filter }
         filtered.keys
       else
         url_hash.keys
       end
     end
-
-    def notify(message)
-      Youtube.notify(message)
-    end
   end
 end

metadata

@@ -1,12 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: viddl-rb
 version: !ruby/object:Gem::Version 
-  hash: 163
+  hash: 161
   prerelease: 
   segments: 
   - 0
-  - 84
-  version: "0.84"
+  - 85
+  version: "0.85"
 platform: ruby
 authors: 
 - Marc Seeger
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2013-09-02 00:00:00 Z
+date: 2013-10-05 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: nokogiri
@@ -61,7 +61,7 @@ dependencies:
   type: :runtime
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
-  name: rake
+  name: multi_json
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
@@ -72,10 +72,10 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  type: :development
+  type: :runtime
   version_requirements: *id004
 - !ruby/object:Gem::Dependency 
-  name: rest-client
+  name: rake
   prerelease: false
   requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
@@ -89,7 +89,7 @@ dependencies:
   type: :development
   version_requirements: *id005
 - !ruby/object:Gem::Dependency 
-  name: minitest
+  name: rest-client
   prerelease: false
   requirement: &id006 !ruby/object:Gem::Requirement 
     none: false
@@ -102,6 +102,20 @@ dependencies:
         version: "0"
   type: :development
   version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: minitest
+  prerelease: false
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id007
 description: An extendable commandline video downloader for flash video sites. Includes plugins for vimeo, youtube, dailymotion and more
 email: mail@marc-seeger.de
 executables: 
@@ -120,6 +134,7 @@ files:
 - helper/download-helper.rb
 - helper/plugin-helper.rb
 - helper/utility-helper.rb
+- plugins/arte_plus_seven.rb
 - plugins/blip.rb
 - plugins/dailymotion.rb
 - plugins/metacafe.rb
@@ -131,7 +146,6 @@ files:
 - Gemfile.lock
 - Rakefile
 - README.md
-- TODO.txt
 homepage: https://github.com/rb2k/viddl-rb
 licenses: []
 

data/TODO.txt

@@ -1,3 +0,0 @@
-* wrap all classes used by the lib in a module (for namespace reasons)
-* add save_file method to library
-