viddl-rb 0.99 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 --- 
 SHA1: 
-  metadata.gz: 869f88d6332b09dea9e9c01d3205a1965b9c8c48
-  data.tar.gz: f208de4df1e7fb57d66e2e005aee7f1dbda147f6
+  metadata.gz: aff24988a745ce18c4928aca3c4e50fb47232f1d
+  data.tar.gz: c0fff623ca863b24999582c94d582a2de1288c1f
 SHA512: 
-  metadata.gz: c656fa95771d40f473c698a46fa6395129b6ad5100794479a245d1ade7c51d7d0131077baae825ecf120925abc67c62a24a3924d0470b91b26e5b92d8b165784
-  data.tar.gz: 01be0e2e4510fc211f8661999e5d34f2e74c4582a57ce05619babb2f80b9c48c34a448c669e5f873db5ef250e1f27789932e658e3400ac4fd03019c32dd4c404
+  metadata.gz: 04af8ad42b49b38d890fb2e1536a2e8de0573c6a419c984bb3135078c18cc98a1f3fd9c760d885c540372ebdbf303d164147f240e3ca760edf0161bac16042b7
+  data.tar.gz: 9a7b77633e663298a5ea5872afef78491de249906ff598d73401c15e9f2db4ba99635118e83e74fcebb61ca65da30d59b0955b03846595fd9720c4b70d357b63

data/Rakefile

@@ -2,12 +2,13 @@ require 'rubygems'
 require 'bundler/setup'
 require 'rake/testtask'
 
-ALL_INTEGRATION = FileList["spec/integration/*.rb", "spec/integration/*/*.rb"]
-ALL_UNIT        = FileList["spec/unit/*/*.rb"]
+SKIPPED_INTEGRATION = FileList["spec/integration/youtube/cipher_guesser_spec.rb"]
+ALL_INTEGRATION     = FileList["spec/integration/*.rb", "spec/integration/*/*.rb"] - SKIPPED_INTEGRATION
+ALL_UNIT            = FileList["spec/unit/*.rb", "spec/unit/*/*.rb"]
 
-task :default => [:all]
+task :default => [:test_all]
 
-Rake::TestTask.new(:all) do |t|
+Rake::TestTask.new(:test_all) do |t|
   t.test_files =  ALL_INTEGRATION + ALL_UNIT
 end
 
@@ -34,3 +35,11 @@ end
 Rake::TestTask.new(:test_cipher_loader) do |t|
   t.test_files = FileList["spec/integration/youtube/cipher_loader_spec.rb"]
 end
+
+Rake::TestTask.new(:test_cipher_guesser) do |t|
+  t.test_files = FileList["spec/unit/youtube/cipher_guesser_spec.rb"]
+end
+
+Rake::TestTask.new(:test_decipherer) do |t|
+  t.test_files = FileList["spec/unit/youtube/decipherer_spec.rb"]
+end

data/bin/helper/downloader.rb

@@ -15,15 +15,17 @@ class Downloader
                                                  name,
                                                  :save_dir => params[:save_dir],
                                                  :tool => params[:tool] && params[:tool].to_sym
-      unless result
+      if result
+        puts "Download for #{name} successful."
+        url_name[:on_downloaded].call(true) if url_name[:on_downloaded]
+        ViddlRb::AudioHelper.extract(name, params[:save_dir]) if params[:extract_audio]
+      else
+        url_name[:on_downloaded].call(false) if url_name[:on_downloaded]
         if params[:abort_on_failure]
           raise DownloadFailedError, "Download for #{name} failed."
         else
           puts "Download for #{name} failed. Moving onto next file."
         end
-      else
-        puts "Download for #{name} successful."
-        ViddlRb::AudioHelper.extract(name, params[:save_dir]) if params[:extract_audio]
       end
     end
   end

data/bin/viddl-rb

@@ -28,14 +28,14 @@ begin
   puts "Will try to extract audio: #{params[:extract_audio] == true}."
   puts "Analyzing URL: #{params[:url]}"
 
-  app = Driver.new(params)   
-  app.start                  # starts the download process
+  driver = Driver.new(params)   
+  driver.start                  # starts the download process
 
 rescue OptionParser::ParseError, ViddlRb::RequirementError => e
   puts "Error: #{e.message}"
   exit(1)
 
-rescue StandardError => e
+rescue => e
   puts "Error: #{e.message}"
   puts "\nBacktrace:"
   puts e.backtrace

data/helper/utility-helper.rb

@@ -52,27 +52,28 @@ module ViddlRb
       File.join(File.dirname(File.expand_path(__FILE__)), "..")
     end
 
-    #checks to see whether the os has a certain utility like wget or curl
-    #`` returns the standard output of the process
-    #system returns the exit code of the process
+    # checks to see whether the os has a certain utility like wget or curl
+    # `` returns the standard output of the process
+    # system returns the exit code of the process
     def self.os_has?(utility)
-      
-      unless windows?
-        `which #{utility}`.include?(utility.to_s)
-      else
+      if windows?
         if !system("where /q where").nil?   #if Windows has the where utility
           system("where /q #{utility}")     #/q is the quiet mode flag
         else
-          begin                             #as a fallback we just run the utility itself
+          begin #as a fallback we just run the utility itself
             system(utility)
           rescue Errno::ENOENT
             false
           end
         end
+      else
+        # This might work in windows too... I am not quite sure :-/
+        ENV['PATH'].split(':').any?{|dir| File.exist?( File.join(dir, utility.to_s) ) }
       end
     end
 
-    #recursively get the final location (after following all redirects) for an url.
+    # recursively get the final location (after following all redirects)
+    # for an url.
     def self.get_final_location(url)
       Net::HTTP.get_response(URI.parse(url)) do |res|
         location = res["location"]

data/plugins/blip.rb

@@ -1,4 +1,3 @@
-
 class Blip < PluginBase
   # this will be called by the main app to check whether this plugin is responsible for the url passed
   def self.matches_provider?(url)

data/plugins/vimeo.rb

@@ -1,4 +1,3 @@
-
 class Vimeo < PluginBase
 
   #this will be called by the main app to check whether this plugin is responsible for the url passed

data/plugins/youtube.rb

@@ -1,18 +1,16 @@
-
 class Youtube < PluginBase
 
+  #TODO: TEST THIS: https://www.youtube.com/watch?v=Qapou-3-fM8&list=PL_Z529zmzNGcOVBJA0MgjjQoKiBcmMQWh
+
   # this will be called by the main app to check whether this plugin is responsible for the url passed
   def self.matches_provider?(url)
     url.include?("youtube.com") || url.include?("youtu.be")
   end
 
   def self.get_urls_and_filenames(url, options = {})
+    initialize_components(options)
 
-    @url_resolver   = UrlResolver.new
-    @video_resolver = VideoResolver.new(Decipherer.new(CipherLoader.new))
-    @format_picker  = FormatPicker.new(options)
-
-    urls = @url_resolver.get_all_urls(url, options[:filter])
+    urls   = @url_resolver.get_all_urls(url, options[:filter])
     videos = get_videos(urls)
 
     return_value = videos.map do |video|
@@ -23,6 +21,14 @@ class Youtube < PluginBase
     return_value.empty? ? download_error("No videos could be downloaded.") : return_value
   end
 
+  def self.initialize_components(options)
+    @cipher_io      = CipherIO.new
+    coordinator     = DecipherCoordinator.new(Decipherer.new(@cipher_io), CipherGuesser.new)
+    @video_resolver = VideoResolver.new(coordinator)
+    @url_resolver   = UrlResolver.new
+    @format_picker  = FormatPicker.new(options)
+  end
+
   def self.notify(message)
     puts "[YOUTUBE] #{message}"
   end
@@ -37,17 +43,26 @@ class Youtube < PluginBase
         @video_resolver.get_video(url)
       rescue VideoResolver::VideoRemovedError
         notify "The video #{url} has been removed."
+        nil
       rescue => e
         notify "Error getting the video: #{e.message}"
+        nil
       end
     end
-
     videos.reject(&:nil?)
   end
 
   def self.make_url_filname_hash(video, format)
     url = video.get_download_url(format.itag)
     name = PluginBase.make_filename_safe(video.title) + ".#{format.extension}"
-    {url: url, name: name}
+    {url: url, name: name, on_downloaded: make_downloaded_callback(video)}
+  end
+
+  def self.make_downloaded_callback(video)
+    return nil unless video.signature_guess?
+
+    lambda do |success|
+      @cipher_io.add_cipher(video.cipher_version, video.cipher_operations) if success
+    end
   end
 end

data/plugins/youtube/cipher_guesser.rb

@@ -0,0 +1,114 @@
+require 'open-uri'
+
+class CipherGuesser
+  class CipherGuessError < StandardError; end
+
+  JS_URL = "http://s.ytimg.com/yts/jsbin/html5player-%s.js"
+
+  def guess(cipher_version)
+    js   = download_player_javascript(cipher_version)
+    body = extract_decipher_function_body(js)
+
+    parse_function_body(body)
+  end
+
+  private
+
+  def download_player_javascript(cipher_version)
+    open(JS_URL % cipher_version).read
+  end
+
+  def extract_decipher_function_body(js)
+    function_name  = js[decipher_function_name_regex, 1]
+    function_regex = get_function_regex(function_name)
+    match          = function_regex.match(js)
+
+    raise(CipherGuessError, "Could not extract the decipher function") unless match
+    match[:brace]
+  end
+
+  def parse_function_body(body)
+    lines = body.split(";")
+
+    remove_non_decipher_lines!(lines)
+    do_pre_transformations!(lines)
+
+    lines.map do |line|
+      if /\(\w+,(?<index>\d+)\)/     =~ line  # calling a two argument function (swap)
+        "w#{index}"
+      elsif /slice\((?<index>\d+)\)/ =~ line  # calling slice
+        "s#{index}"
+      elsif /reverse\(\)/            =~ line  # calling reverse
+        "r"
+      else
+        raise "Cannot parse line: #{line}"
+      end
+    end
+  end
+
+  def remove_non_decipher_lines!(lines)
+    # The first line splits the string into an array and the last joins and returns
+    lines.delete_at(0)
+    lines.delete_at(-1)
+  end
+
+  def do_pre_transformations!(lines)
+    change_inline_swap_to_function_call(lines) if inline_swap?(lines)
+  end
+
+  def inline_swap?(lines)
+    # Defining a variable = inline swap function
+    lines.any? { |line| line.include?("var ") }
+  end
+
+  def change_inline_swap_to_function_call(lines)
+    start_index = lines.find_index { |line| line.include?("var ") }
+    swap_lines  = lines.slice!(start_index, 3)  # inline swap is 3 lines long
+    i1, i2      = get_swap_indices(swap_lines)
+
+    lines.insert(start_index, "swap(#{i1},#{i2})")
+    lines
+  end
+
+  def get_swap_indices(lines)
+    i1 = lines.first[/(\d+)/, 1]
+    i2 = lines.last[/(\d+)/, 1]
+    [i1, i2]
+  end
+
+  def decipher_function_name_regex
+    # Find "C" in this: var A = B.sig || C (B.s)
+    /
+    \.sig
+    \s*
+    \|\|
+    (\w+)
+    \(
+    /x
+  end
+
+  def get_function_regex(function_name)
+  # Match the function function_name (that has one argument)
+    /
+    #{function_name}
+    \(
+    \w+
+    \)
+    #{function_body_regex}
+    /x
+  end
+
+  def function_body_regex
+  # Match nested braces
+    /
+    (?<brace>
+    {
+    (
+    [^{}]
+    | \g<brace>
+    )*
+    }
+    )
+    /x
+  end
+end

data/plugins/youtube/{cipher_loader.rb → cipher_io.rb}

@@ -4,7 +4,7 @@ require 'net/http'
 require 'openssl'
 require 'yaml'
 
-class CipherLoader
+class CipherIO
 
   CIPHER_YAML_URL = "https://raw.github.com/rb2k/viddl-rb/master/plugins/youtube/ciphers.yml"
   CIPHER_YAML_PATH = File.join(ViddlRb::UtilityHelper.base_path, "plugins/youtube/ciphers.yml")
@@ -23,24 +23,29 @@ class CipherLoader
     @ciphers.dup
   end
 
+  def add_cipher(version, operations)
+    File.open(CIPHER_YAML_PATH, "a") do |file|
+      file.puts("#{version}: #{operations}")
+    end
+  end
+
   private
 
   def update_ciphers
-    return if get_server_file_size == get_local_file_size
+    server_etag = get_server_etag
+    return if server_etag == @ciphers["ETag"]
 
     @ciphers.merge!(download_server_ciphers)
+    @ciphers["ETag"] = server_etag
     save_local_ciphers(@ciphers)
   end
 
-  def get_local_file_size
-    File.size(CIPHER_YAML_PATH)
-  end
-
-  def get_server_file_size 
+  def get_server_etag
     uri = URI.parse(CIPHER_YAML_URL)
     http = make_http(uri)
     head = Net::HTTP::Head.new(uri.request_uri)
-    http.request(head)["Content-Length"].to_i
+    etag = http.request(head)["ETag"]
+    etag.gsub('"', '')  # remove leading and trailing quotes
   end
 
   def make_http(uri)

data/plugins/youtube/ciphers.yml

@@ -84,3 +84,16 @@ ima-vflxBu-5R: w40 w62 r s2 w21 s3 r w7 s3
 ima-vflrGwWV9: w36 w45 r s2 r
 ima-vflCME3y0: w8 s2 r w52
 ima-vfl1LZyZ5: w8 s2 r w52
+ima-vfl4_saJa: r s1 w19 w9 w57 w38 s3 r s2
+ima-en_US-vflP9269H: r w63 w37 s3 r w14 r
+ima-en_US-vflkClbFb: s1 w12 w24 s1 w52 w70 s2
+ima-en_US-vflYhChiG: w27 r s3
+ima-en_US-vflWnCYSF: r s1 r s3 w19 r w35 w61 s2
+en_US-vflbT9-GA: w51 w15 s1 w22 s1 w41 r w43 r
+en_US-vflAYBrl7: s2 r w39 w43
+en_US-vflS1POwl: w48 s2 r s1 w4 w35
+en_US-vflLMtkhg: w30 r w30 w39
+en_US-vflbJnZqE: w26 s1 w15 w3 w62 w54 w22
+en_US-vflgd5txb: w26 s1 w15 w3 w62 w54 w22
+en_US-vflTm330y: w26 s1 w15 w3 w62 w54 w22
+en_US-vflnwMARr: s3 r w24 s2

data/plugins/youtube/decipher_coordinator.rb

@@ -0,0 +1,28 @@
+
+class DecipherCoordinator
+
+  def initialize(decipherer, cipher_guesser)
+    @decipherer = decipherer
+    @cipher_guesser = cipher_guesser
+  end
+
+  def get_decipher_data(cipher_version)
+    ops = @decipherer.get_operations(cipher_version)
+    Youtube.notify "Cipher guess: no"
+    {version: cipher_version, operations: ops.join(" "), guess?: false}
+
+  rescue Decipherer::UnknownCipherVersionError => e
+    ops = @cipher_guesser.guess(cipher_version)
+    Youtube.notify "Cipher guess: yes"
+    {version: cipher_version, operations: ops.join(" "), guess?: true}
+
+  rescue Decipherer::UnknownCipherOperationError => e
+    Youtube.notify "Failed to parse the cipher from the Youtube player version #{cipher_version}\n" +
+                   "Please submit a bug report at https://github.com/rb2k/viddl-rb"
+    raise e
+  end
+
+  def decipher_with_operations(cipher, operations)
+    @decipherer.decipher_with_operations(cipher, operations)
+  end
+end

data/plugins/youtube/decipherer.rb

@@ -1,22 +1,36 @@
 
 class Decipherer
 
-  class UnknownCipherVersionError < StandardError; end 
   class UnknownCipherOperationError < StandardError; end
 
+  class UnknownCipherVersionError < StandardError
+
+    attr_reader :cipher_version
+
+    def initialize(cipher_version)
+      super("Unknown cipher version: #{cipher_version}")
+      @cipher_version = cipher_version
+    end
+  end
+
   def initialize(loader)
     @ciphers = loader.load_ciphers
   end
 
   def decipher_with_version(cipher, cipher_version)
-    operations = @ciphers[cipher_version] 
-    raise UnknownCipherVersionError.new("Unknown cipher version: #{cipher_version}") unless operations
+    ops = get_operations(cipher_version)
+    decipher_with_operations(cipher, ops)
+  end
 
-    decipher_with_operations(cipher, operations.split)
+  def get_operations(cipher_version)
+    operations = @ciphers[cipher_version] 
+    raise UnknownCipherVersionError.new(cipher_version) unless operations
+    operations.split
   end
 
   def decipher_with_operations(cipher, operations)
     cipher = cipher.dup
+    operations = operations.split if operations.is_a?(String)
 
     operations.each do |op|
       cipher = apply_operation(cipher, op)

data/plugins/youtube/format_picker.rb

@@ -110,7 +110,7 @@ class FormatPicker
     when 1
       formats.first
     else
-      get_default_format(matches_resolution)
+      get_default_format(formats)
     end
   end
 end

data/plugins/youtube/video_resolver.rb

@@ -11,8 +11,13 @@ class VideoResolver
   end
 
   def get_video(url)
-    @json = load_json(url)
-    Video.new(get_title, parse_stream_map(get_stream_map))
+    @json         = load_json(url)
+    decipher_data = @decipherer.get_decipher_data(get_html5player_version)
+    url_data      = parse_stream_map(get_stream_map)
+
+    decipher_signatures!(url_data, decipher_data)
+
+    Video.new(get_title, url_data, decipher_data)
   end
 
   private
@@ -47,10 +52,7 @@ class VideoResolver
   #
   def parse_stream_map(stream_map)
     entries = stream_map.split(",")
-
-    parsed = entries.map { |entry| parse_stream_map_entry(entry) }
-    parsed.each { |entry| apply_signature!(entry) if entry[:sig] }
-    parsed
+    entries.map { |entry| parse_stream_map_entry(entry) }
   end
 
   def parse_stream_map_entry(entry)
@@ -65,7 +67,7 @@ class VideoResolver
   end
 
   # The signature key can be either "sig" or "s".
-  # Very rarely there is no "s" or "sig" paramater. In this case the signature is already
+  # Very rarely there is no "s" or "sig" parameter. In this case the signature is already
   # applied and the the video can be downloaded directly.
   def fetch_signature(params)
     sig = params.fetch("sig", nil) || params.fetch("s", nil)
@@ -79,32 +81,45 @@ class VideoResolver
     text
   end
 
-  def apply_signature!(entry)
-    sig = get_deciphered_sig(entry[:sig])
-    entry[:url] << "&#{SIGNATURE_URL_PARAMETER}=#{sig}"
-    entry.delete(:sig)
-  end
+  def decipher_signatures!(url_data, decipher_data)
+    url_data.each do |entry|
+      next unless entry[:sig]
 
-  def get_deciphered_sig(sig)
-    return sig if sig.length == CORRECT_SIGNATURE_LENGTH
-    @decipherer.decipher_with_version(sig, get_html5player_version)
+      sig = @decipherer.decipher_with_operations(entry[:sig], decipher_data[:operations])
+      entry[:url] << "&#{SIGNATURE_URL_PARAMETER}=#{sig}"
+      entry.delete(:sig)
+    end
   end
 
+
   class Video
     attr_reader :title
 
-    def initialize(title, itags_urls)
+    def initialize(title, url_data, decipher_data)
       @title = title
-      @itags_urls = itags_urls
+      @url_data = url_data
+      @decipher_data = decipher_data
     end
 
     def available_itags
-      @itags_urls.map { |iu| iu[:itag] }
+      @url_data.map { |entry| entry[:itag] }
     end
 
     def get_download_url(itag)
-      itag_url = @itags_urls.find { |iu| iu[:itag] == itag }
-      itag_url[:url] if itag_url
+      entry = @url_data.find { |entry| entry[:itag] == itag }
+      entry[:url] if entry
+    end
+
+    def signature_guess?
+      @decipher_data[:guess?]
+    end
+
+    def cipher_operations
+      @decipher_data[:operations]
+    end
+
+    def cipher_version
+      @decipher_data[:version]
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: viddl-rb
 version: !ruby/object:Gem::Version 
-  version: "0.99"
+  version: 1.0.0
 platform: ruby
 authors: 
 - Marc Seeger
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2014-03-02 00:00:00 Z
+date: 2014-04-07 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: mime-types
@@ -95,8 +95,10 @@ files:
 - plugins/soundcloud.rb
 - plugins/veoh.rb
 - plugins/vimeo.rb
-- plugins/youtube/cipher_loader.rb
+- plugins/youtube/cipher_guesser.rb
+- plugins/youtube/cipher_io.rb
 - plugins/youtube/ciphers.yml
+- plugins/youtube/decipher_coordinator.rb
 - plugins/youtube/decipherer.rb
 - plugins/youtube/format_picker.rb
 - plugins/youtube/url_resolver.rb