viddl-rb 0.75 → 0.76

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -23,7 +23,7 @@ GEM
23
23
  mime-types (1.19)
24
24
  minitest (4.1.0)
25
25
  net-http-digest_auth (1.2.1)
26
- net-http-persistent (2.7)
26
+ net-http-persistent (2.8)
27
27
  nokogiri (1.5.5)
28
28
  nokogiri (1.5.5-java)
29
29
  ntlm-http (0.1.1)
data/README.md CHANGED
@@ -20,6 +20,7 @@ Viddl-rb supports the following command line options:
20
20
  -f, --filter REGEX Filters a video playlist according to the regex (Youtube only right now)
21
21
  -s, --save-dir DIRECTORY Specifies the directory where videos should be saved
22
22
  -d, --downloader TOOL Specifies the tool to download with. Supports 'wget', 'curl' and 'net-http'
23
+ -q, --quality QUALITY Specifies the video format and resolution in the following way => resolution:extension (e.g. 720:mp4). Currently only supported by the Youtube plugin.
23
24
  -h, --help Displays the help screen
24
25
  ```
25
26
 
@@ -109,7 +110,6 @@ __Requirements:__
109
110
  * curl/wget or the [progress bar](http://github.com/nex3/ruby-progressbar/) gem
110
111
  * [Nokogiri](http://nokogiri.org/)
111
112
  * [Mechanize](http://mechanize.rubyforge.org/)
112
- * [Rest-Client](https://github.com/archiloque/rest-client)
113
113
  * ffmpeg if you want to extract audio tracks from the videos
114
114
 
115
115
  __Co Maintainer:__
@@ -14,6 +14,8 @@ class ParameterParser
14
14
  # :title_only => do not download, only print the titles to stdout
15
15
  # :playlist_filter => a regular expression used to filter playlists
16
16
  # :save_dir => the directory where the videos are saved
17
+ # :tool => the download tool (wget, curl, net/http) to use
18
+ # :quality => the resolution and format to download
17
19
  def self.parse_app_parameters(args)
18
20
 
19
21
  # Default option values are set here
@@ -23,7 +25,8 @@ class ParameterParser
23
25
  :title_only => false,
24
26
  :playlist_filter => nil,
25
27
  :save_dir => DEFAULT_SAVE_DIR,
26
- :tool => nil
28
+ :tool => nil,
29
+ :quality => nil
27
30
  }
28
31
 
29
32
  optparse = OptionParser.new do |opts|
@@ -65,6 +68,20 @@ class ParameterParser
65
68
  end
66
69
  end
67
70
 
71
+ opts.on("-q", "--quality QUALITY",
72
+ "Specifies the video format and resolution in the following way => resolution:extension (e.g. 720:mp4)") do |quality|
73
+ if match = quality.match(/(\d+):(.*)/)
74
+ res = match[1]
75
+ ext = match[2]
76
+ elsif match = quality.match(/\d+/)
77
+ res = match[0]
78
+ ext = nil
79
+ else
80
+ raise OptionParse.InvalidArgument.new("#{quality} is not a valid argument.")
81
+ end
82
+ options[:quality] = {:extension => ext, :resolution => res}
83
+ end
84
+
68
85
  opts.on_tail('-h', '--help', 'Display this screen') do
69
86
  print_help_and_exit(opts)
70
87
  end
@@ -6,15 +6,15 @@ module ViddlRb
6
6
 
7
7
  #viddl will use the first of these tools it finds on the system to download the video.
8
8
  #if the system does not have any of these tools, net/http is used instead.
9
- TOOLS_PRIORITY_LIST = [:wget, :curl]
10
-
9
+ TOOLS_PRIORITY_LIST = [:wget, :curl]
10
+
11
11
  #simple helper that will save a file from the web and save it with a progress bar
12
12
  def self.save_file(file_url, file_name, opts = {})
13
13
  trap("SIGINT") { puts "goodbye"; exit }
14
14
 
15
15
  #default options
16
- options = {:save_dir => ".",
17
- :amount_of_retries => 6,
16
+ options = {:save_dir => ".",
17
+ :amount_of_retries => 6,
18
18
  :tool => get_tool}
19
19
 
20
20
  opts[:tool] = options[:tool] if opts[:tool].nil?
@@ -37,15 +37,15 @@ module ViddlRb
37
37
  require_progressbar
38
38
  puts "Using net/http"
39
39
  success = download_and_save_file(file_url, file_path)
40
- end
40
+ end
41
41
  #we were successful, we're outta here
42
42
  if success
43
43
  break
44
44
  else
45
- puts "Download seems to have failed (retrying, attempt #{i+1}/#{amount_of_retries})"
45
+ puts "Download seems to have failed (retrying, attempt #{i+1}/#{options[:amount_of_retries]})"
46
46
  sleep 2
47
47
  end
48
- end
48
+ end
49
49
  success
50
50
  end
51
51
 
@@ -73,7 +73,7 @@ module ViddlRb
73
73
 
74
74
  Net::HTTP.start(uri.host, uri.port) do |http|
75
75
  http.request_get(uri.request_uri) do |res|
76
- file_size = res.read_header["content-length"].to_i
76
+ file_size = res.read_header["content-length"].to_i
77
77
  bar = ProgressBar.new(File.basename(full_path), file_size)
78
78
  bar.file_transfer_mode
79
79
  res.read_body do |segment|
@@ -1,4 +1,4 @@
1
- require 'rest_client'
1
+ require 'open-uri'
2
2
  class Soundcloud < PluginBase
3
3
  # this will be called by the main app to check whether this plugin is responsible for the url passed
4
4
  def self.matches_provider?(url)
@@ -7,7 +7,7 @@ class Soundcloud < PluginBase
7
7
 
8
8
  # return the url for original video file and title
9
9
  def self.get_urls_and_filenames(url, options = {})
10
- doc = Nokogiri::HTML(RestClient.get(url).body)
10
+ doc = Nokogiri::HTML(open(url))
11
11
  download_filename = doc.at("#main-content-inner img[class=waveform]").attributes["src"].value.to_s.match(/\.com\/(.+)\_/)[1]
12
12
  download_url = "http://media.soundcloud.com/stream/#{download_filename}"
13
13
  file_name = transliterate("#{doc.at('//h1/em').text.chomp}") + ".mp3"
@@ -1,5 +1,24 @@
1
+ require 'open-uri'
1
2
 
2
3
  class Youtube < PluginBase
4
+
5
+ VIDEO_INFO_URL = "http://www.youtube.com/get_video_info?video_id="
6
+
7
+ VIDEO_FORMATS = {
8
+ "38" => {:extension => "mp4", :name => "MP4 Highest Quality 4096x3027 (H.264, AAC)"},
9
+ "37" => {:extension => "mp4", :name => "MP4 Highest Quality 1920x1080 (H.264, AAC)"},
10
+ "22" => {:extension => "mp4", :name => "MP4 1280x720 (H.264, AAC)"},
11
+ "45" => {:extension => "webm", :name => "WebM 1280x720 (VP8, Vorbis)"},
12
+ "44" => {:extension => "webm", :name => "WebM 854x480 (VP8, Vorbis)"},
13
+ "18" => {:extension => "mp4", :name => "MP4 640x360 (H.264, AAC)"},
14
+ "35" => {:extension => "flv", :name => "FLV 854x480 (H.264, AAC)"},
15
+ "34" => {:extension => "flv", :name => "FLV 640x360 (H.264, AAC)"},
16
+ "5" => {:extension => "flv", :name => "FLV 400x240 (Soerenson H.263)"},
17
+ "17" => {:extension => "3gp", :name => "3gp"}
18
+ }
19
+
20
+ DEFAULT_FORMAT_ORDER = %w[38 37 22 45 44 18 35 34 5 7]
21
+
3
22
  #this will be called by the main app to check whether this plugin is responsible for the url passed
4
23
  def self.matches_provider?(url)
5
24
  url.include?("youtube.com") || url.include?("youtu.be")
@@ -7,17 +26,17 @@ class Youtube < PluginBase
7
26
 
8
27
  #get all videos and return their urls in an array
9
28
  def self.get_video_urls(feed_url)
10
- puts "[YOUTUBE] Retrieving videos..."
29
+ notify "Retrieving videos..."
11
30
  urls_titles = Hash.new
12
31
  result_feed = Nokogiri::XML(open(feed_url))
13
- urls_titles.merge!(grab_ut(result_feed))
32
+ urls_titles.merge!(grab_urls_and_titles(result_feed))
14
33
 
15
34
  #as long as the feed has a next link we follow it and add the resulting video urls
16
35
  loop do
17
36
  next_link = result_feed.search("//feed/link[@rel='next']").first
18
37
  break if next_link.nil?
19
38
  result_feed = Nokogiri::HTML(open(next_link["href"]))
20
- urls_titles.merge!(grab_ut(result_feed))
39
+ urls_titles.merge!(grab_urls_and_titles(result_feed))
21
40
  end
22
41
 
23
42
  self.filter_urls(urls_titles)
@@ -26,7 +45,7 @@ class Youtube < PluginBase
26
45
  #returns only the urls that match the --filter argument regex (if present)
27
46
  def self.filter_urls(url_hash)
28
47
  if @filter
29
- puts "[YOUTUBE] Using filter: #{@filter}"
48
+ notify "Using filter: #{@filter}"
30
49
  filtered = url_hash.select { |url, title| title =~ @filter }
31
50
  filtered.keys
32
51
  else
@@ -35,7 +54,7 @@ class Youtube < PluginBase
35
54
  end
36
55
 
37
56
  #extract all video urls and their titles from a feed and return in a hash
38
- def self.grab_ut(feed)
57
+ def self.grab_urls_and_titles(feed)
39
58
  feed.remove_namespaces! #so that we can get to the titles easily
40
59
  urls = feed.search("//entry/link[@rel='alternate']").map { |link| link["href"] }
41
60
  titles = feed.search("//entry/group/title").map { |title| title.text }
@@ -48,44 +67,48 @@ class Youtube < PluginBase
48
67
  #http://www.youtube.com/watch?v=Tk78sr5JMIU&videos=jKY836_WMhE
49
68
 
50
69
  playlist_ID = url[/(?:list=PL|p=)(\w{16})&?/,1]
51
- puts "[YOUTUBE] Playlist ID: #{playlist_ID}"
70
+ notify "Playlist ID: #{playlist_ID}"
52
71
  feed_url = "http://gdata.youtube.com/feeds/api/playlists/#{playlist_ID}?&max-results=50&v=2"
53
72
  url_array = self.get_video_urls(feed_url)
54
- puts "[YOUTUBE] #{url_array.size} links found!"
73
+ notify "#{url_array.size} links found!"
55
74
  url_array
56
75
  end
57
76
 
58
77
  def self.parse_user(username)
59
- puts "[YOUTUBE] User: #{username}"
78
+ notify "User: #{username}"
60
79
  feed_url = "http://gdata.youtube.com/feeds/api/users/#{username}/uploads?&max-results=50&v=2"
61
80
  url_array = get_video_urls(feed_url)
62
- puts "[YOUTUBE] #{url_array.size} links found!"
81
+ notify "#{url_array.size} links found!"
63
82
  url_array
64
83
  end
65
84
 
66
85
  def self.get_urls_and_filenames(url, options = {})
67
86
  @filter = options[:playlist_filter] #used to filter a playlist in self.filter_urls
87
+ @quality = options[:quality]
88
+
68
89
  return_values = []
90
+
69
91
  if url.include?("view_play_list") || url.include?("playlist?list=") #if playlist
70
- puts "[YOUTUBE] playlist found! analyzing..."
71
- files = self.parse_playlist(url)
72
- puts "[YOUTUBE] Starting playlist download"
92
+ notify "playlist found! analyzing..."
93
+ files = parse_playlist(url)
94
+ notify "Starting playlist download"
73
95
  files.each do |file|
74
- puts "[YOUTUBE] Downloading next movie on the playlist (#{file})"
75
- return_values << self.grab_single_url_filename(file)
96
+ notify "Downloading next movie on the playlist (#{file})"
97
+ return_values << grab_single_url_filename(file)
76
98
  end
77
99
  elsif match = url.match(/\/user\/([\w\d]+)$/) #if user url, e.g. youtube.com/user/woot
78
100
  username = match[1]
79
- video_urls = self.parse_user(username)
80
- puts "[YOUTUBE] Starting user videos download"
101
+ video_urls = parse_user(username)
102
+ notify "Starting user videos download"
81
103
  video_urls.each do |url|
82
- puts "[YOUTUBE] Downloading next user video (#{url})"
83
- return_values << self.grab_single_url_filename(url)
104
+ notify "Downloading next user video (#{url})"
105
+ return_values << grab_single_url_filename(url)
84
106
  end
85
107
  else #if single video
86
- return_values << self.grab_single_url_filename(url)
108
+ return_values << grab_single_url_filename(url)
87
109
  end
88
- return_values.reject! { |value| value == :no_embed } #remove results that can not be downloaded
110
+
111
+ return_values.reject! { |value| value == :no_embed } #remove results that can not be downloaded
89
112
 
90
113
  if return_values.empty?
91
114
  raise CouldNotDownloadVideoError, "No videos could be downloaded - embedding disabled."
@@ -99,18 +122,14 @@ class Youtube < PluginBase
99
122
  #addition: might also look like this /v/abc5-a5afe5agae6g
100
123
  # alternative: video_id = url[/v[\/=]([\w-]*)&?/, 1]
101
124
  # First get the redirect
102
- if url.include?("youtu.be")
103
- url = open(url).base_uri.to_s
104
- end
125
+
126
+ url = open(url).base_uri.to_s if url.include?("youtu.be")
105
127
  video_id = url[/(v|embed)[=\/]([^\/\?\&]*)/,2]
106
- if video_id.nil?
107
- raise CouldNotDownloadVideoError, "No video id found."
108
- else
109
- puts "[YOUTUBE] ID FOUND: #{video_id}"
110
- end
128
+ video_id ? notify("ID FOUND: #{video_id}") : download_error("No video id found.")
129
+
111
130
  #let's get some infos about the video. data is urlencoded
112
- yt_url = "http://www.youtube.com/get_video_info?video_id=#{video_id}"
113
- video_info = RestClient.get(yt_url).body
131
+ video_info = open(VIDEO_INFO_URL + video_id).read
132
+
114
133
  #converting the huge infostring into a hash. simply by splitting it at the & and then splitting it into key and value arround the =
115
134
  #[...]blabla=blubb&narf=poit&marc=awesome[...]
116
135
  video_info_hash = Hash[*video_info.split("&").collect { |v|
@@ -150,42 +169,22 @@ class Youtube < PluginBase
150
169
  [key, value]
151
170
  }.flatten]
152
171
 
153
- if video_info_hash["status"] == "fail"
154
- return :no_embed
155
- end
156
-
172
+ return :no_embed if video_info_hash["status"] == "fail"
173
+
157
174
  title = video_info_hash["title"]
158
175
  length_s = video_info_hash["length_seconds"]
159
176
  token = video_info_hash["token"]
160
177
 
178
+ notify "Title: #{title}"
179
+ notify "Length: #{length_s} s"
180
+ notify "t-parameter: #{token}"
181
+
161
182
  #for the formats, see: http://en.wikipedia.org/wiki/YouTube#Quality_and_codecs
162
183
  fmt_list = video_info_hash["fmt_list"].split(",")
163
- available_formats = fmt_list.map{|format| format.split("/").first}
164
-
165
- format_ext = {}
166
- format_ext["38"] = {:extension => "mp4", :name => "MP4 Highest Quality 4096x3027 (H.264, AAC)"}
167
- format_ext["37"] = {:extension => "mp4", :name => "MP4 Highest Quality 1920x1080 (H.264, AAC)"}
168
- format_ext["22"] = {:extension => "mp4", :name => "MP4 1280x720 (H.264, AAC)"}
169
- format_ext["45"] = {:extension => "webm", :name => "WebM 1280x720 (VP8, Vorbis)"}
170
- format_ext["44"] = {:extension => "webm", :name => "WebM 854x480 (VP8, Vorbis)"}
171
- format_ext["18"] = {:extension => "mp4", :name => "MP4 640x360 (H.264, AAC)"}
172
- format_ext["35"] = {:extension => "flv", :name => "FLV 854x480 (H.264, AAC)"}
173
- format_ext["34"] = {:extension => "flv", :name => "FLV 640x360 (H.264, AAC)"}
174
- format_ext["5"] = {:extension => "flv", :name => "FLV 400x240 (Soerenson H.263)"}
175
- format_ext["17"] = {:extension => "3gp", :name => "3gp"}
176
-
177
- #since 1.8 doesn't do ordered hashes
178
- prefered_order = ["38","37","22","45","44","18","35","34","5","17"]
179
-
180
- selected_format = prefered_order.select{|possible_format| available_formats.include?(possible_format)}.first
181
-
182
- puts "[YOUTUBE] Title: #{title}"
183
- puts "[YOUTUBE] Length: #{length_s} s"
184
- puts "[YOUTUBE] t-parameter: #{token}"
185
- #best quality seems always to be firsts
186
- puts "[YOUTUBE] formats available: #{available_formats.inspect} (downloading format #{selected_format} -> #{format_ext[selected_format][:name]})"
187
184
 
188
- #video_info_hash.keys.sort.each{|key| puts "#{key} : #{video_info_hash[key]}" }
185
+ selected_format = pick_video_format(fmt_list)
186
+ puts "(downloading format #{selected_format} -> #{VIDEO_FORMATS[selected_format][:name]})"
187
+
189
188
  download_url = video_info_hash["url_encoded_fmt_stream_map"][selected_format]
190
189
 
191
190
  #if download url ends with a ';' followed by a codec string remove that part because it stops URI.parse from working
@@ -199,8 +198,40 @@ class Youtube < PluginBase
199
198
  download_url.sub!("&sig=", "&signature=") #else we just have to change sig to signature
200
199
  end
201
200
 
202
- file_name = PluginBase.make_filename_safe(title) + "." + format_ext[selected_format][:extension]
203
- puts "downloading to " + file_name
201
+ file_name = PluginBase.make_filename_safe(title) + "." + VIDEO_FORMATS[selected_format][:extension]
202
+ puts "downloading to " + file_name + "\n\n"
204
203
  {:url => download_url, :name => file_name}
205
204
  end
205
+
206
+ #returns the format of the video the user picked or the first default format if it does not exist
207
+ def self.pick_video_format(fmt_list)
208
+ available_formats = fmt_list.map { |format| format.split("/").first }
209
+ notify "formats available: #{available_formats.inspect}"
210
+
211
+ if @quality #if the user specified a format
212
+ ext = @quality[:extension]
213
+ res = @quality[:resolution]
214
+
215
+ #gets a nested array with all the formats of the same res as the user wanted
216
+ requested = VIDEO_FORMATS.select { |id, format| format[:name].include?(res) }.to_a
217
+
218
+ if requested.empty?
219
+ notify "Requested format \"#{res}:#{ext}\" not found. Downloading default format."
220
+ get_default_format(available_formats)
221
+ else
222
+ pick = requested.find { |format| format[1][:extension] == ext } #get requsted extension if possible
223
+ pick ? pick.first : get_default_format(requested.map { |req| req.first }) #else return the default format
224
+ end
225
+ else
226
+ get_default_format(available_formats)
227
+ end
228
+ end
229
+
230
+ def self.get_default_format(available)
231
+ DEFAULT_FORMAT_ORDER.find { |default| available.include?(default) }
232
+ end
233
+
234
+ def self.notify(message)
235
+ puts "[YOUTUBE] #{message}"
236
+ end
206
237
  end
metadata CHANGED
@@ -1,12 +1,12 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: viddl-rb
3
3
  version: !ruby/object:Gem::Version
4
- hash: 157
4
+ hash: 147
5
5
  prerelease:
6
6
  segments:
7
7
  - 0
8
- - 75
9
- version: "0.75"
8
+ - 76
9
+ version: "0.76"
10
10
  platform: ruby
11
11
  authors:
12
12
  - Marc Seeger
@@ -14,8 +14,7 @@ autorequire:
14
14
  bindir: bin
15
15
  cert_chain: []
16
16
 
17
- date: 2012-10-16 00:00:00 +02:00
18
- default_executable:
17
+ date: 2013-01-30 00:00:00 Z
19
18
  dependencies:
20
19
  - !ruby/object:Gem::Dependency
21
20
  name: nokogiri
@@ -46,7 +45,7 @@ dependencies:
46
45
  type: :runtime
47
46
  version_requirements: *id002
48
47
  - !ruby/object:Gem::Dependency
49
- name: rest-client
48
+ name: progressbar
50
49
  prerelease: false
51
50
  requirement: &id003 !ruby/object:Gem::Requirement
52
51
  none: false
@@ -60,7 +59,7 @@ dependencies:
60
59
  type: :runtime
61
60
  version_requirements: *id003
62
61
  - !ruby/object:Gem::Dependency
63
- name: progressbar
62
+ name: rake
64
63
  prerelease: false
65
64
  requirement: &id004 !ruby/object:Gem::Requirement
66
65
  none: false
@@ -71,10 +70,10 @@ dependencies:
71
70
  segments:
72
71
  - 0
73
72
  version: "0"
74
- type: :runtime
73
+ type: :development
75
74
  version_requirements: *id004
76
75
  - !ruby/object:Gem::Dependency
77
- name: rake
76
+ name: rest-client
78
77
  prerelease: false
79
78
  requirement: &id005 !ruby/object:Gem::Requirement
80
79
  none: false
@@ -131,7 +130,6 @@ files:
131
130
  - Rakefile
132
131
  - README.md
133
132
  - TODO.txt
134
- has_rdoc: false
135
133
  homepage: https://github.com/rb2k/viddl-rb
136
134
  licenses: []
137
135
 
@@ -163,9 +161,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
163
161
  requirements: []
164
162
 
165
163
  rubyforge_project: viddl-rb
166
- rubygems_version: 1.6.2
164
+ rubygems_version: 1.8.24
167
165
  signing_key:
168
166
  specification_version: 3
169
167
  summary: An extendable commandline video downloader for flash video sites.
170
168
  test_files: []
171
169
 
170
+ has_rdoc: false