viddl-rb 0.75 → 0.76
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile.lock +1 -1
- data/README.md +1 -1
- data/bin/helper/parameter-parser.rb +18 -1
- data/helper/download-helper.rb +8 -8
- data/plugins/soundcloud.rb +2 -2
- data/plugins/youtube.rb +92 -61
- metadata +10 -11
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -20,6 +20,7 @@ Viddl-rb supports the following command line options:
|
|
20
20
|
-f, --filter REGEX Filters a video playlist according to the regex (Youtube only right now)
|
21
21
|
-s, --save-dir DIRECTORY Specifies the directory where videos should be saved
|
22
22
|
-d, --downloader TOOL Specifies the tool to download with. Supports 'wget', 'curl' and 'net-http'
|
23
|
+
-q, --quality QUALITY Specifies the video format and resolution in the following way => resolution:extension (e.g. 720:mp4). Currently only supported by the Youtube plugin.
|
23
24
|
-h, --help Displays the help screen
|
24
25
|
```
|
25
26
|
|
@@ -109,7 +110,6 @@ __Requirements:__
|
|
109
110
|
* curl/wget or the [progress bar](http://github.com/nex3/ruby-progressbar/) gem
|
110
111
|
* [Nokogiri](http://nokogiri.org/)
|
111
112
|
* [Mechanize](http://mechanize.rubyforge.org/)
|
112
|
-
* [Rest-Client](https://github.com/archiloque/rest-client)
|
113
113
|
* ffmpeg if you want to extract audio tracks from the videos
|
114
114
|
|
115
115
|
__Co Maintainer:__
|
@@ -14,6 +14,8 @@ class ParameterParser
|
|
14
14
|
# :title_only => do not download, only print the titles to stdout
|
15
15
|
# :playlist_filter => a regular expression used to filter playlists
|
16
16
|
# :save_dir => the directory where the videos are saved
|
17
|
+
# :tool => the download tool (wget, curl, net/http) to use
|
18
|
+
# :quality => the resolution and format to download
|
17
19
|
def self.parse_app_parameters(args)
|
18
20
|
|
19
21
|
# Default option values are set here
|
@@ -23,7 +25,8 @@ class ParameterParser
|
|
23
25
|
:title_only => false,
|
24
26
|
:playlist_filter => nil,
|
25
27
|
:save_dir => DEFAULT_SAVE_DIR,
|
26
|
-
:tool => nil
|
28
|
+
:tool => nil,
|
29
|
+
:quality => nil
|
27
30
|
}
|
28
31
|
|
29
32
|
optparse = OptionParser.new do |opts|
|
@@ -65,6 +68,20 @@ class ParameterParser
|
|
65
68
|
end
|
66
69
|
end
|
67
70
|
|
71
|
+
opts.on("-q", "--quality QUALITY",
|
72
|
+
"Specifies the video format and resolution in the following way => resolution:extension (e.g. 720:mp4)") do |quality|
|
73
|
+
if match = quality.match(/(\d+):(.*)/)
|
74
|
+
res = match[1]
|
75
|
+
ext = match[2]
|
76
|
+
elsif match = quality.match(/\d+/)
|
77
|
+
res = match[0]
|
78
|
+
ext = nil
|
79
|
+
else
|
80
|
+
raise OptionParse.InvalidArgument.new("#{quality} is not a valid argument.")
|
81
|
+
end
|
82
|
+
options[:quality] = {:extension => ext, :resolution => res}
|
83
|
+
end
|
84
|
+
|
68
85
|
opts.on_tail('-h', '--help', 'Display this screen') do
|
69
86
|
print_help_and_exit(opts)
|
70
87
|
end
|
data/helper/download-helper.rb
CHANGED
@@ -6,15 +6,15 @@ module ViddlRb
|
|
6
6
|
|
7
7
|
#viddl will use the first of these tools it finds on the system to download the video.
|
8
8
|
#if the system does not have any of these tools, net/http is used instead.
|
9
|
-
TOOLS_PRIORITY_LIST = [:wget, :curl]
|
10
|
-
|
9
|
+
TOOLS_PRIORITY_LIST = [:wget, :curl]
|
10
|
+
|
11
11
|
#simple helper that will save a file from the web and save it with a progress bar
|
12
12
|
def self.save_file(file_url, file_name, opts = {})
|
13
13
|
trap("SIGINT") { puts "goodbye"; exit }
|
14
14
|
|
15
15
|
#default options
|
16
|
-
options = {:save_dir => ".",
|
17
|
-
:amount_of_retries => 6,
|
16
|
+
options = {:save_dir => ".",
|
17
|
+
:amount_of_retries => 6,
|
18
18
|
:tool => get_tool}
|
19
19
|
|
20
20
|
opts[:tool] = options[:tool] if opts[:tool].nil?
|
@@ -37,15 +37,15 @@ module ViddlRb
|
|
37
37
|
require_progressbar
|
38
38
|
puts "Using net/http"
|
39
39
|
success = download_and_save_file(file_url, file_path)
|
40
|
-
end
|
40
|
+
end
|
41
41
|
#we were successful, we're outta here
|
42
42
|
if success
|
43
43
|
break
|
44
44
|
else
|
45
|
-
puts "Download seems to have failed (retrying, attempt #{i+1}/#{amount_of_retries})"
|
45
|
+
puts "Download seems to have failed (retrying, attempt #{i+1}/#{options[:amount_of_retries]})"
|
46
46
|
sleep 2
|
47
47
|
end
|
48
|
-
end
|
48
|
+
end
|
49
49
|
success
|
50
50
|
end
|
51
51
|
|
@@ -73,7 +73,7 @@ module ViddlRb
|
|
73
73
|
|
74
74
|
Net::HTTP.start(uri.host, uri.port) do |http|
|
75
75
|
http.request_get(uri.request_uri) do |res|
|
76
|
-
file_size = res.read_header["content-length"].to_i
|
76
|
+
file_size = res.read_header["content-length"].to_i
|
77
77
|
bar = ProgressBar.new(File.basename(full_path), file_size)
|
78
78
|
bar.file_transfer_mode
|
79
79
|
res.read_body do |segment|
|
data/plugins/soundcloud.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
require '
|
1
|
+
require 'open-uri'
|
2
2
|
class Soundcloud < PluginBase
|
3
3
|
# this will be called by the main app to check whether this plugin is responsible for the url passed
|
4
4
|
def self.matches_provider?(url)
|
@@ -7,7 +7,7 @@ class Soundcloud < PluginBase
|
|
7
7
|
|
8
8
|
# return the url for original video file and title
|
9
9
|
def self.get_urls_and_filenames(url, options = {})
|
10
|
-
doc = Nokogiri::HTML(
|
10
|
+
doc = Nokogiri::HTML(open(url))
|
11
11
|
download_filename = doc.at("#main-content-inner img[class=waveform]").attributes["src"].value.to_s.match(/\.com\/(.+)\_/)[1]
|
12
12
|
download_url = "http://media.soundcloud.com/stream/#{download_filename}"
|
13
13
|
file_name = transliterate("#{doc.at('//h1/em').text.chomp}") + ".mp3"
|
data/plugins/youtube.rb
CHANGED
@@ -1,5 +1,24 @@
|
|
1
|
+
require 'open-uri'
|
1
2
|
|
2
3
|
class Youtube < PluginBase
|
4
|
+
|
5
|
+
VIDEO_INFO_URL = "http://www.youtube.com/get_video_info?video_id="
|
6
|
+
|
7
|
+
VIDEO_FORMATS = {
|
8
|
+
"38" => {:extension => "mp4", :name => "MP4 Highest Quality 4096x3027 (H.264, AAC)"},
|
9
|
+
"37" => {:extension => "mp4", :name => "MP4 Highest Quality 1920x1080 (H.264, AAC)"},
|
10
|
+
"22" => {:extension => "mp4", :name => "MP4 1280x720 (H.264, AAC)"},
|
11
|
+
"45" => {:extension => "webm", :name => "WebM 1280x720 (VP8, Vorbis)"},
|
12
|
+
"44" => {:extension => "webm", :name => "WebM 854x480 (VP8, Vorbis)"},
|
13
|
+
"18" => {:extension => "mp4", :name => "MP4 640x360 (H.264, AAC)"},
|
14
|
+
"35" => {:extension => "flv", :name => "FLV 854x480 (H.264, AAC)"},
|
15
|
+
"34" => {:extension => "flv", :name => "FLV 640x360 (H.264, AAC)"},
|
16
|
+
"5" => {:extension => "flv", :name => "FLV 400x240 (Soerenson H.263)"},
|
17
|
+
"17" => {:extension => "3gp", :name => "3gp"}
|
18
|
+
}
|
19
|
+
|
20
|
+
DEFAULT_FORMAT_ORDER = %w[38 37 22 45 44 18 35 34 5 7]
|
21
|
+
|
3
22
|
#this will be called by the main app to check whether this plugin is responsible for the url passed
|
4
23
|
def self.matches_provider?(url)
|
5
24
|
url.include?("youtube.com") || url.include?("youtu.be")
|
@@ -7,17 +26,17 @@ class Youtube < PluginBase
|
|
7
26
|
|
8
27
|
#get all videos and return their urls in an array
|
9
28
|
def self.get_video_urls(feed_url)
|
10
|
-
|
29
|
+
notify "Retrieving videos..."
|
11
30
|
urls_titles = Hash.new
|
12
31
|
result_feed = Nokogiri::XML(open(feed_url))
|
13
|
-
urls_titles.merge!(
|
32
|
+
urls_titles.merge!(grab_urls_and_titles(result_feed))
|
14
33
|
|
15
34
|
#as long as the feed has a next link we follow it and add the resulting video urls
|
16
35
|
loop do
|
17
36
|
next_link = result_feed.search("//feed/link[@rel='next']").first
|
18
37
|
break if next_link.nil?
|
19
38
|
result_feed = Nokogiri::HTML(open(next_link["href"]))
|
20
|
-
urls_titles.merge!(
|
39
|
+
urls_titles.merge!(grab_urls_and_titles(result_feed))
|
21
40
|
end
|
22
41
|
|
23
42
|
self.filter_urls(urls_titles)
|
@@ -26,7 +45,7 @@ class Youtube < PluginBase
|
|
26
45
|
#returns only the urls that match the --filter argument regex (if present)
|
27
46
|
def self.filter_urls(url_hash)
|
28
47
|
if @filter
|
29
|
-
|
48
|
+
notify "Using filter: #{@filter}"
|
30
49
|
filtered = url_hash.select { |url, title| title =~ @filter }
|
31
50
|
filtered.keys
|
32
51
|
else
|
@@ -35,7 +54,7 @@ class Youtube < PluginBase
|
|
35
54
|
end
|
36
55
|
|
37
56
|
#extract all video urls and their titles from a feed and return in a hash
|
38
|
-
def self.
|
57
|
+
def self.grab_urls_and_titles(feed)
|
39
58
|
feed.remove_namespaces! #so that we can get to the titles easily
|
40
59
|
urls = feed.search("//entry/link[@rel='alternate']").map { |link| link["href"] }
|
41
60
|
titles = feed.search("//entry/group/title").map { |title| title.text }
|
@@ -48,44 +67,48 @@ class Youtube < PluginBase
|
|
48
67
|
#http://www.youtube.com/watch?v=Tk78sr5JMIU&videos=jKY836_WMhE
|
49
68
|
|
50
69
|
playlist_ID = url[/(?:list=PL|p=)(\w{16})&?/,1]
|
51
|
-
|
70
|
+
notify "Playlist ID: #{playlist_ID}"
|
52
71
|
feed_url = "http://gdata.youtube.com/feeds/api/playlists/#{playlist_ID}?&max-results=50&v=2"
|
53
72
|
url_array = self.get_video_urls(feed_url)
|
54
|
-
|
73
|
+
notify "#{url_array.size} links found!"
|
55
74
|
url_array
|
56
75
|
end
|
57
76
|
|
58
77
|
def self.parse_user(username)
|
59
|
-
|
78
|
+
notify "User: #{username}"
|
60
79
|
feed_url = "http://gdata.youtube.com/feeds/api/users/#{username}/uploads?&max-results=50&v=2"
|
61
80
|
url_array = get_video_urls(feed_url)
|
62
|
-
|
81
|
+
notify "#{url_array.size} links found!"
|
63
82
|
url_array
|
64
83
|
end
|
65
84
|
|
66
85
|
def self.get_urls_and_filenames(url, options = {})
|
67
86
|
@filter = options[:playlist_filter] #used to filter a playlist in self.filter_urls
|
87
|
+
@quality = options[:quality]
|
88
|
+
|
68
89
|
return_values = []
|
90
|
+
|
69
91
|
if url.include?("view_play_list") || url.include?("playlist?list=") #if playlist
|
70
|
-
|
71
|
-
files =
|
72
|
-
|
92
|
+
notify "playlist found! analyzing..."
|
93
|
+
files = parse_playlist(url)
|
94
|
+
notify "Starting playlist download"
|
73
95
|
files.each do |file|
|
74
|
-
|
75
|
-
return_values <<
|
96
|
+
notify "Downloading next movie on the playlist (#{file})"
|
97
|
+
return_values << grab_single_url_filename(file)
|
76
98
|
end
|
77
99
|
elsif match = url.match(/\/user\/([\w\d]+)$/) #if user url, e.g. youtube.com/user/woot
|
78
100
|
username = match[1]
|
79
|
-
video_urls =
|
80
|
-
|
101
|
+
video_urls = parse_user(username)
|
102
|
+
notify "Starting user videos download"
|
81
103
|
video_urls.each do |url|
|
82
|
-
|
83
|
-
return_values <<
|
104
|
+
notify "Downloading next user video (#{url})"
|
105
|
+
return_values << grab_single_url_filename(url)
|
84
106
|
end
|
85
107
|
else #if single video
|
86
|
-
return_values <<
|
108
|
+
return_values << grab_single_url_filename(url)
|
87
109
|
end
|
88
|
-
|
110
|
+
|
111
|
+
return_values.reject! { |value| value == :no_embed } #remove results that can not be downloaded
|
89
112
|
|
90
113
|
if return_values.empty?
|
91
114
|
raise CouldNotDownloadVideoError, "No videos could be downloaded - embedding disabled."
|
@@ -99,18 +122,14 @@ class Youtube < PluginBase
|
|
99
122
|
#addition: might also look like this /v/abc5-a5afe5agae6g
|
100
123
|
# alternative: video_id = url[/v[\/=]([\w-]*)&?/, 1]
|
101
124
|
# First get the redirect
|
102
|
-
|
103
|
-
|
104
|
-
end
|
125
|
+
|
126
|
+
url = open(url).base_uri.to_s if url.include?("youtu.be")
|
105
127
|
video_id = url[/(v|embed)[=\/]([^\/\?\&]*)/,2]
|
106
|
-
|
107
|
-
|
108
|
-
else
|
109
|
-
puts "[YOUTUBE] ID FOUND: #{video_id}"
|
110
|
-
end
|
128
|
+
video_id ? notify("ID FOUND: #{video_id}") : download_error("No video id found.")
|
129
|
+
|
111
130
|
#let's get some infos about the video. data is urlencoded
|
112
|
-
|
113
|
-
|
131
|
+
video_info = open(VIDEO_INFO_URL + video_id).read
|
132
|
+
|
114
133
|
#converting the huge infostring into a hash. simply by splitting it at the & and then splitting it into key and value arround the =
|
115
134
|
#[...]blabla=blubb&narf=poit&marc=awesome[...]
|
116
135
|
video_info_hash = Hash[*video_info.split("&").collect { |v|
|
@@ -150,42 +169,22 @@ class Youtube < PluginBase
|
|
150
169
|
[key, value]
|
151
170
|
}.flatten]
|
152
171
|
|
153
|
-
if video_info_hash["status"] == "fail"
|
154
|
-
|
155
|
-
end
|
156
|
-
|
172
|
+
return :no_embed if video_info_hash["status"] == "fail"
|
173
|
+
|
157
174
|
title = video_info_hash["title"]
|
158
175
|
length_s = video_info_hash["length_seconds"]
|
159
176
|
token = video_info_hash["token"]
|
160
177
|
|
178
|
+
notify "Title: #{title}"
|
179
|
+
notify "Length: #{length_s} s"
|
180
|
+
notify "t-parameter: #{token}"
|
181
|
+
|
161
182
|
#for the formats, see: http://en.wikipedia.org/wiki/YouTube#Quality_and_codecs
|
162
183
|
fmt_list = video_info_hash["fmt_list"].split(",")
|
163
|
-
available_formats = fmt_list.map{|format| format.split("/").first}
|
164
|
-
|
165
|
-
format_ext = {}
|
166
|
-
format_ext["38"] = {:extension => "mp4", :name => "MP4 Highest Quality 4096x3027 (H.264, AAC)"}
|
167
|
-
format_ext["37"] = {:extension => "mp4", :name => "MP4 Highest Quality 1920x1080 (H.264, AAC)"}
|
168
|
-
format_ext["22"] = {:extension => "mp4", :name => "MP4 1280x720 (H.264, AAC)"}
|
169
|
-
format_ext["45"] = {:extension => "webm", :name => "WebM 1280x720 (VP8, Vorbis)"}
|
170
|
-
format_ext["44"] = {:extension => "webm", :name => "WebM 854x480 (VP8, Vorbis)"}
|
171
|
-
format_ext["18"] = {:extension => "mp4", :name => "MP4 640x360 (H.264, AAC)"}
|
172
|
-
format_ext["35"] = {:extension => "flv", :name => "FLV 854x480 (H.264, AAC)"}
|
173
|
-
format_ext["34"] = {:extension => "flv", :name => "FLV 640x360 (H.264, AAC)"}
|
174
|
-
format_ext["5"] = {:extension => "flv", :name => "FLV 400x240 (Soerenson H.263)"}
|
175
|
-
format_ext["17"] = {:extension => "3gp", :name => "3gp"}
|
176
|
-
|
177
|
-
#since 1.8 doesn't do ordered hashes
|
178
|
-
prefered_order = ["38","37","22","45","44","18","35","34","5","17"]
|
179
|
-
|
180
|
-
selected_format = prefered_order.select{|possible_format| available_formats.include?(possible_format)}.first
|
181
|
-
|
182
|
-
puts "[YOUTUBE] Title: #{title}"
|
183
|
-
puts "[YOUTUBE] Length: #{length_s} s"
|
184
|
-
puts "[YOUTUBE] t-parameter: #{token}"
|
185
|
-
#best quality seems always to be firsts
|
186
|
-
puts "[YOUTUBE] formats available: #{available_formats.inspect} (downloading format #{selected_format} -> #{format_ext[selected_format][:name]})"
|
187
184
|
|
188
|
-
|
185
|
+
selected_format = pick_video_format(fmt_list)
|
186
|
+
puts "(downloading format #{selected_format} -> #{VIDEO_FORMATS[selected_format][:name]})"
|
187
|
+
|
189
188
|
download_url = video_info_hash["url_encoded_fmt_stream_map"][selected_format]
|
190
189
|
|
191
190
|
#if download url ends with a ';' followed by a codec string remove that part because it stops URI.parse from working
|
@@ -199,8 +198,40 @@ class Youtube < PluginBase
|
|
199
198
|
download_url.sub!("&sig=", "&signature=") #else we just have to change sig to signature
|
200
199
|
end
|
201
200
|
|
202
|
-
file_name = PluginBase.make_filename_safe(title) + "." +
|
203
|
-
puts "downloading to " + file_name
|
201
|
+
file_name = PluginBase.make_filename_safe(title) + "." + VIDEO_FORMATS[selected_format][:extension]
|
202
|
+
puts "downloading to " + file_name + "\n\n"
|
204
203
|
{:url => download_url, :name => file_name}
|
205
204
|
end
|
205
|
+
|
206
|
+
#returns the format of the video the user picked or the first default format if it does not exist
|
207
|
+
def self.pick_video_format(fmt_list)
|
208
|
+
available_formats = fmt_list.map { |format| format.split("/").first }
|
209
|
+
notify "formats available: #{available_formats.inspect}"
|
210
|
+
|
211
|
+
if @quality #if the user specified a format
|
212
|
+
ext = @quality[:extension]
|
213
|
+
res = @quality[:resolution]
|
214
|
+
|
215
|
+
#gets a nested array with all the formats of the same res as the user wanted
|
216
|
+
requested = VIDEO_FORMATS.select { |id, format| format[:name].include?(res) }.to_a
|
217
|
+
|
218
|
+
if requested.empty?
|
219
|
+
notify "Requested format \"#{res}:#{ext}\" not found. Downloading default format."
|
220
|
+
get_default_format(available_formats)
|
221
|
+
else
|
222
|
+
pick = requested.find { |format| format[1][:extension] == ext } #get requsted extension if possible
|
223
|
+
pick ? pick.first : get_default_format(requested.map { |req| req.first }) #else return the default format
|
224
|
+
end
|
225
|
+
else
|
226
|
+
get_default_format(available_formats)
|
227
|
+
end
|
228
|
+
end
|
229
|
+
|
230
|
+
def self.get_default_format(available)
|
231
|
+
DEFAULT_FORMAT_ORDER.find { |default| available.include?(default) }
|
232
|
+
end
|
233
|
+
|
234
|
+
def self.notify(message)
|
235
|
+
puts "[YOUTUBE] #{message}"
|
236
|
+
end
|
206
237
|
end
|
metadata
CHANGED
@@ -1,12 +1,12 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: viddl-rb
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 147
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 0
|
8
|
-
-
|
9
|
-
version: "0.
|
8
|
+
- 76
|
9
|
+
version: "0.76"
|
10
10
|
platform: ruby
|
11
11
|
authors:
|
12
12
|
- Marc Seeger
|
@@ -14,8 +14,7 @@ autorequire:
|
|
14
14
|
bindir: bin
|
15
15
|
cert_chain: []
|
16
16
|
|
17
|
-
date:
|
18
|
-
default_executable:
|
17
|
+
date: 2013-01-30 00:00:00 Z
|
19
18
|
dependencies:
|
20
19
|
- !ruby/object:Gem::Dependency
|
21
20
|
name: nokogiri
|
@@ -46,7 +45,7 @@ dependencies:
|
|
46
45
|
type: :runtime
|
47
46
|
version_requirements: *id002
|
48
47
|
- !ruby/object:Gem::Dependency
|
49
|
-
name:
|
48
|
+
name: progressbar
|
50
49
|
prerelease: false
|
51
50
|
requirement: &id003 !ruby/object:Gem::Requirement
|
52
51
|
none: false
|
@@ -60,7 +59,7 @@ dependencies:
|
|
60
59
|
type: :runtime
|
61
60
|
version_requirements: *id003
|
62
61
|
- !ruby/object:Gem::Dependency
|
63
|
-
name:
|
62
|
+
name: rake
|
64
63
|
prerelease: false
|
65
64
|
requirement: &id004 !ruby/object:Gem::Requirement
|
66
65
|
none: false
|
@@ -71,10 +70,10 @@ dependencies:
|
|
71
70
|
segments:
|
72
71
|
- 0
|
73
72
|
version: "0"
|
74
|
-
type: :
|
73
|
+
type: :development
|
75
74
|
version_requirements: *id004
|
76
75
|
- !ruby/object:Gem::Dependency
|
77
|
-
name:
|
76
|
+
name: rest-client
|
78
77
|
prerelease: false
|
79
78
|
requirement: &id005 !ruby/object:Gem::Requirement
|
80
79
|
none: false
|
@@ -131,7 +130,6 @@ files:
|
|
131
130
|
- Rakefile
|
132
131
|
- README.md
|
133
132
|
- TODO.txt
|
134
|
-
has_rdoc: false
|
135
133
|
homepage: https://github.com/rb2k/viddl-rb
|
136
134
|
licenses: []
|
137
135
|
|
@@ -163,9 +161,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
163
161
|
requirements: []
|
164
162
|
|
165
163
|
rubyforge_project: viddl-rb
|
166
|
-
rubygems_version: 1.
|
164
|
+
rubygems_version: 1.8.24
|
167
165
|
signing_key:
|
168
166
|
specification_version: 3
|
169
167
|
summary: An extendable commandline video downloader for flash video sites.
|
170
168
|
test_files: []
|
171
169
|
|
170
|
+
has_rdoc: false
|