viddl-rb 0.7 → 0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile.lock +21 -19
- data/README.md +23 -11
- data/Rakefile +5 -1
- data/bin/helper/downloader.rb +2 -7
- data/bin/helper/parameter-parser.rb +93 -51
- data/bin/viddl-rb +10 -4
- data/helper/audio-helper.rb +2 -3
- data/helper/download-helper.rb +69 -61
- data/helper/utility-helper.rb +31 -2
- data/lib/viddl-rb.rb +1 -11
- data/plugins/soundcloud.rb +18 -15
- data/plugins/youtube.rb +271 -196
- metadata +25 -27
- data/CHANGELOG.txt +0 -14
data/helper/utility-helper.rb
CHANGED
@@ -10,7 +10,36 @@ module ViddlRb
|
|
10
10
|
ViddlRb.class_eval(File.read(plugin))
|
11
11
|
end
|
12
12
|
end
|
13
|
-
end
|
14
13
|
|
15
|
-
|
14
|
+
#checks to see whether the os has a certain utility like wget or curl
|
15
|
+
#`` returns the standard output of the process
|
16
|
+
#system returns the exit code of the process
|
17
|
+
def self.os_has?(utility)
|
18
|
+
windows = ENV['OS'] =~ /windows/i
|
19
|
+
|
20
|
+
unless windows
|
21
|
+
`which #{utility}`.include?(utility.to_s)
|
22
|
+
else
|
23
|
+
if !system("where /q where").nil? #if Windows has the where utility
|
24
|
+
system("where /q #{utility}") #/q is the quiet mode flag
|
25
|
+
else
|
26
|
+
begin #as a fallback we just run the utility itself
|
27
|
+
system(utility)
|
28
|
+
rescue Errno::ENOENT
|
29
|
+
false
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
16
34
|
|
35
|
+
#recursively get the final location (after following all redirects) for an url.
|
36
|
+
def self.get_final_location(url)
|
37
|
+
Net::HTTP.get_response(URI(url)) do |res|
|
38
|
+
location = res["location"]
|
39
|
+
return url if location.nil?
|
40
|
+
return get_final_location(location)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
45
|
+
end
|
data/lib/viddl-rb.rb
CHANGED
@@ -95,19 +95,9 @@ module ViddlRb
|
|
95
95
|
def self.follow_all_redirects(urls_filenames)
|
96
96
|
urls_filenames.map do |uf|
|
97
97
|
url = uf[:url]
|
98
|
-
final_location = get_final_location(url)
|
98
|
+
final_location = UtilityHelper.get_final_location(url)
|
99
99
|
{:url => final_location, :name => uf[:name]}
|
100
100
|
end
|
101
101
|
end
|
102
102
|
private_class_method :follow_all_redirects
|
103
|
-
|
104
|
-
#recursively get the final location (after following all redirects) for an url.
|
105
|
-
def self.get_final_location(url)
|
106
|
-
Net::HTTP.get_response(URI(url)) do |res|
|
107
|
-
location = res["location"]
|
108
|
-
return url if location.nil?
|
109
|
-
return get_final_location(location)
|
110
|
-
end
|
111
|
-
end
|
112
|
-
private_class_method :get_final_location
|
113
103
|
end
|
data/plugins/soundcloud.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
require '
|
1
|
+
require 'open-uri'
|
2
2
|
class Soundcloud < PluginBase
|
3
3
|
# this will be called by the main app to check whether this plugin is responsible for the url passed
|
4
4
|
def self.matches_provider?(url)
|
@@ -7,7 +7,7 @@ class Soundcloud < PluginBase
|
|
7
7
|
|
8
8
|
# return the url for original video file and title
|
9
9
|
def self.get_urls_and_filenames(url, options = {})
|
10
|
-
doc = Nokogiri::HTML(
|
10
|
+
doc = Nokogiri::HTML(open(get_http_url(url)))
|
11
11
|
download_filename = doc.at("#main-content-inner img[class=waveform]").attributes["src"].value.to_s.match(/\.com\/(.+)\_/)[1]
|
12
12
|
download_url = "http://media.soundcloud.com/stream/#{download_filename}"
|
13
13
|
file_name = transliterate("#{doc.at('//h1/em').text.chomp}") + ".mp3"
|
@@ -16,24 +16,27 @@ class Soundcloud < PluginBase
|
|
16
16
|
end
|
17
17
|
|
18
18
|
def self.transliterate(str)
|
19
|
-
|
19
|
+
# Based on permalink_fu by Rick Olsen
|
20
20
|
|
21
|
-
|
22
|
-
|
21
|
+
# Downcase string
|
22
|
+
str.downcase!
|
23
23
|
|
24
|
-
|
25
|
-
|
24
|
+
# Remove apostrophes so isn't changes to isnt
|
25
|
+
str.gsub!(/'/, '')
|
26
26
|
|
27
|
-
|
28
|
-
|
27
|
+
# Replace any non-letter or non-number character with a space
|
28
|
+
str.gsub!(/[^A-Za-z0-9]+/, ' ')
|
29
29
|
|
30
|
-
|
31
|
-
|
30
|
+
# Remove spaces from beginning and end of string
|
31
|
+
str.strip!
|
32
32
|
|
33
|
-
|
34
|
-
|
33
|
+
# Replace groups of spaces with single hyphen
|
34
|
+
str.gsub!(/\ +/, '-')
|
35
35
|
|
36
|
-
|
37
|
-
end
|
36
|
+
str
|
37
|
+
end
|
38
38
|
|
39
|
+
def self.get_http_url(url)
|
40
|
+
url.sub(/https?:\/\//, "http:\/\/")
|
41
|
+
end
|
39
42
|
end
|
data/plugins/youtube.rb
CHANGED
@@ -1,196 +1,271 @@
|
|
1
|
-
|
2
|
-
class Youtube < PluginBase
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
end
|
44
|
-
|
45
|
-
def self.
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
#
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
if
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
1
|
+
|
2
|
+
class Youtube < PluginBase
|
3
|
+
|
4
|
+
# see http://en.wikipedia.org/wiki/YouTube#Quality_and_codecs
|
5
|
+
# TODO: we don't have all the formats from the wiki article here
|
6
|
+
VIDEO_FORMATS = {
|
7
|
+
"38" => {:extension => "mp4", :name => "MP4 Highest Quality 4096x3027 (H.264, AAC)"},
|
8
|
+
"37" => {:extension => "mp4", :name => "MP4 Highest Quality 1920x1080 (H.264, AAC)"},
|
9
|
+
"22" => {:extension => "mp4", :name => "MP4 1280x720 (H.264, AAC)"},
|
10
|
+
"46" => {:extension => "webm", :name => "WebM 1920x1080 (VP8, Vorbis)"},
|
11
|
+
"45" => {:extension => "webm", :name => "WebM 1280x720 (VP8, Vorbis)"},
|
12
|
+
"44" => {:extension => "webm", :name => "WebM 854x480 (VP8, Vorbis)"},
|
13
|
+
"43" => {:extension => "webm", :name => "WebM 480×360 (VP8, Vorbis)"},
|
14
|
+
"18" => {:extension => "mp4", :name => "MP4 640x360 (H.264, AAC)"},
|
15
|
+
"35" => {:extension => "flv", :name => "FLV 854x480 (H.264, AAC)"},
|
16
|
+
"34" => {:extension => "flv", :name => "FLV 640x360 (H.264, AAC)"},
|
17
|
+
"5" => {:extension => "flv", :name => "FLV 400x240 (Soerenson H.263)"},
|
18
|
+
"17" => {:extension => "3gp", :name => "3gp"}
|
19
|
+
}
|
20
|
+
|
21
|
+
DEFAULT_FORMAT_ORDER = %w[38 37 22 46 45 44 43 18 35 34 5 17]
|
22
|
+
VIDEO_INFO_URL = "http://www.youtube.com/get_video_info?video_id="
|
23
|
+
VIDEO_INFO_PARMS = "&ps=default&eurl=&gl=US&hl=en"
|
24
|
+
|
25
|
+
# this will be called by the main app to check whether this plugin is responsible for the url passed
|
26
|
+
def self.matches_provider?(url)
|
27
|
+
url.include?("youtube.com") || url.include?("youtu.be")
|
28
|
+
end
|
29
|
+
|
30
|
+
def self.get_urls_and_filenames(url, options = {})
|
31
|
+
@quality = options[:quality]
|
32
|
+
filter = options[:playlist_filter]
|
33
|
+
parser = PlaylistParser.new
|
34
|
+
return_vals = []
|
35
|
+
|
36
|
+
if playlist_urls = parser.get_playlist_urls(url, filter)
|
37
|
+
playlist_urls.each { |url| return_vals << grab_single_url_filename(url) }
|
38
|
+
else
|
39
|
+
return_vals << grab_single_url_filename(url)
|
40
|
+
end
|
41
|
+
|
42
|
+
clean_return_values(return_vals)
|
43
|
+
end
|
44
|
+
|
45
|
+
def self.clean_return_values(return_values)
|
46
|
+
cleaned = return_values.reject { |val| val == :no_embed }
|
47
|
+
|
48
|
+
if cleaned.empty?
|
49
|
+
download_error("No videos could be downloaded.")
|
50
|
+
else
|
51
|
+
cleaned
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def self.grab_single_url_filename(url)
|
56
|
+
grab_url_embeddable(url) || grab_url_non_embeddable(url)
|
57
|
+
end
|
58
|
+
|
59
|
+
def self.grab_url_embeddable(url)
|
60
|
+
video_info = get_video_info(url)
|
61
|
+
video_params = extract_video_parameters(video_info)
|
62
|
+
unless video_params[:embeddable]
|
63
|
+
notify("VIDEO IS NOT EMBEDDABLE")
|
64
|
+
return false
|
65
|
+
end
|
66
|
+
|
67
|
+
urls_formats = extract_urls_formats(video_info)
|
68
|
+
selected_format = choose_format(urls_formats)
|
69
|
+
title = video_params[:title]
|
70
|
+
file_name = PluginBase.make_filename_safe(title) + "." + VIDEO_FORMATS[selected_format][:extension]
|
71
|
+
|
72
|
+
{:url => urls_formats[selected_format], :name => file_name}
|
73
|
+
end
|
74
|
+
|
75
|
+
def self.grab_url_non_embeddable(url)
|
76
|
+
video_info = open(url).read
|
77
|
+
stream_map = video_info[/url_encoded_fmt_stream_map\" *: *\"([^\"]+)\"/,1]
|
78
|
+
urls_formats = parse_stream_map(url_decode(stream_map))
|
79
|
+
selected_format = choose_format(urls_formats)
|
80
|
+
title = video_info[/<meta name="title" content="([^"]*)">/, 1]
|
81
|
+
file_name = PluginBase.make_filename_safe(title) + "." + VIDEO_FORMATS[selected_format][:extension]
|
82
|
+
|
83
|
+
# cleaning
|
84
|
+
clean_url = urls_formats[selected_format].gsub(/\\u0026[^&]*/, "")
|
85
|
+
|
86
|
+
{:url => clean_url, :name => file_name}
|
87
|
+
end
|
88
|
+
|
89
|
+
def self.get_video_info(url)
|
90
|
+
id = extract_video_id(url)
|
91
|
+
request_url = VIDEO_INFO_URL + id + VIDEO_INFO_PARMS
|
92
|
+
open(request_url).read
|
93
|
+
end
|
94
|
+
|
95
|
+
def self.extract_video_id(url)
|
96
|
+
# the youtube video ID looks like this: [...]v=abc5a5_afe5agae6g&[...], we only want the ID (the \w in the brackets)
|
97
|
+
# addition: might also look like this /v/abc5-a5afe5agae6g
|
98
|
+
# alternative: video_id = url[/v[\/=]([\w-]*)&?/, 1]
|
99
|
+
url = open(url).base_uri.to_s if url.include?("youtu.be")
|
100
|
+
video_id = url[/(v|embed)[=\/]([^\/\?\&]*)/, 2]
|
101
|
+
|
102
|
+
if video_id
|
103
|
+
notify("ID FOUND: #{video_id}")
|
104
|
+
video_id
|
105
|
+
else
|
106
|
+
download_error("No video id found.")
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
def self.extract_video_parameters(video_info)
|
111
|
+
decoded = url_decode(video_info)
|
112
|
+
|
113
|
+
{:title => decoded[/title=(.+?)(?:&|$)/, 1],
|
114
|
+
:length_sec => decoded[/length_seconds=(.+?)(?:&|$)/, 1],
|
115
|
+
:author => decoded[/author=(.+?)(?:&|$)/, 1],
|
116
|
+
:embeddable => !decoded.include?("status=fail")}
|
117
|
+
end
|
118
|
+
|
119
|
+
def self.extract_urls_formats(video_info)
|
120
|
+
stream_map = video_info[/url_encoded_fmt_stream_map=(.+?)(?:&|$)/, 1]
|
121
|
+
parse_stream_map(stream_map)
|
122
|
+
end
|
123
|
+
|
124
|
+
def self.parse_stream_map(stream_map)
|
125
|
+
urls = extract_download_urls(stream_map)
|
126
|
+
formats_urls = {}
|
127
|
+
|
128
|
+
urls.each do |url|
|
129
|
+
format = url[/itag=(\d+)/, 1]
|
130
|
+
formats_urls[format] = url
|
131
|
+
end
|
132
|
+
|
133
|
+
formats_urls
|
134
|
+
end
|
135
|
+
|
136
|
+
def self.extract_download_urls(stream_map)
|
137
|
+
entries = stream_map.split("%2C")
|
138
|
+
decoded = entries.map { |entry| url_decode(entry) }
|
139
|
+
|
140
|
+
decoded.map do |entry|
|
141
|
+
url = entry[/url=(.*?itag=.+?)(?:itag=|;|$)/, 1]
|
142
|
+
sig = entry[/sig=(.+?)(?:&|$)/, 1]
|
143
|
+
|
144
|
+
url + "&signature=#{sig}"
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
def self.choose_format(urls_formats)
|
149
|
+
available_formats = urls_formats.keys
|
150
|
+
|
151
|
+
if @quality #if the user specified a format
|
152
|
+
ext = @quality[:extension]
|
153
|
+
res = @quality[:resolution]
|
154
|
+
#gets a nested array with all the formats of the same res as the user wanted
|
155
|
+
requested = VIDEO_FORMATS.select { |id, format| format[:name].include?(res) }.to_a
|
156
|
+
|
157
|
+
if requested.empty?
|
158
|
+
notify "Requested format \"#{res}:#{ext}\" not found. Downloading default format."
|
159
|
+
get_default_format(available_formats)
|
160
|
+
else
|
161
|
+
pick = requested.find { |format| format[1][:extension] == ext } # get requsted extension if possible
|
162
|
+
pick ? pick.first : get_default_format(requested.map { |req| req.first }) # else return the default format
|
163
|
+
end
|
164
|
+
else
|
165
|
+
get_default_format(available_formats)
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
def self.get_default_format(available)
|
170
|
+
DEFAULT_FORMAT_ORDER.find { |default| available.include?(default) }
|
171
|
+
end
|
172
|
+
|
173
|
+
def self.url_decode(text)
|
174
|
+
while text != (decoded = CGI::unescape(text)) do
|
175
|
+
text = decoded
|
176
|
+
end
|
177
|
+
text
|
178
|
+
end
|
179
|
+
|
180
|
+
def self.notify(message)
|
181
|
+
puts "[YOUTUBE] #{message}"
|
182
|
+
end
|
183
|
+
|
184
|
+
def self.download_error(message)
|
185
|
+
raise CouldNotDownloadVideoError, message
|
186
|
+
end
|
187
|
+
|
188
|
+
#
|
189
|
+
# class PlaylistParser
|
190
|
+
#_____________________
|
191
|
+
|
192
|
+
class PlaylistParser
|
193
|
+
|
194
|
+
PLAYLIST_FEED = "http://gdata.youtube.com/feeds/api/playlists/%s?&max-results=50&v=2"
|
195
|
+
USER_FEED = "http://gdata.youtube.com/feeds/api/users/%s/uploads?&max-results=50&v=2"
|
196
|
+
|
197
|
+
def get_playlist_urls(url, filter = nil)
|
198
|
+
@filter = filter
|
199
|
+
|
200
|
+
if url.include?("view_play_list") || url.include?("playlist?list=") # if playlist URL
|
201
|
+
parse_playlist(url)
|
202
|
+
elsif username = url[/\/user\/([\w\d]+)(?:\/|$)/, 1] # if user URL
|
203
|
+
parse_user(username)
|
204
|
+
else # if neither return nil
|
205
|
+
nil
|
206
|
+
end
|
207
|
+
end
|
208
|
+
|
209
|
+
def parse_playlist(url)
|
210
|
+
#http://www.youtube.com/view_play_list?p=F96B063007B44E1E&search_query=welt+auf+schwäbisch
|
211
|
+
#http://www.youtube.com/watch?v=9WEP5nCxkEY&videos=jKY836_WMhE&playnext_from=TL&playnext=1
|
212
|
+
#http://www.youtube.com/watch?v=Tk78sr5JMIU&videos=jKY836_WMhE
|
213
|
+
|
214
|
+
playlist_ID = url[/(?:list=PL|p=)(.+?)(?:&|\/|$)/, 1]
|
215
|
+
notify "Playlist ID: #{playlist_ID}"
|
216
|
+
feed_url = PLAYLIST_FEED % playlist_ID
|
217
|
+
url_array = get_video_urls(feed_url)
|
218
|
+
notify "#{url_array.size} links found!"
|
219
|
+
url_array
|
220
|
+
end
|
221
|
+
|
222
|
+
def parse_user(username)
|
223
|
+
notify "User: #{username}"
|
224
|
+
feed_url = USER_FEED % username
|
225
|
+
url_array = get_video_urls(feed_url)
|
226
|
+
notify "#{url_array.size} links found!"
|
227
|
+
url_array
|
228
|
+
end
|
229
|
+
|
230
|
+
#get all videos and return their urls in an array
|
231
|
+
def get_video_urls(feed_url)
|
232
|
+
notify "Retrieving videos..."
|
233
|
+
urls_titles = {}
|
234
|
+
result_feed = Nokogiri::XML(open(feed_url))
|
235
|
+
urls_titles.merge!(grab_urls_and_titles(result_feed))
|
236
|
+
|
237
|
+
#as long as the feed has a next link we follow it and add the resulting video urls
|
238
|
+
loop do
|
239
|
+
next_link = result_feed.search("//feed/link[@rel='next']").first
|
240
|
+
break if next_link.nil?
|
241
|
+
result_feed = Nokogiri::HTML(open(next_link["href"]))
|
242
|
+
urls_titles.merge!(grab_urls_and_titles(result_feed))
|
243
|
+
end
|
244
|
+
|
245
|
+
filter_urls(urls_titles)
|
246
|
+
end
|
247
|
+
|
248
|
+
#extract all video urls and their titles from a feed and return in a hash
|
249
|
+
def grab_urls_and_titles(feed)
|
250
|
+
feed.remove_namespaces! #so that we can get to the titles easily
|
251
|
+
urls = feed.search("//entry/link[@rel='alternate']").map { |link| link["href"] }
|
252
|
+
titles = feed.search("//entry/group/title").map { |title| title.text }
|
253
|
+
Hash[urls.zip(titles)] #hash like this: url => title
|
254
|
+
end
|
255
|
+
|
256
|
+
#returns only the urls that match the --filter argument regex (if present)
|
257
|
+
def filter_urls(url_hash)
|
258
|
+
if @filter
|
259
|
+
notify "Using filter: #{@filter}"
|
260
|
+
filtered = url_hash.select { |url, title| title =~ @filter }
|
261
|
+
filtered.keys
|
262
|
+
else
|
263
|
+
url_hash.keys
|
264
|
+
end
|
265
|
+
end
|
266
|
+
|
267
|
+
def notify(message)
|
268
|
+
Youtube.notify(message)
|
269
|
+
end
|
270
|
+
end
|
271
|
+
end
|