rubytube 0.3.0 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rspec +3 -3
- data/.standard.yml +3 -3
- data/CHANGELOG.md +5 -5
- data/CODE_OF_CONDUCT.md +84 -84
- data/Gemfile +12 -12
- data/LICENSE.txt +21 -21
- data/README.md +28 -19
- data/Rakefile +10 -10
- data/lib/rubytube/cipher.rb +370 -371
- data/lib/rubytube/client.rb +173 -173
- data/lib/rubytube/error.rb +49 -0
- data/lib/rubytube/extractor.rb +171 -177
- data/lib/rubytube/innertube.rb +105 -105
- data/lib/rubytube/monostate.rb +5 -5
- data/lib/rubytube/parser.rb +159 -164
- data/lib/rubytube/request.rb +73 -75
- data/lib/rubytube/stream.rb +95 -97
- data/lib/rubytube/stream_format.rb +152 -152
- data/lib/rubytube/stream_query.rb +61 -36
- data/lib/rubytube/utils.rb +24 -24
- data/lib/rubytube/version.rb +5 -5
- data/lib/rubytube.rb +27 -68
- data/sig/rubytube.rbs +4 -4
- metadata +7 -6
data/lib/rubytube/extractor.rb
CHANGED
@@ -1,177 +1,171 @@
|
|
1
|
-
module RubyTube
|
2
|
-
class Extractor
|
3
|
-
class << self
|
4
|
-
def playability_status(watch_html)
|
5
|
-
player_response = initial_player_response(watch_html)
|
6
|
-
player_response = JSON.parse(player_response)
|
7
|
-
status_obj = player_response[
|
8
|
-
|
9
|
-
if status_obj.has_key?(
|
10
|
-
return [
|
11
|
-
end
|
12
|
-
|
13
|
-
if status_obj.has_key?(
|
14
|
-
if status_obj.has_key?(
|
15
|
-
return [status_obj[
|
16
|
-
end
|
17
|
-
|
18
|
-
if status_obj.has_key?(
|
19
|
-
return [status_obj[
|
20
|
-
end
|
21
|
-
end
|
22
|
-
|
23
|
-
[nil, [nil]]
|
24
|
-
end
|
25
|
-
|
26
|
-
def video_id(url)
|
27
|
-
|
28
|
-
end
|
29
|
-
|
30
|
-
def js_url(html)
|
31
|
-
begin
|
32
|
-
base_js = get_ytplayer_config(html)[
|
33
|
-
rescue RegexMatchError, NoMethodError
|
34
|
-
base_js = get_ytplayer_js(html)
|
35
|
-
end
|
36
|
-
|
37
|
-
"https://youtube.com#{base_js}"
|
38
|
-
end
|
39
|
-
|
40
|
-
def mime_type_codec(mime_type_codec)
|
41
|
-
pattern = %r{(\w
|
42
|
-
results = mime_type_codec.match(pattern)
|
43
|
-
|
44
|
-
raise RegexMatchError.new("mime_type_codec, pattern=#{pattern}") if results.nil?
|
45
|
-
|
46
|
-
mime_type, codecs = results.captures
|
47
|
-
[mime_type, codecs.split(",").map(&:strip)]
|
48
|
-
end
|
49
|
-
|
50
|
-
def get_ytplayer_js(html)
|
51
|
-
js_url_patterns = [
|
52
|
-
%r{(/s/player/[\w\d]+/[\w\d_/.]+/base\.js)}
|
53
|
-
]
|
54
|
-
|
55
|
-
js_url_patterns.each do |pattern|
|
56
|
-
function_match = html.match(pattern)
|
57
|
-
if function_match
|
58
|
-
return function_match[1]
|
59
|
-
end
|
60
|
-
end
|
61
|
-
|
62
|
-
raise RegexMatchError.new(
|
63
|
-
end
|
64
|
-
|
65
|
-
def get_ytplayer_config(html)
|
66
|
-
config_patterns = [
|
67
|
-
/ytplayer\.config\s*=\s*/,
|
68
|
-
/ytInitialPlayerResponse\s*=\s*/
|
69
|
-
]
|
70
|
-
|
71
|
-
config_patterns.each do |pattern|
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
formats
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
raise RegexMatchError.new('initial_player_response', 'initial_player_response_pattern')
|
174
|
-
end
|
175
|
-
end
|
176
|
-
end
|
177
|
-
end
|
1
|
+
module RubyTube
|
2
|
+
class Extractor
|
3
|
+
class << self
|
4
|
+
def playability_status(watch_html)
|
5
|
+
player_response = initial_player_response(watch_html)
|
6
|
+
player_response = JSON.parse(player_response)
|
7
|
+
status_obj = player_response["playabilityStatus"] || {}
|
8
|
+
|
9
|
+
if status_obj.has_key?("liveStreamability")
|
10
|
+
return ["LIVE_STREAM", "Video is a live stream."]
|
11
|
+
end
|
12
|
+
|
13
|
+
if status_obj.has_key?("status")
|
14
|
+
if status_obj.has_key?("reason")
|
15
|
+
return [status_obj["status"], [status_obj["reason"]]]
|
16
|
+
end
|
17
|
+
|
18
|
+
if status_obj.has_key?("messages")
|
19
|
+
return [status_obj["status"], status_obj["messages"]]
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
[nil, [nil]]
|
24
|
+
end
|
25
|
+
|
26
|
+
def video_id(url)
|
27
|
+
Utils.regex_search(/(?:v=|\/)([0-9A-Za-z_-]{11}).*/, url, 1)
|
28
|
+
end
|
29
|
+
|
30
|
+
def js_url(html)
|
31
|
+
begin
|
32
|
+
base_js = get_ytplayer_config(html)["assets"]["js"]
|
33
|
+
rescue RegexMatchError, NoMethodError
|
34
|
+
base_js = get_ytplayer_js(html)
|
35
|
+
end
|
36
|
+
|
37
|
+
"https://youtube.com#{base_js}"
|
38
|
+
end
|
39
|
+
|
40
|
+
def mime_type_codec(mime_type_codec)
|
41
|
+
pattern = %r{(\w+/\w+);\scodecs="([a-zA-Z\-0-9.,\s]*)"}
|
42
|
+
results = mime_type_codec.match(pattern)
|
43
|
+
|
44
|
+
raise RegexMatchError.new("mime_type_codec, pattern=#{pattern}") if results.nil?
|
45
|
+
|
46
|
+
mime_type, codecs = results.captures
|
47
|
+
[mime_type, codecs.split(",").map(&:strip)]
|
48
|
+
end
|
49
|
+
|
50
|
+
def get_ytplayer_js(html)
|
51
|
+
js_url_patterns = [
|
52
|
+
%r{(/s/player/[\w\d]+/[\w\d_/.]+/base\.js)}
|
53
|
+
]
|
54
|
+
|
55
|
+
js_url_patterns.each do |pattern|
|
56
|
+
function_match = html.match(pattern)
|
57
|
+
if function_match
|
58
|
+
return function_match[1]
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
raise RegexMatchError.new("get_ytplayer_js", "js_url_patterns")
|
63
|
+
end
|
64
|
+
|
65
|
+
def get_ytplayer_config(html)
|
66
|
+
config_patterns = [
|
67
|
+
/ytplayer\.config\s*=\s*/,
|
68
|
+
/ytInitialPlayerResponse\s*=\s*/
|
69
|
+
]
|
70
|
+
|
71
|
+
config_patterns.each do |pattern|
|
72
|
+
return Parser.parse_for_object(html, pattern)
|
73
|
+
rescue HTMLParseError => e
|
74
|
+
next
|
75
|
+
end
|
76
|
+
|
77
|
+
setconfig_patterns = [
|
78
|
+
/yt\.setConfig\(.*['\"]PLAYER_CONFIG['\"]:\s*/
|
79
|
+
]
|
80
|
+
|
81
|
+
setconfig_patterns.each do |pattern|
|
82
|
+
return Parser.parse_for_object(html, pattern)
|
83
|
+
rescue HTMLParseError => e
|
84
|
+
next
|
85
|
+
end
|
86
|
+
|
87
|
+
raise RegexMatchError.new("get_ytplayer_config", "config_patterns, setconfig_patterns")
|
88
|
+
end
|
89
|
+
|
90
|
+
def apply_signature(stream_manifest, vid_info, js)
|
91
|
+
cipher = Cipher.new(js)
|
92
|
+
|
93
|
+
stream_manifest.each_with_index do |stream, i|
|
94
|
+
begin
|
95
|
+
url = stream["url"]
|
96
|
+
rescue NoMethodError
|
97
|
+
live_stream = vid_info.fetch("playabilityStatus", {})["liveStreamability"]
|
98
|
+
if live_stream
|
99
|
+
raise LiveStreamError.new("UNKNOWN")
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
if url.include?("signature") ||
|
104
|
+
(!stream.key?("s") && (url.include?("&sig=") || url.include?("&lsig=")))
|
105
|
+
# For certain videos, YouTube will just provide them pre-signed, in
|
106
|
+
# which case there's no real magic to download them and we can skip
|
107
|
+
# the whole signature descrambling entirely.
|
108
|
+
next
|
109
|
+
end
|
110
|
+
|
111
|
+
signature = cipher.get_signature(stream["s"])
|
112
|
+
|
113
|
+
parsed_url = URI.parse(url)
|
114
|
+
|
115
|
+
query_params = CGI.parse(parsed_url.query)
|
116
|
+
query_params.transform_values!(&:first)
|
117
|
+
query_params["sig"] = signature
|
118
|
+
unless query_params.key?("ratebypass")
|
119
|
+
initial_n = query_params["n"].chars
|
120
|
+
new_n = cipher.calculate_n(initial_n)
|
121
|
+
query_params["n"] = new_n
|
122
|
+
end
|
123
|
+
|
124
|
+
url = "#{parsed_url.scheme}://#{parsed_url.host}#{parsed_url.path}?#{URI.encode_www_form(query_params)}"
|
125
|
+
|
126
|
+
stream_manifest[i]["url"] = url
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
def apply_descrambler(stream_data)
|
131
|
+
return if stream_data.has_key?("url")
|
132
|
+
|
133
|
+
# Merge formats and adaptiveFormats into a single array
|
134
|
+
formats = []
|
135
|
+
formats += stream_data["formats"] if stream_data.has_key?("formats")
|
136
|
+
formats += stream_data["adaptiveFormats"] if stream_data.has_key?("adaptiveFormats")
|
137
|
+
|
138
|
+
# Extract url and s from signatureCiphers as necessary
|
139
|
+
formats.each do |data|
|
140
|
+
unless data.has_key?("url")
|
141
|
+
if data.has_key?("signatureCipher")
|
142
|
+
cipher_url = URI.decode_www_form(data["signatureCipher"]).to_h
|
143
|
+
data["url"] = cipher_url["url"]
|
144
|
+
data["s"] = cipher_url["s"]
|
145
|
+
end
|
146
|
+
end
|
147
|
+
data["is_otf"] = data["type"] == "FORMAT_STREAM_TYPE_OTF"
|
148
|
+
end
|
149
|
+
|
150
|
+
formats
|
151
|
+
end
|
152
|
+
|
153
|
+
private
|
154
|
+
|
155
|
+
def initial_player_response(watch_html)
|
156
|
+
patterns = [
|
157
|
+
"window\\[['\"]ytInitialPlayerResponse['\"]\\]\\s*=\\s*",
|
158
|
+
"ytInitialPlayerResponse\\s*=\\s*"
|
159
|
+
]
|
160
|
+
|
161
|
+
patterns.each do |pattern|
|
162
|
+
return Parser.parse_for_object(watch_html, pattern)
|
163
|
+
rescue HTMLParseError
|
164
|
+
next
|
165
|
+
end
|
166
|
+
|
167
|
+
raise RegexMatchError.new("initial_player_response", "initial_player_response_pattern")
|
168
|
+
end
|
169
|
+
end
|
170
|
+
end
|
171
|
+
end
|
data/lib/rubytube/innertube.rb
CHANGED
@@ -1,105 +1,105 @@
|
|
1
|
-
module RubyTube
|
2
|
-
class InnerTube
|
3
|
-
DEFALUT_CLIENTS = {
|
4
|
-
|
5
|
-
context: {
|
6
|
-
client: {
|
7
|
-
clientName:
|
8
|
-
clientVersion:
|
9
|
-
}
|
10
|
-
},
|
11
|
-
header: {
|
12
|
-
api_key:
|
13
|
-
},
|
14
|
-
|
15
|
-
context: {
|
16
|
-
client: {
|
17
|
-
clientName:
|
18
|
-
clientVersion:
|
19
|
-
androidSdkVersion: 30
|
20
|
-
}
|
21
|
-
},
|
22
|
-
header: {
|
23
|
-
api_key:
|
24
|
-
},
|
25
|
-
|
26
|
-
context: {
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
},
|
34
|
-
header: {
|
35
|
-
api_key:
|
36
|
-
}
|
37
|
-
}
|
38
|
-
|
39
|
-
BASE_URL =
|
40
|
-
|
41
|
-
attr_accessor :context, :header, :api_key, :access_token, :refresh_token, :use_oauth, :allow_cache, :expires
|
42
|
-
|
43
|
-
def initialize(client:
|
44
|
-
self.context = DEFALUT_CLIENTS[client][:context]
|
45
|
-
self.header
|
46
|
-
self.api_key = DEFALUT_CLIENTS[client][:api_key]
|
47
|
-
self.use_oauth = use_oauth
|
48
|
-
self.allow_cache = allow_cache
|
49
|
-
end
|
50
|
-
|
51
|
-
def cache_tokens
|
52
|
-
|
53
|
-
|
54
|
-
# TODO:
|
55
|
-
end
|
56
|
-
|
57
|
-
def refresh_bearer_token(force: false)
|
58
|
-
# TODO:
|
59
|
-
end
|
60
|
-
|
61
|
-
def fetch_bearer_token
|
62
|
-
# TODO:
|
63
|
-
end
|
64
|
-
|
65
|
-
def send(endpoint, query, data)
|
66
|
-
if use_oauth
|
67
|
-
query.delete(:key)
|
68
|
-
end
|
69
|
-
|
70
|
-
headers = {
|
71
|
-
|
72
|
-
}
|
73
|
-
|
74
|
-
if use_oauth
|
75
|
-
if access_token
|
76
|
-
refresh_bearer_token
|
77
|
-
headers[
|
78
|
-
else
|
79
|
-
fetch_bearer_token
|
80
|
-
headers[
|
81
|
-
end
|
82
|
-
end
|
83
|
-
|
84
|
-
options = {}
|
85
|
-
options[:headers] = headers.merge(header)
|
86
|
-
|
87
|
-
options[:query] = {
|
88
|
-
key: api_key,
|
89
|
-
contentCheckOk: true,
|
90
|
-
racyCheckOk: true
|
91
|
-
}.merge(query)
|
92
|
-
options[:data] = data
|
93
|
-
|
94
|
-
resp = Request.post(endpoint, options)
|
95
|
-
JSON.parse(resp)
|
96
|
-
end
|
97
|
-
|
98
|
-
def player(video_id)
|
99
|
-
endpoint = "#{BASE_URL}/player"
|
100
|
-
query = {
|
101
|
-
|
102
|
-
send(endpoint, query, {context: context})
|
103
|
-
end
|
104
|
-
end
|
105
|
-
end
|
1
|
+
module RubyTube
|
2
|
+
class InnerTube
|
3
|
+
DEFALUT_CLIENTS = {
|
4
|
+
"WEB" => {
|
5
|
+
context: {
|
6
|
+
client: {
|
7
|
+
clientName: "WEB",
|
8
|
+
clientVersion: "2.20200720.00.02"
|
9
|
+
}
|
10
|
+
},
|
11
|
+
header: {"User-Agent": "Mozilla/5.0"},
|
12
|
+
api_key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
|
13
|
+
},
|
14
|
+
"ANDROID_MUSIC" => {
|
15
|
+
context: {
|
16
|
+
client: {
|
17
|
+
clientName: "ANDROID_MUSIC",
|
18
|
+
clientVersion: "5.16.51",
|
19
|
+
androidSdkVersion: 30
|
20
|
+
}
|
21
|
+
},
|
22
|
+
header: {"User-Agent": "com.google.android.apps.youtube.music/"},
|
23
|
+
api_key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
|
24
|
+
},
|
25
|
+
"ANDROID_EMBED" => {
|
26
|
+
context: {
|
27
|
+
client: {
|
28
|
+
clientName: "ANDROID_EMBEDDED_PLAYER",
|
29
|
+
clientVersion: "17.31.35",
|
30
|
+
clientScreen: "EMBED",
|
31
|
+
androidSdkVersion: 30
|
32
|
+
}
|
33
|
+
},
|
34
|
+
header: {"User-Agent": "com.google.android.youtube/"},
|
35
|
+
api_key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
|
36
|
+
}
|
37
|
+
}
|
38
|
+
|
39
|
+
BASE_URL = "https://www.youtube.com/youtubei/v1"
|
40
|
+
|
41
|
+
attr_accessor :context, :header, :api_key, :access_token, :refresh_token, :use_oauth, :allow_cache, :expires
|
42
|
+
|
43
|
+
def initialize(client: "WEB", use_oauth: false, allow_cache: false)
|
44
|
+
self.context = DEFALUT_CLIENTS[client][:context]
|
45
|
+
self.header = DEFALUT_CLIENTS[client][:header]
|
46
|
+
self.api_key = DEFALUT_CLIENTS[client][:api_key]
|
47
|
+
self.use_oauth = use_oauth
|
48
|
+
self.allow_cache = allow_cache
|
49
|
+
end
|
50
|
+
|
51
|
+
def cache_tokens
|
52
|
+
nil unless allow_cache
|
53
|
+
|
54
|
+
# TODO:
|
55
|
+
end
|
56
|
+
|
57
|
+
def refresh_bearer_token(force: false)
|
58
|
+
# TODO:
|
59
|
+
end
|
60
|
+
|
61
|
+
def fetch_bearer_token
|
62
|
+
# TODO:
|
63
|
+
end
|
64
|
+
|
65
|
+
def send(endpoint, query, data)
|
66
|
+
if use_oauth
|
67
|
+
query.delete(:key)
|
68
|
+
end
|
69
|
+
|
70
|
+
headers = {
|
71
|
+
"Content-Type": "application/json"
|
72
|
+
}
|
73
|
+
|
74
|
+
if use_oauth
|
75
|
+
if access_token
|
76
|
+
refresh_bearer_token
|
77
|
+
headers["Authorization"] = "Bearer #{access_token}"
|
78
|
+
else
|
79
|
+
fetch_bearer_token
|
80
|
+
headers["Authorization"] = "Bearer #{access_token}"
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
options = {}
|
85
|
+
options[:headers] = headers.merge(header)
|
86
|
+
|
87
|
+
options[:query] = {
|
88
|
+
key: api_key,
|
89
|
+
contentCheckOk: true,
|
90
|
+
racyCheckOk: true
|
91
|
+
}.merge(query)
|
92
|
+
options[:data] = data
|
93
|
+
|
94
|
+
resp = Request.post(endpoint, options)
|
95
|
+
JSON.parse(resp)
|
96
|
+
end
|
97
|
+
|
98
|
+
def player(video_id)
|
99
|
+
endpoint = "#{BASE_URL}/player"
|
100
|
+
query = {"videoId" => video_id}
|
101
|
+
|
102
|
+
send(endpoint, query, {context: context})
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
data/lib/rubytube/monostate.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
|
-
module RubyTube
|
2
|
-
class Monostate
|
3
|
-
attr_accessor :title, :duration
|
4
|
-
end
|
5
|
-
end
|
1
|
+
module RubyTube
|
2
|
+
class Monostate
|
3
|
+
attr_accessor :title, :duration
|
4
|
+
end
|
5
|
+
end
|