puppeteer-ruby 0.0.15 → 0.0.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.circleci/config.yml +25 -2
- data/.github/workflows/reviewdog.yml +1 -1
- data/.rubocop.yml +49 -3
- data/Dockerfile +9 -0
- data/README.md +26 -1
- data/docker-compose.yml +34 -0
- data/lib/puppeteer.rb +17 -12
- data/lib/puppeteer/browser.rb +36 -9
- data/lib/puppeteer/browser_context.rb +35 -5
- data/lib/puppeteer/browser_runner.rb +1 -1
- data/lib/puppeteer/cdp_session.rb +10 -0
- data/lib/puppeteer/connection.rb +8 -3
- data/lib/puppeteer/debug_print.rb +2 -2
- data/lib/puppeteer/define_async_method.rb +1 -1
- data/lib/puppeteer/dialog.rb +34 -0
- data/lib/puppeteer/dom_world.rb +37 -14
- data/lib/puppeteer/element_handle.rb +49 -20
- data/lib/puppeteer/env.rb +23 -0
- data/lib/puppeteer/frame.rb +14 -32
- data/lib/puppeteer/js_handle.rb +37 -27
- data/lib/puppeteer/keyboard.rb +3 -2
- data/lib/puppeteer/launcher.rb +11 -1
- data/lib/puppeteer/launcher/base.rb +14 -4
- data/lib/puppeteer/launcher/chrome.rb +1 -1
- data/lib/puppeteer/launcher/firefox.rb +392 -0
- data/lib/puppeteer/mouse.rb +16 -0
- data/lib/puppeteer/network_manager.rb +163 -5
- data/lib/puppeteer/page.rb +158 -85
- data/lib/puppeteer/remote_object.rb +28 -1
- data/lib/puppeteer/request.rb +330 -0
- data/lib/puppeteer/response.rb +113 -0
- data/lib/puppeteer/version.rb +1 -1
- data/lib/puppeteer/wait_task.rb +1 -1
- data/lib/puppeteer/web_socket.rb +7 -0
- data/puppeteer-ruby.gemspec +2 -1
- metadata +25 -4
@@ -6,6 +6,7 @@ class Puppeteer::RemoteObject
|
|
6
6
|
# @param payload [Hash]
|
7
7
|
def initialize(payload)
|
8
8
|
@object_id = payload['objectId']
|
9
|
+
@type = payload['type']
|
9
10
|
@sub_type = payload['subtype']
|
10
11
|
@unserializable_value = payload['unserializableValue']
|
11
12
|
@value = payload['value']
|
@@ -42,6 +43,21 @@ class Puppeteer::RemoteObject
|
|
42
43
|
end
|
43
44
|
end
|
44
45
|
|
46
|
+
# @return [String]
|
47
|
+
def type_str
|
48
|
+
# used in JSHandle#to_s
|
49
|
+
# original logic:
|
50
|
+
# if (this._remoteObject.objectId) {
|
51
|
+
# const type = this._remoteObject.subtype || this._remoteObject.type;
|
52
|
+
# return 'JSHandle@' + type;
|
53
|
+
# }
|
54
|
+
if @object_id
|
55
|
+
@sub_type || @type
|
56
|
+
else
|
57
|
+
nil
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
45
61
|
# used in JSHandle#properties
|
46
62
|
def properties(client)
|
47
63
|
# original logic:
|
@@ -64,7 +80,18 @@ class Puppeteer::RemoteObject
|
|
64
80
|
|
65
81
|
# used in ElementHandle#_box_model
|
66
82
|
def box_model(client)
|
67
|
-
client.send_message('DOM.getBoxModel', objectId: @object_id)
|
83
|
+
result = client.send_message('DOM.getBoxModel', objectId: @object_id)
|
84
|
+
|
85
|
+
# Firefox returns width/height = 0, content/padding/border/margin = [nil, nil, nil, nil, nil, nil, nil, nil]
|
86
|
+
# while Chrome throws Error(Could not compute box model)
|
87
|
+
model = result['model']
|
88
|
+
if model['width'] == 0 && model['height'] == 0 &&
|
89
|
+
%w(content padding border margin).all? { |key| model[key].all?(&:nil?) }
|
90
|
+
|
91
|
+
debug_puts('Could not compute box model in Firefox.')
|
92
|
+
return nil
|
93
|
+
end
|
94
|
+
result
|
68
95
|
rescue => err
|
69
96
|
debug_puts(err)
|
70
97
|
nil
|
@@ -0,0 +1,330 @@
|
|
1
|
+
class Puppeteer::Request
|
2
|
+
include Puppeteer::DebugPrint
|
3
|
+
include Puppeteer::IfPresent
|
4
|
+
|
5
|
+
# defines some methods used only in NetworkManager, Response
|
6
|
+
class InternalAccessor
|
7
|
+
def initialize(request)
|
8
|
+
@request = request
|
9
|
+
end
|
10
|
+
|
11
|
+
def request_id
|
12
|
+
@request.instance_variable_get(:@request_id)
|
13
|
+
end
|
14
|
+
|
15
|
+
def interception_id
|
16
|
+
@request.instance_variable_get(:@interception_id)
|
17
|
+
end
|
18
|
+
|
19
|
+
# @param response [Puppeteer::Response]
|
20
|
+
def response=(response)
|
21
|
+
@request.instance_variable_set(:@response, response)
|
22
|
+
end
|
23
|
+
|
24
|
+
def redirect_chain
|
25
|
+
@request.instance_variable_get(:@redirect_chain)
|
26
|
+
end
|
27
|
+
|
28
|
+
def failure_text=(failure_text)
|
29
|
+
@request.instance_variable_set(:@failure_text, failure_text)
|
30
|
+
end
|
31
|
+
|
32
|
+
def from_memory_cache=(from_memory_cache)
|
33
|
+
@request.instance_variable_set(:@from_memory_cache, from_memory_cache)
|
34
|
+
end
|
35
|
+
|
36
|
+
def from_memory_cache?
|
37
|
+
@request.instance_variable_get(:@from_memory_cache)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
# @param client [Puppeteer::CDPSession]
|
42
|
+
# @param frame [Puppeteer::Frame]
|
43
|
+
# @param interception_id [string|nil]
|
44
|
+
# @param allow_interception [boolean]
|
45
|
+
# @param event [Hash]
|
46
|
+
# @param redirect_chain Array<Request>
|
47
|
+
def initialize(client, frame, interception_id, allow_interception, event, redirect_chain)
|
48
|
+
@client = client
|
49
|
+
@request_id = event['requestId']
|
50
|
+
@is_navigation_request = event['requestId'] == event['loaderId'] && event['type'] == 'Document'
|
51
|
+
@interception_id = interception_id
|
52
|
+
@allow_interception = allow_interception
|
53
|
+
@url = event['request']['url']
|
54
|
+
@resource_type = event['type'].downcase
|
55
|
+
@method = event['request']['method']
|
56
|
+
@post_data = event['request']['postData']
|
57
|
+
@frame = frame
|
58
|
+
@redirect_chain = redirect_chain
|
59
|
+
@headers = {}
|
60
|
+
event['request']['headers'].each do |key, value|
|
61
|
+
@headers[key.downcase] = value
|
62
|
+
end
|
63
|
+
@from_memory_cache = false
|
64
|
+
|
65
|
+
@internal = InternalAccessor.new(self)
|
66
|
+
end
|
67
|
+
|
68
|
+
attr_reader :internal
|
69
|
+
attr_reader :url, :resource_type, :method, :post_data, :headers, :response, :frame
|
70
|
+
|
71
|
+
def navigation_request?
|
72
|
+
@is_navigation_request
|
73
|
+
end
|
74
|
+
|
75
|
+
def redirect_chain
|
76
|
+
@redirect_chain.dup
|
77
|
+
end
|
78
|
+
|
79
|
+
def failure
|
80
|
+
if_present(@failure_text) do |failure_text|
|
81
|
+
{ errorText: @failure_text }
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
private def headers_to_array(headers)
|
86
|
+
return nil unless headers
|
87
|
+
|
88
|
+
headers.map do |key, value|
|
89
|
+
{ name: key, value: value.to_s }
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
class InterceptionNotEnabledError < StandardError
|
94
|
+
def initialize
|
95
|
+
super('Request Interception is not enabled!')
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
class AlreadyHandledError < StandardError
|
100
|
+
def initialize
|
101
|
+
super('Request is already handled!')
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
# proceed request on request interception.
|
106
|
+
#
|
107
|
+
# Example:
|
108
|
+
#
|
109
|
+
# page.on 'request' do |req|
|
110
|
+
# # Override headers
|
111
|
+
# headers = req.headers.merge(
|
112
|
+
# foo: 'bar', # set "foo" header
|
113
|
+
# origin: nil, # remove "origin" header
|
114
|
+
# )
|
115
|
+
# req.continue(headers: headers)
|
116
|
+
# end
|
117
|
+
#
|
118
|
+
# @param error_code [String|Symbol]
|
119
|
+
def continue(url: nil, method: nil, post_data: nil, headers: nil)
|
120
|
+
# Request interception is not supported for data: urls.
|
121
|
+
return if @url.start_with?('data:')
|
122
|
+
|
123
|
+
unless @allow_interception
|
124
|
+
raise InterceptionNotEnabledError.new
|
125
|
+
end
|
126
|
+
if @interception_handled
|
127
|
+
raise AlreadyHandledError.new
|
128
|
+
end
|
129
|
+
@interception_handled = true
|
130
|
+
|
131
|
+
overrides = {
|
132
|
+
url: url,
|
133
|
+
method: method,
|
134
|
+
post_data: post_data,
|
135
|
+
headers: headers_to_array(headers),
|
136
|
+
}.compact
|
137
|
+
begin
|
138
|
+
@client.send_message('Fetch.continueRequest',
|
139
|
+
requestId: @interception_id,
|
140
|
+
**overrides,
|
141
|
+
)
|
142
|
+
rescue => err
|
143
|
+
# In certain cases, protocol will return error if the request was already canceled
|
144
|
+
# or the page was closed. We should tolerate these errors.
|
145
|
+
debug_puts(err)
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
149
|
+
# Mocking response.
|
150
|
+
#
|
151
|
+
# Example:
|
152
|
+
#
|
153
|
+
# page.on 'request' do |req|
|
154
|
+
# req.respond(
|
155
|
+
# status: 404,
|
156
|
+
# content_type: 'text/plain',
|
157
|
+
# body: 'Not Found!'
|
158
|
+
# )
|
159
|
+
# end
|
160
|
+
#
|
161
|
+
# @param status [Integer]
|
162
|
+
# @param headers [Hash<String, String>]
|
163
|
+
# @param content_type [String]
|
164
|
+
# @param body [String]
|
165
|
+
def respond(status: nil, headers: nil, content_type: nil, body: nil)
|
166
|
+
# Mocking responses for dataURL requests is not currently supported.
|
167
|
+
return if @url.start_with?('data:')
|
168
|
+
|
169
|
+
unless @allow_interception
|
170
|
+
raise InterceptionNotEnabledError.new
|
171
|
+
end
|
172
|
+
if @interception_handled
|
173
|
+
raise AlreadyHandledError.new
|
174
|
+
end
|
175
|
+
@interception_handled = true
|
176
|
+
|
177
|
+
mock_response_headers = {}
|
178
|
+
headers&.each do |key, value|
|
179
|
+
mock_response_headers[key.downcase] = value
|
180
|
+
end
|
181
|
+
if content_type
|
182
|
+
mock_response_headers['content-type'] = content_type
|
183
|
+
end
|
184
|
+
if body
|
185
|
+
mock_response_headers['content-length'] = body.length
|
186
|
+
end
|
187
|
+
|
188
|
+
mock_response = {
|
189
|
+
responseCode: status || 200,
|
190
|
+
responsePhrase: STATUS_TEXTS[(status || 200).to_s],
|
191
|
+
responseHeaders: headers_to_array(mock_response_headers),
|
192
|
+
body: if_present(body) { |mock_body| Base64.strict_encode64(mock_body) },
|
193
|
+
}.compact
|
194
|
+
begin
|
195
|
+
@client.send_message('Fetch.fulfillRequest',
|
196
|
+
requestId: @interception_id,
|
197
|
+
**mock_response,
|
198
|
+
)
|
199
|
+
rescue => err
|
200
|
+
# In certain cases, protocol will return error if the request was already canceled
|
201
|
+
# or the page was closed. We should tolerate these errors.
|
202
|
+
debug_puts(err)
|
203
|
+
end
|
204
|
+
end
|
205
|
+
|
206
|
+
# abort request on request interception.
|
207
|
+
#
|
208
|
+
# Example:
|
209
|
+
#
|
210
|
+
# page.on 'request' do |req|
|
211
|
+
# if req.url.include?("porn")
|
212
|
+
# req.abort
|
213
|
+
# else
|
214
|
+
# req.continue
|
215
|
+
# end
|
216
|
+
# end
|
217
|
+
#
|
218
|
+
# @param error_code [String|Symbol]
|
219
|
+
def abort(error_code: :failed)
|
220
|
+
# Request interception is not supported for data: urls.
|
221
|
+
return if @url.start_with?('data:')
|
222
|
+
|
223
|
+
error_reason = ERROR_REASONS[error_code.to_s]
|
224
|
+
unless error_reason
|
225
|
+
raise ArgumentError.new("Unknown error code: #{error_code}")
|
226
|
+
end
|
227
|
+
unless @allow_interception
|
228
|
+
raise InterceptionNotEnabledError.new
|
229
|
+
end
|
230
|
+
if @interception_handled
|
231
|
+
raise AlreadyHandledError.new
|
232
|
+
end
|
233
|
+
@interception_handled = true
|
234
|
+
|
235
|
+
begin
|
236
|
+
@client.send_message('Fetch.failRequest',
|
237
|
+
requestId: @interception_id,
|
238
|
+
errorReason: error_reason,
|
239
|
+
)
|
240
|
+
rescue => err
|
241
|
+
# In certain cases, protocol will return error if the request was already canceled
|
242
|
+
# or the page was closed. We should tolerate these errors.
|
243
|
+
debug_puts(err)
|
244
|
+
end
|
245
|
+
end
|
246
|
+
|
247
|
+
ERROR_REASONS = {
|
248
|
+
'aborted' => 'Aborted',
|
249
|
+
'accessdenied' => 'AccessDenied',
|
250
|
+
'addressunreachable' => 'AddressUnreachable',
|
251
|
+
'blockedbyclient' => 'BlockedByClient',
|
252
|
+
'blockedbyresponse' => 'BlockedByResponse',
|
253
|
+
'connectionaborted' => 'ConnectionAborted',
|
254
|
+
'connectionclosed' => 'ConnectionClosed',
|
255
|
+
'connectionfailed' => 'ConnectionFailed',
|
256
|
+
'connectionrefused' => 'ConnectionRefused',
|
257
|
+
'connectionreset' => 'ConnectionReset',
|
258
|
+
'internetdisconnected' => 'InternetDisconnected',
|
259
|
+
'namenotresolved' => 'NameNotResolved',
|
260
|
+
'timedout' => 'TimedOut',
|
261
|
+
'failed' => 'Failed',
|
262
|
+
}.freeze
|
263
|
+
|
264
|
+
# List taken from https://www.iana.org/assignments/http-status-codes/http-status-codes.xhtml with extra 306 and 418 codes.
|
265
|
+
STATUS_TEXTS = {
|
266
|
+
'100' => 'Continue',
|
267
|
+
'101' => 'Switching Protocols',
|
268
|
+
'102' => 'Processing',
|
269
|
+
'103' => 'Early Hints',
|
270
|
+
'200' => 'OK',
|
271
|
+
'201' => 'Created',
|
272
|
+
'202' => 'Accepted',
|
273
|
+
'203' => 'Non-Authoritative Information',
|
274
|
+
'204' => 'No Content',
|
275
|
+
'205' => 'Reset Content',
|
276
|
+
'206' => 'Partial Content',
|
277
|
+
'207' => 'Multi-Status',
|
278
|
+
'208' => 'Already Reported',
|
279
|
+
'226' => 'IM Used',
|
280
|
+
'300' => 'Multiple Choices',
|
281
|
+
'301' => 'Moved Permanently',
|
282
|
+
'302' => 'Found',
|
283
|
+
'303' => 'See Other',
|
284
|
+
'304' => 'Not Modified',
|
285
|
+
'305' => 'Use Proxy',
|
286
|
+
'306' => 'Switch Proxy',
|
287
|
+
'307' => 'Temporary Redirect',
|
288
|
+
'308' => 'Permanent Redirect',
|
289
|
+
'400' => 'Bad Request',
|
290
|
+
'401' => 'Unauthorized',
|
291
|
+
'402' => 'Payment Required',
|
292
|
+
'403' => 'Forbidden',
|
293
|
+
'404' => 'Not Found',
|
294
|
+
'405' => 'Method Not Allowed',
|
295
|
+
'406' => 'Not Acceptable',
|
296
|
+
'407' => 'Proxy Authentication Required',
|
297
|
+
'408' => 'Request Timeout',
|
298
|
+
'409' => 'Conflict',
|
299
|
+
'410' => 'Gone',
|
300
|
+
'411' => 'Length Required',
|
301
|
+
'412' => 'Precondition Failed',
|
302
|
+
'413' => 'Payload Too Large',
|
303
|
+
'414' => 'URI Too Long',
|
304
|
+
'415' => 'Unsupported Media Type',
|
305
|
+
'416' => 'Range Not Satisfiable',
|
306
|
+
'417' => 'Expectation Failed',
|
307
|
+
'418' => 'I\'m a teapot',
|
308
|
+
'421' => 'Misdirected Request',
|
309
|
+
'422' => 'Unprocessable Entity',
|
310
|
+
'423' => 'Locked',
|
311
|
+
'424' => 'Failed Dependency',
|
312
|
+
'425' => 'Too Early',
|
313
|
+
'426' => 'Upgrade Required',
|
314
|
+
'428' => 'Precondition Required',
|
315
|
+
'429' => 'Too Many Requests',
|
316
|
+
'431' => 'Request Header Fields Too Large',
|
317
|
+
'451' => 'Unavailable For Legal Reasons',
|
318
|
+
'500' => 'Internal Server Error',
|
319
|
+
'501' => 'Not Implemented',
|
320
|
+
'502' => 'Bad Gateway',
|
321
|
+
'503' => 'Service Unavailable',
|
322
|
+
'504' => 'Gateway Timeout',
|
323
|
+
'505' => 'HTTP Version Not Supported',
|
324
|
+
'506' => 'Variant Also Negotiates',
|
325
|
+
'507' => 'Insufficient Storage',
|
326
|
+
'508' => 'Loop Detected',
|
327
|
+
'510' => 'Not Extended',
|
328
|
+
'511' => 'Network Authentication Required',
|
329
|
+
}.freeze
|
330
|
+
end
|
@@ -0,0 +1,113 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
class Puppeteer::Response
|
4
|
+
include Puppeteer::IfPresent
|
5
|
+
|
6
|
+
class Redirected < StandardError
|
7
|
+
def initialize
|
8
|
+
super('Response body is unavailable for redirect responses')
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
# defines methods used only in NetworkManager
|
13
|
+
class InternalAccessor
|
14
|
+
def initialize(response)
|
15
|
+
@response = response
|
16
|
+
end
|
17
|
+
|
18
|
+
def body_loaded_promise
|
19
|
+
@response.instance_variable_get(:@body_loaded_promise)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
class RemoteAddress
|
24
|
+
def initialize(ip:, port:)
|
25
|
+
@ip = ip
|
26
|
+
@port = port
|
27
|
+
end
|
28
|
+
attr_reader :ip, :port
|
29
|
+
end
|
30
|
+
|
31
|
+
# @param client [Puppeteer::CDPSession]
|
32
|
+
# @param request [Puppeteer::Request]
|
33
|
+
# @param response_payload [Hash]
|
34
|
+
def initialize(client, request, response_payload)
|
35
|
+
@client = client
|
36
|
+
@request = request
|
37
|
+
|
38
|
+
@body_loaded_promise = resolvable_future
|
39
|
+
@remote_address = RemoteAddress.new(
|
40
|
+
ip: response_payload['remoteIPAddress'],
|
41
|
+
port: response_payload['remotePort'],
|
42
|
+
)
|
43
|
+
|
44
|
+
@status = response_payload['status']
|
45
|
+
@status_text = response_payload['statusText']
|
46
|
+
@url = request.url
|
47
|
+
@from_disk_cache = !!response_payload['fromDiskCache']
|
48
|
+
@from_service_worker = !!response_payload['fromServiceWorker']
|
49
|
+
|
50
|
+
@headers = {}
|
51
|
+
response_payload['headers'].each do |key, value|
|
52
|
+
@headers[key.downcase] = value
|
53
|
+
end
|
54
|
+
@security_details = if_present(response_payload['securityDetails']) do |security_payload|
|
55
|
+
SecurityDetails.new(security_payload)
|
56
|
+
end
|
57
|
+
|
58
|
+
@internal = InternalAccessor.new(self)
|
59
|
+
end
|
60
|
+
|
61
|
+
attr_reader :internal
|
62
|
+
|
63
|
+
attr_reader :remote_address, :url, :status, :status_text, :headers, :security_details, :request
|
64
|
+
|
65
|
+
# @return [Boolean]
|
66
|
+
def ok?
|
67
|
+
@status == 0 || (@status >= 200 && @status <= 299)
|
68
|
+
end
|
69
|
+
|
70
|
+
def buffer
|
71
|
+
await @body_loaded_promise
|
72
|
+
response = @client.send_message('Network.getResponseBody', requestId: @request.internal.request_id)
|
73
|
+
if response['base64Encoded']
|
74
|
+
Base64.decode64(response['body'])
|
75
|
+
else
|
76
|
+
response['body']
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
# @param text [String]
|
81
|
+
def text
|
82
|
+
buffer
|
83
|
+
end
|
84
|
+
|
85
|
+
# @param json [Hash]
|
86
|
+
def json
|
87
|
+
JSON.parse(text)
|
88
|
+
end
|
89
|
+
|
90
|
+
def from_cache?
|
91
|
+
@from_disk_cache || @request.internal.from_memory_cache?
|
92
|
+
end
|
93
|
+
|
94
|
+
def from_service_worker?
|
95
|
+
@from_service_worker
|
96
|
+
end
|
97
|
+
|
98
|
+
def frame
|
99
|
+
@request.frame
|
100
|
+
end
|
101
|
+
|
102
|
+
class SecurityDetails
|
103
|
+
def initialize(security_payload)
|
104
|
+
@subject_name = security_payload['subjectName']
|
105
|
+
@issuer = security_payload['issuer']
|
106
|
+
@valid_from = security_payload['validFrom']
|
107
|
+
@valid_to = security_payload['validTo']
|
108
|
+
@protocol = security_payload['protocol']
|
109
|
+
end
|
110
|
+
|
111
|
+
attr_reader :subject_name, :issuer, :valid_from, :valid_to, :protocol
|
112
|
+
end
|
113
|
+
end
|