ronin-support-web 0.1.0.rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.document +4 -0
- data/.github/workflows/ruby.yml +47 -0
- data/.gitignore +13 -0
- data/.rspec +1 -0
- data/.rubocop.yml +19 -0
- data/.ruby-version +1 -0
- data/.yardopts +1 -0
- data/COPYING.txt +165 -0
- data/ChangeLog.md +10 -0
- data/Gemfile +36 -0
- data/README.md +321 -0
- data/Rakefile +31 -0
- data/gemspec.yml +31 -0
- data/lib/ronin/support/web/agent/mixin.rb +361 -0
- data/lib/ronin/support/web/agent.rb +1386 -0
- data/lib/ronin/support/web/html/mixin.rb +115 -0
- data/lib/ronin/support/web/html.rb +116 -0
- data/lib/ronin/support/web/mixin.rb +40 -0
- data/lib/ronin/support/web/version.rb +28 -0
- data/lib/ronin/support/web/websocket/client.rb +129 -0
- data/lib/ronin/support/web/websocket/mixin.rb +888 -0
- data/lib/ronin/support/web/websocket/server.rb +212 -0
- data/lib/ronin/support/web/websocket/socket.rb +185 -0
- data/lib/ronin/support/web/websocket/url_methods.rb +91 -0
- data/lib/ronin/support/web/websocket.rb +360 -0
- data/lib/ronin/support/web/xml/mixin.rb +111 -0
- data/lib/ronin/support/web/xml.rb +110 -0
- data/lib/ronin/support/web.rb +43 -0
- data/ronin-support-web.gemspec +62 -0
- metadata +150 -0
data/gemspec.yml
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
name: ronin-support-web
|
2
|
+
summary: A web support library for ronin-rb.
|
3
|
+
description: |
|
4
|
+
ronin-support-web is a web specific support library for ronin-rb.
|
5
|
+
ronin-support-web provides many helper methods for parsing HTML/XML,
|
6
|
+
fetching web pages, etc.
|
7
|
+
|
8
|
+
license: LGPL-3.0
|
9
|
+
authors: Postmodern
|
10
|
+
email: postmodern.mod3@gmail.com
|
11
|
+
homepage: https://ronin-rb.dev/
|
12
|
+
has_yard: true
|
13
|
+
|
14
|
+
metadata:
|
15
|
+
documentation_uri: https://ronin-rb.dev/docs/ronin-support-web
|
16
|
+
source_code_uri: https://github.com/ronin-rb/ronin-support-web
|
17
|
+
bug_tracker_uri: https://github.com/ronin-rb/ronin-support-web/issues
|
18
|
+
changelog_uri: https://github.com/ronin-rb/ronin-support-web/blob/main/ChangeLog.md
|
19
|
+
rubygems_mfa_required: 'true'
|
20
|
+
|
21
|
+
required_ruby_version: ">= 3.0.0"
|
22
|
+
|
23
|
+
dependencies:
|
24
|
+
nokogiri: ~> 1.4
|
25
|
+
nokogiri-ext: ~> 0.1
|
26
|
+
# Ronin dependencies:
|
27
|
+
ronin-support: ~> 1.1.0.rc1
|
28
|
+
websocket: ~> 1.2
|
29
|
+
|
30
|
+
development_dependencies:
|
31
|
+
bundler: ~> 2.0
|
@@ -0,0 +1,361 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
#
|
3
|
+
# ronin-support-web - A web support library for ronin-rb.
|
4
|
+
#
|
5
|
+
# Copyright (c) 2023-2024 Hal Brodigan (postmodern.mod3@gmail.com)
|
6
|
+
#
|
7
|
+
# ronin-support-web is free software: you can redistribute it and/or modify
|
8
|
+
# it under the terms of the GNU Lesser General Public License as published
|
9
|
+
# by the Free Software Foundation, either version 3 of the License, or
|
10
|
+
# (at your option) any later version.
|
11
|
+
#
|
12
|
+
# ronin-support-web is distributed in the hope that it will be useful,
|
13
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
14
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
15
|
+
# GNU Lesser General Public License for more details.
|
16
|
+
#
|
17
|
+
# You should have received a copy of the GNU Lesser General Public License
|
18
|
+
# along with ronin-support-web. If not, see <https://www.gnu.org/licenses/>.
|
19
|
+
#
|
20
|
+
|
21
|
+
require 'ronin/support/web/agent'
|
22
|
+
|
23
|
+
module Ronin
|
24
|
+
module Support
|
25
|
+
module Web
|
26
|
+
class Agent
|
27
|
+
#
|
28
|
+
# Provides helper methods for performing high-level web requests.
|
29
|
+
#
|
30
|
+
# ## Features
|
31
|
+
#
|
32
|
+
# * Automatically follows redirects.
|
33
|
+
# * Provides high-level methods for requesting and parsing HTML, XML, or
|
34
|
+
# JSON.
|
35
|
+
# * Maintains a persistent connection pool.
|
36
|
+
#
|
37
|
+
# ## Anti-Features
|
38
|
+
#
|
39
|
+
# * Does not cache files or write to the disk.
|
40
|
+
# * Does not evaluate JavaScript.
|
41
|
+
#
|
42
|
+
module Mixin
|
43
|
+
#
|
44
|
+
# The web agent object.
|
45
|
+
#
|
46
|
+
# @return [Agent]
|
47
|
+
#
|
48
|
+
def web_agent
|
49
|
+
@web_agent ||= Agent.new
|
50
|
+
end
|
51
|
+
|
52
|
+
#
|
53
|
+
# @!macro request_kwargs
|
54
|
+
# @option kwargs [String, nil] :query
|
55
|
+
# The query-string to append to the request path.
|
56
|
+
#
|
57
|
+
# @option kwargs [Hash, nil] :query_params
|
58
|
+
# The query-params to append to the request path.
|
59
|
+
#
|
60
|
+
# @option kwargs [String, nil] :user
|
61
|
+
# The user to authenticate as.
|
62
|
+
#
|
63
|
+
# @option kwargs [String, nil] :password
|
64
|
+
# The password to authenticate with.
|
65
|
+
#
|
66
|
+
# @option kwargs [Hash{Symbol,String => String}, nil] :headers
|
67
|
+
# Additional HTTP headers to use for the request.
|
68
|
+
#
|
69
|
+
# @option kwargs [String, :text, :xml, :html, :json, nil] :content_type
|
70
|
+
# The `Content-Type` header value for the request.
|
71
|
+
# If a Symbol is given it will be resolved to a common MIME type:
|
72
|
+
# * `:text` - `text/plain`
|
73
|
+
# * `:xml` - `text/xml`
|
74
|
+
# * `:html` - `text/html`
|
75
|
+
# * `:json` - `application/json`
|
76
|
+
#
|
77
|
+
# @option kwargs [String, :text, :xml, :html, :json, nil] :accept
|
78
|
+
# The `Accept` header value for the request.
|
79
|
+
# If a Symbol is given it will be resolved to a common MIME type:
|
80
|
+
# * `:text` - `text/plain`
|
81
|
+
# * `:xml` - `text/xml`
|
82
|
+
# * `:html` - `text/html`
|
83
|
+
# * `:json` - `application/json`
|
84
|
+
#
|
85
|
+
# @option kwargs [String, Hash{String => String}, Ronin::Support::Network::HTTP::Cookie, nil] :cookie
|
86
|
+
# Additional `Cookie` header.
|
87
|
+
# * If a `Hash` is given, it will be converted to a `String` using
|
88
|
+
# [Ronin::Support::Network::HTTP::Cookie](https://ronin-rb.dev/docs/ronin-support/Ronin/Support/Network/HTTP/Cookie.html).
|
89
|
+
# * If the cookie value is empty, the `Cookie` header will not be
|
90
|
+
# set.
|
91
|
+
#
|
92
|
+
# @option kwargs [String, nil] :body
|
93
|
+
# The body of the request.
|
94
|
+
#
|
95
|
+
# @option kwargs [Hash, String, nil] :form_data
|
96
|
+
# The form data that may be sent in the body of the request.
|
97
|
+
#
|
98
|
+
# @option kwargs [#to_json, nil] :json
|
99
|
+
# The JSON data that will be sent in the body of the request.
|
100
|
+
# Will also default the `Content-Type` header to
|
101
|
+
# `application/json`, unless already set.
|
102
|
+
#
|
103
|
+
|
104
|
+
#
|
105
|
+
# Gets a URL and returns the response.
|
106
|
+
#
|
107
|
+
# @param [URI::HTTP, Addressable::URI, String] url
|
108
|
+
# The URL to create the HTTP GET request for.
|
109
|
+
#
|
110
|
+
# @!macro request_kwargs
|
111
|
+
#
|
112
|
+
# @yield [response]
|
113
|
+
# If a block is given it will be passed the received HTTP response.
|
114
|
+
#
|
115
|
+
# @yieldparam [Net::HTTPResponse] response
|
116
|
+
# The received HTTP response object.
|
117
|
+
#
|
118
|
+
# @return [Net::HTTPResponse]
|
119
|
+
# The HTTP response object.
|
120
|
+
#
|
121
|
+
# @raise [TooManyRedirects]
|
122
|
+
# Maximum number of redirects reached.
|
123
|
+
#
|
124
|
+
# @note This method will follow redirects by default.
|
125
|
+
#
|
126
|
+
# @example
|
127
|
+
# response = web_get('https://example.com/')
|
128
|
+
# # => #<Net::HTTPResponse:...>
|
129
|
+
#
|
130
|
+
def web_get(url,**kwargs,&block)
|
131
|
+
web_agent.get(url,**kwargs,&block)
|
132
|
+
end
|
133
|
+
|
134
|
+
alias get web_get
|
135
|
+
|
136
|
+
#
|
137
|
+
# Gets the URL and returns the parsed HTML.
|
138
|
+
#
|
139
|
+
# @param [URI::HTTP, Addressable::URI, String] url
|
140
|
+
# The URL to create the HTTP GET request for.
|
141
|
+
#
|
142
|
+
# @!macro request_kwargs
|
143
|
+
#
|
144
|
+
# @return [Nokogiri::HTML::Document]
|
145
|
+
# The parsed HTML response.
|
146
|
+
#
|
147
|
+
# @raise [ContentTypeError]
|
148
|
+
# Did not receive a response with a `Content-Type` of `text/html`.
|
149
|
+
#
|
150
|
+
# @raise [TooManyRedirects]
|
151
|
+
# Maximum number of redirects reached.
|
152
|
+
#
|
153
|
+
# @note This method will follow redirects by default.
|
154
|
+
#
|
155
|
+
# @example
|
156
|
+
# doc = web_get_html('https://example.com/page.html')
|
157
|
+
# # => #<Nokogiri::HTML::Document:...>
|
158
|
+
#
|
159
|
+
def web_get_html(url,**kwargs)
|
160
|
+
web_agent.get_html(url,**kwargs)
|
161
|
+
end
|
162
|
+
|
163
|
+
alias get_html web_get_html
|
164
|
+
|
165
|
+
#
|
166
|
+
# Gets the URL and returns the parsed XML.
|
167
|
+
#
|
168
|
+
# @param [URI::HTTP, Addressable::URI, String] url
|
169
|
+
# The URL to create the HTTP GET request for.
|
170
|
+
#
|
171
|
+
# @!macro request_kwargs
|
172
|
+
#
|
173
|
+
# @return [Nokogiri::XML::Document]
|
174
|
+
# The parsed XML response.
|
175
|
+
#
|
176
|
+
# @raise [ContentTypeError]
|
177
|
+
# Did not receive a response with a `Content-Type` of `text/xml`.
|
178
|
+
#
|
179
|
+
# @raise [TooManyRedirects]
|
180
|
+
# Maximum number of redirects reached.
|
181
|
+
#
|
182
|
+
# @note This method will follow redirects by default.
|
183
|
+
#
|
184
|
+
# @example
|
185
|
+
# doc = web_get_xml('https://example.com/data.xml')
|
186
|
+
# # => #<Nokogiri::XML::Document:...>
|
187
|
+
#
|
188
|
+
def web_get_xml(url,**kwargs)
|
189
|
+
web_agent.get_xml(url,**kwargs)
|
190
|
+
end
|
191
|
+
|
192
|
+
alias get_xml web_get_xml
|
193
|
+
|
194
|
+
#
|
195
|
+
# Gets the URL and returns the parsed JSON.
|
196
|
+
#
|
197
|
+
# @param [URI::HTTP, Addressable::URI, String] url
|
198
|
+
# The URL to create the HTTP GET request for.
|
199
|
+
#
|
200
|
+
# @!macro request_kwargs
|
201
|
+
#
|
202
|
+
# @return [Hash{String => Object}, Array]
|
203
|
+
# The parsed JSON.
|
204
|
+
#
|
205
|
+
# @raise [ContentTypeError]
|
206
|
+
# Did not receive a response with a `Content-Type` of
|
207
|
+
# `application/json`.
|
208
|
+
#
|
209
|
+
# @raise [TooManyRedirects]
|
210
|
+
# Maximum number of redirects reached.
|
211
|
+
#
|
212
|
+
# @note This method will follow redirects by default.
|
213
|
+
#
|
214
|
+
# @example
|
215
|
+
# json = web_get_json('https://example.com/data.json')
|
216
|
+
# # => {...}
|
217
|
+
#
|
218
|
+
def web_get_json(url,**kwargs)
|
219
|
+
web_agent.get_json(url,**kwargs)
|
220
|
+
end
|
221
|
+
|
222
|
+
alias get_json web_get_json
|
223
|
+
|
224
|
+
#
|
225
|
+
# Performs an HTTP POST to the URL.
|
226
|
+
#
|
227
|
+
# @param [URI::HTTP, Addressable::URI, String] url
|
228
|
+
# The URL to create the HTTP GET request for.
|
229
|
+
#
|
230
|
+
# @!macro request_kwargs
|
231
|
+
#
|
232
|
+
# @yield [response]
|
233
|
+
# If a block is given it will be passed the received HTTP response.
|
234
|
+
#
|
235
|
+
# @yieldparam [Net::HTTPResponse] response
|
236
|
+
# The received HTTP response object.
|
237
|
+
#
|
238
|
+
# @return [Net::HTTPResponse]
|
239
|
+
# The HTTP response object.
|
240
|
+
#
|
241
|
+
# @raise [TooManyRedirects]
|
242
|
+
# Maximum number of redirects reached.
|
243
|
+
#
|
244
|
+
# @note
|
245
|
+
# If the response is an HTTP redirect, then {#get} will be called to
|
246
|
+
# follow any redirects.
|
247
|
+
#
|
248
|
+
# @example
|
249
|
+
# response = web_post('https://example.com/form', form_data: {'foo' => 'bar'})
|
250
|
+
# # => #<Net::HTTPResponse:...>
|
251
|
+
#
|
252
|
+
def web_post(url,**kwargs,&block)
|
253
|
+
web_agent.post(url,**kwargs,&block)
|
254
|
+
end
|
255
|
+
|
256
|
+
alias post web_post
|
257
|
+
|
258
|
+
#
|
259
|
+
# Performs an HTTP POST to the URL and parses the HTML response.
|
260
|
+
#
|
261
|
+
# @param [URI::HTTP, Addressable::URI, String] url
|
262
|
+
# The URL to create the HTTP POST request for.
|
263
|
+
#
|
264
|
+
# @!macro request_kwargs
|
265
|
+
#
|
266
|
+
# @return [Nokogiri::HTML::Document]
|
267
|
+
# The parsed HTML response.
|
268
|
+
#
|
269
|
+
# @raise [TooManyRedirects]
|
270
|
+
# Maximum number of redirects reached.
|
271
|
+
#
|
272
|
+
# @raise [ContentTypeError]
|
273
|
+
# Did not receive a response with a `Content-Type` of
|
274
|
+
# `text/html`.
|
275
|
+
#
|
276
|
+
# @note
|
277
|
+
# If the response is an HTTP redirect, then {#get} will be called to
|
278
|
+
# follow any redirects.
|
279
|
+
#
|
280
|
+
# @example Send a POST request and parses the HTML response:
|
281
|
+
# doc = web_post_html 'https://example.com/form', form_data: {foo: 'bar'})
|
282
|
+
# # => #<Nokogiri::HTML::Document:...>
|
283
|
+
#
|
284
|
+
def web_post_html(url,**kwargs)
|
285
|
+
web_agent.post_html(url,**kwargs)
|
286
|
+
end
|
287
|
+
|
288
|
+
alias post_html web_post_html
|
289
|
+
|
290
|
+
#
|
291
|
+
# Performs an HTTP POST to the URL and parses the XML response.
|
292
|
+
#
|
293
|
+
# @param [URI::HTTP, Addressable::URI, String] url
|
294
|
+
# The URL to create the HTTP POST request for.
|
295
|
+
#
|
296
|
+
# @!macro request_kwargs
|
297
|
+
#
|
298
|
+
# @return [Nokogiri::XML::Document]
|
299
|
+
# The parsed XML response.
|
300
|
+
#
|
301
|
+
# @raise [TooManyRedirects]
|
302
|
+
# Maximum number of redirects reached.
|
303
|
+
#
|
304
|
+
# @raise [ContentTypeError]
|
305
|
+
# Did not receive a response with a `Content-Type` of
|
306
|
+
# `text/xml`.
|
307
|
+
#
|
308
|
+
# @note
|
309
|
+
# If the response is an HTTP redirect, then {#get} will be called to
|
310
|
+
# follow any redirects.
|
311
|
+
#
|
312
|
+
# @example Send a POST request to the form and parses the XML response:
|
313
|
+
# doc = web_post_xml 'https://example.com/form', form_data: {foo: 'bar'}
|
314
|
+
# # => #<Nokogiri::XML::Document:...>
|
315
|
+
#
|
316
|
+
def web_post_xml(url,**kwargs)
|
317
|
+
web_agent.post_xml(url,**kwargs)
|
318
|
+
end
|
319
|
+
|
320
|
+
alias post_xml web_post_xml
|
321
|
+
|
322
|
+
#
|
323
|
+
# Performs an HTTP POST to the URL and parses the JSON response.
|
324
|
+
#
|
325
|
+
# @param [URI::HTTP, Addressable::URI, String] url
|
326
|
+
# The URL to create the HTTP POST request for.
|
327
|
+
#
|
328
|
+
# @!macro request_kwargs
|
329
|
+
#
|
330
|
+
# @return [Hash{String => Object}, Array]
|
331
|
+
# The parses JSON response.
|
332
|
+
#
|
333
|
+
# @raise [TooManyRedirects]
|
334
|
+
# Maximum number of redirects reached.
|
335
|
+
#
|
336
|
+
# @raise [ContentTypeError]
|
337
|
+
# Did not receive a response with a `Content-Type` of
|
338
|
+
# `application/json`.
|
339
|
+
#
|
340
|
+
# @note
|
341
|
+
# If the response is an HTTP redirect, then {#get} will be called to
|
342
|
+
# follow any redirects.
|
343
|
+
#
|
344
|
+
# @example Send a POST request to the form and parse the JSON response:
|
345
|
+
# json = web_post_json 'https://example.com/form', form_data: {foo: 'bar'}
|
346
|
+
# # => {...}
|
347
|
+
#
|
348
|
+
# @example Send a POST request containing JSON and parse the JSON response:
|
349
|
+
# json = web_post_json 'https://example.com/api/end-point', json: {foo: 'bar'}
|
350
|
+
# # => {...}
|
351
|
+
#
|
352
|
+
def web_post_json(url,**kwargs)
|
353
|
+
web_agent.post_json(url,**kwargs)
|
354
|
+
end
|
355
|
+
|
356
|
+
alias post_json web_post_json
|
357
|
+
end
|
358
|
+
end
|
359
|
+
end
|
360
|
+
end
|
361
|
+
end
|