ronin-web 1.0.0 → 1.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ruby.yml +14 -0
- data/.rubocop.yml +20 -0
- data/ChangeLog.md +6 -1
- data/Gemfile +12 -8
- data/README.md +241 -0
- data/Rakefile +1 -1
- data/bin/ronin-web +6 -7
- data/data/new/webapp/Gemfile +2 -2
- data/data/new/webapp/config.ru +0 -2
- data/gemspec.yml +1 -1
- data/lib/ronin/web/cli/command.rb +4 -0
- data/lib/ronin/web/cli/commands/diff.rb +2 -1
- data/lib/ronin/web/cli/commands/html.rb +3 -2
- data/lib/ronin/web/cli/commands/irb.rb +1 -0
- data/lib/ronin/web/cli/commands/new/spider.rb +3 -4
- data/lib/ronin/web/cli/commands/reverse_proxy.rb +5 -4
- data/lib/ronin/web/cli/commands/server.rb +4 -3
- data/lib/ronin/web/cli/commands/spider.rb +74 -72
- data/lib/ronin/web/cli/ruby_shell.rb +1 -0
- data/lib/ronin/web/cli.rb +1 -0
- data/lib/ronin/web/html.rb +1 -0
- data/lib/ronin/web/mechanize.rb +6 -1
- data/lib/ronin/web/root.rb +1 -0
- data/lib/ronin/web/version.rb +2 -1
- data/lib/ronin/web/xml.rb +1 -0
- data/lib/ronin/web.rb +7 -2
- data/ronin-web.gemspec +6 -5
- metadata +9 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d74a1711d3d63c034cb9a4e37f765c27cc1dd4b7e5cbf21154fc936a1db02e02
|
4
|
+
data.tar.gz: 06b58a6806d23dd2a201942978349cedfd57341f9acd9c9c687dcf1c41ec95bc
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: be61036695c8a3ec44f095008352ab9d7fbc741218ca202f3dfbea9fd235bbb7a38ddc3d79896d62d349512f17085473ac2ac73aff7ae2bad4e2d7fda013c49a
|
7
|
+
data.tar.gz: 2ea04d21c0cd5dbdee6ffe714ffd94f89a4b2c8f2dceefeafd9daba0908f890da0b78a78b9df302ae14b39b44ca51e674c262b5fd470671382d9786f3904b6af
|
data/.github/workflows/ruby.yml
CHANGED
@@ -26,3 +26,17 @@ jobs:
|
|
26
26
|
run: bundle install --jobs 4 --retry 3
|
27
27
|
- name: Run tests
|
28
28
|
run: bundle exec rake test
|
29
|
+
|
30
|
+
# rubocop linting
|
31
|
+
rubocop:
|
32
|
+
runs-on: ubuntu-latest
|
33
|
+
steps:
|
34
|
+
- uses: actions/checkout@v2
|
35
|
+
- name: Set up Ruby
|
36
|
+
uses: ruby/setup-ruby@v1
|
37
|
+
with:
|
38
|
+
ruby-version: 3.0
|
39
|
+
- name: Install dependencies
|
40
|
+
run: bundle install --jobs 4 --retry 3
|
41
|
+
- name: Run rubocop
|
42
|
+
run: bundle exec rubocop --parallel
|
data/.rubocop.yml
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
AllCops:
|
2
|
+
NewCops: enable
|
3
|
+
SuggestExtensions: false
|
4
|
+
TargetRubyVersion: 3.1
|
5
|
+
|
6
|
+
inherit_gem:
|
7
|
+
rubocop-ronin: rubocop.yml
|
8
|
+
|
9
|
+
#
|
10
|
+
# ronin-web specific exceptions
|
11
|
+
#
|
12
|
+
|
13
|
+
Style/FrozenStringLiteralComment:
|
14
|
+
Exclude:
|
15
|
+
- 'data/new/**/*'
|
16
|
+
|
17
|
+
Lint/ShadowingOuterLocalVariable:
|
18
|
+
Exclude:
|
19
|
+
- 'lib/ronin/web/cli/commands/reverse_proxy.rb'
|
20
|
+
- 'lib/ronin/web/cli/commands/spider.rb'
|
data/ChangeLog.md
CHANGED
@@ -1,4 +1,9 @@
|
|
1
|
-
### 1.0.
|
1
|
+
### 1.0.1 / 2023-03-01
|
2
|
+
|
3
|
+
* Require `ronin-web-server` ~> 0.1, >= 0.1.1.
|
4
|
+
* Disable SSL/TLS verification by default in {Ronin::Web::Mechanize}.
|
5
|
+
|
6
|
+
### 1.0.0 / 2023-02-01
|
2
7
|
|
3
8
|
* Added [nokogiri-ext] ~> 0.1 as a dependency.
|
4
9
|
* Added [nokogiri-diff] ~> 0.1 as a dependency.
|
data/Gemfile
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
source 'https://rubygems.org'
|
2
4
|
|
3
5
|
gemspec
|
@@ -16,21 +18,21 @@ end
|
|
16
18
|
# gem 'rack', '~> 1.2', github: 'rack/rack'
|
17
19
|
# gem 'sinatra', '~> 1.2', github: 'sinatra/sinatra'
|
18
20
|
|
19
|
-
# gem 'command_kit',
|
20
|
-
#
|
21
|
+
# gem 'command_kit', '~> 0.4', github: 'postmodern/command_kit.rb',
|
22
|
+
# branch: '0.4.0'
|
21
23
|
|
22
24
|
# gem 'spidr', '~> 0.7', github: 'postmodern/spidr'
|
23
25
|
|
24
26
|
# Ronin dependencies
|
25
|
-
# gem 'ronin-support',
|
27
|
+
# gem 'ronin-support', '~> 1.0', github: "ronin-rb/ronin-support",
|
26
28
|
# branch: 'main'
|
27
|
-
# gem 'ronin-web-server',
|
29
|
+
# gem 'ronin-web-server', '~> 0.1', github: "ronin-rb/ronin-web-server",
|
28
30
|
# branch: 'main'
|
29
|
-
# gem 'ronin-web-spider',
|
31
|
+
# gem 'ronin-web-spider', '~> 0.1', github: "ronin-rb/ronin-web-spider",
|
30
32
|
# branch: 'main'
|
31
33
|
# gem 'ronin-web-user_agents', '~> 0.1', github: "ronin-rb/ronin-web-user_agents",
|
32
34
|
# branch: 'main'
|
33
|
-
# gem 'ronin-core',
|
35
|
+
# gem 'ronin-core', '~> 0.1', github: "ronin-rb/ronin-core",
|
34
36
|
# branch: 'main'
|
35
37
|
|
36
38
|
group :development do
|
@@ -49,6 +51,8 @@ group :development do
|
|
49
51
|
gem 'kramdown-man', '~> 0.1'
|
50
52
|
|
51
53
|
gem 'dead_end', require: false
|
52
|
-
gem 'sord', require: false
|
53
|
-
gem 'stackprof', require: false
|
54
|
+
gem 'sord', require: false, platform: :mri
|
55
|
+
gem 'stackprof', require: false, platform: :mri
|
56
|
+
gem 'rubocop', require: false, platform: :mri
|
57
|
+
gem 'rubocop-ronin', require: false, platform: :mri
|
54
58
|
end
|
data/README.md
CHANGED
@@ -57,6 +57,247 @@ Commands:
|
|
57
57
|
spider
|
58
58
|
```
|
59
59
|
|
60
|
+
Open the `ronin-web` Ruby REPL:
|
61
|
+
|
62
|
+
```shell
|
63
|
+
$ ronin-web irb
|
64
|
+
```
|
65
|
+
|
66
|
+
Diff two HTML files:
|
67
|
+
|
68
|
+
```shell
|
69
|
+
$ ronin-web diff index1.html index2.html
|
70
|
+
+
|
71
|
+
|
72
|
+
+ <div>hax</div>
|
73
|
+
```
|
74
|
+
|
75
|
+
Diff two URLs:
|
76
|
+
|
77
|
+
```shell
|
78
|
+
$ ronin-web diff http://example.com/index.html http://example.com/index2.html
|
79
|
+
```
|
80
|
+
|
81
|
+
Perform an XPath query on an HTML file:
|
82
|
+
|
83
|
+
```shell
|
84
|
+
$ ronin-web html --xpath //meta index.html
|
85
|
+
<meta charset="utf-8">
|
86
|
+
<meta http-equiv="Content-type" content="text/html; charset=utf-8">
|
87
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
88
|
+
```
|
89
|
+
|
90
|
+
Perform an XPath query on a URL:
|
91
|
+
|
92
|
+
```shell
|
93
|
+
$ ronin-web html --xpath //meta https://example.com/
|
94
|
+
<meta charset="utf-8">
|
95
|
+
<meta http-equiv="Content-type" content="text/html; charset=utf-8">
|
96
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
97
|
+
```
|
98
|
+
|
99
|
+
Dump all links from a web page:
|
100
|
+
|
101
|
+
```shell
|
102
|
+
$ ronin-web html --links https://www.google.com/
|
103
|
+
https://www.google.com/imghp?hl=en&tab=wi
|
104
|
+
https://maps.google.com/maps?hl=en&tab=wl
|
105
|
+
https://play.google.com/?hl=en&tab=w8
|
106
|
+
https://www.youtube.com/?tab=w1
|
107
|
+
https://news.google.com/?tab=wn
|
108
|
+
https://mail.google.com/mail/?tab=wm
|
109
|
+
https://drive.google.com/?tab=wo
|
110
|
+
https://www.google.com/intl/en/about/products?tab=wh
|
111
|
+
http://www.google.com/history/optout?hl=en
|
112
|
+
/preferences?hl=en
|
113
|
+
https://accounts.google.com/ServiceLogin?hl=en&passive=true&continue=https://www.google.com/&ec=GAZAAQ
|
114
|
+
/advanced_search?hl=en&authuser=0
|
115
|
+
https://www.google.com/url?q=https://blog.google/products/search/google-search-new-fact-checking-misinformation/&source=hpp&id=19034203&ct=3&usg=AOvVaw3UxG35a-5UX1Rl8M_VwPbd&sa=X&ved=0ahUKEwjM4Iq--JD-AhVtGTQIHXMBBaYQ8IcBCAU
|
116
|
+
/intl/en/ads/
|
117
|
+
/services/
|
118
|
+
/intl/en/about.html
|
119
|
+
/intl/en/policies/privacy/
|
120
|
+
/intl/en/policies/terms/
|
121
|
+
```
|
122
|
+
|
123
|
+
Spiders a host and print all visited URLs:
|
124
|
+
|
125
|
+
```shell
|
126
|
+
$ ronin-web spider --host www.ruby-lang.org
|
127
|
+
http://www.ruby-lang.org/
|
128
|
+
http://www.ruby-lang.org/en/
|
129
|
+
http://www.ruby-lang.org/en/downloads/
|
130
|
+
http://www.ruby-lang.org/en/documentation/
|
131
|
+
http://www.ruby-lang.org/en/libraries/
|
132
|
+
http://www.ruby-lang.org/en/community/
|
133
|
+
https://www.ruby-lang.org/en/news/
|
134
|
+
https://www.ruby-lang.org/en/security/
|
135
|
+
https://www.ruby-lang.org/en/about/
|
136
|
+
...
|
137
|
+
```
|
138
|
+
|
139
|
+
Spiders the domain and sub-domains and print every visited URL:
|
140
|
+
|
141
|
+
```shell
|
142
|
+
$ ronin-web spider --domain ruby-lang.org
|
143
|
+
http://ruby-lang.org/
|
144
|
+
https://www.ruby-lang.org/
|
145
|
+
https://www.ruby-lang.org/en/
|
146
|
+
https://www.ruby-lang.org/en/downloads/
|
147
|
+
https://www.ruby-lang.org/en/documentation/
|
148
|
+
https://www.ruby-lang.org/en/libraries/
|
149
|
+
https://www.ruby-lang.org/en/community/
|
150
|
+
https://www.ruby-lang.org/en/news/
|
151
|
+
https://www.ruby-lang.org/en/security/
|
152
|
+
https://www.ruby-lang.org/en/about/
|
153
|
+
...
|
154
|
+
```
|
155
|
+
|
156
|
+
Spiders a specific web-site and print every visited URL:
|
157
|
+
|
158
|
+
```shell
|
159
|
+
$ ronin-web spider --site https://www.ruby-lang.org/
|
160
|
+
https://www.ruby-lang.org/
|
161
|
+
https://www.ruby-lang.org/en/
|
162
|
+
https://www.ruby-lang.org/en/downloads/
|
163
|
+
https://www.ruby-lang.org/en/documentation/
|
164
|
+
https://www.ruby-lang.org/en/libraries/
|
165
|
+
https://www.ruby-lang.org/en/community/
|
166
|
+
https://www.ruby-lang.org/en/news/
|
167
|
+
https://www.ruby-lang.org/en/security/
|
168
|
+
https://www.ruby-lang.org/en/about/
|
169
|
+
...
|
170
|
+
```
|
171
|
+
|
172
|
+
Spider a host and print the response statuses:
|
173
|
+
|
174
|
+
```shell
|
175
|
+
200 http://www.ruby-lang.org/
|
176
|
+
200 http://www.ruby-lang.org/en/
|
177
|
+
200 http://www.ruby-lang.org/en/downloads/
|
178
|
+
200 http://www.ruby-lang.org/en/documentation/
|
179
|
+
200 http://www.ruby-lang.org/en/libraries/
|
180
|
+
200 http://www.ruby-lang.org/en/community/
|
181
|
+
200 http://www.ruby-lang.org/en/news/
|
182
|
+
200 http://www.ruby-lang.org/en/security/
|
183
|
+
200 http://www.ruby-lang.org/en/about/
|
184
|
+
...
|
185
|
+
```
|
186
|
+
|
187
|
+
Spider a host and print the response status and headers:
|
188
|
+
|
189
|
+
```shell
|
190
|
+
ronin-web spider --print-headers --host www.ruby-lang.org
|
191
|
+
200 http://www.ruby-lang.org/
|
192
|
+
Connection: close
|
193
|
+
Content-Length: 887
|
194
|
+
Server: Cowboy
|
195
|
+
Strict-Transport-Security: max-age=31536000
|
196
|
+
Content-Type: text/html
|
197
|
+
Etag: W/"496ac7fab29a6094e490da28025c5857"
|
198
|
+
X-Frame-Options: SAMEORIGIN
|
199
|
+
Via: 1.1 vegur, 1.1 varnish
|
200
|
+
Accept-Ranges: bytes
|
201
|
+
Date: Tue, 04 Apr 2023 19:42:51 GMT
|
202
|
+
Age: 155
|
203
|
+
X-Served-By: cache-pdx12330-PDX
|
204
|
+
X-Cache: HIT
|
205
|
+
X-Cache-Hits: 1
|
206
|
+
X-Timer: S1680637372.808609,VS0,VE1
|
207
|
+
Vary: Accept-Encoding
|
208
|
+
...
|
209
|
+
```
|
210
|
+
|
211
|
+
Start a debug web server on http://localhost:8000/:
|
212
|
+
|
213
|
+
```shell
|
214
|
+
$ ronin-web server
|
215
|
+
[2023-04-04 12:26:59] INFO WEBrick 1.7.0
|
216
|
+
[2023-04-04 12:26:59] INFO ruby 3.1.3 (2022-11-24) [x86_64-linux]
|
217
|
+
== Sinatra (v3.0.4) has taken the stage on 8000 for development with backup from WEBrick
|
218
|
+
[2023-04-04 12:26:59] INFO WEBrick::HTTPServer#start: pid=8966 port=8000
|
219
|
+
```
|
220
|
+
|
221
|
+
Start a debug web server on a different address and port:
|
222
|
+
|
223
|
+
```shell
|
224
|
+
$ ronin-web server --host 0.0.0.0 --port 1337
|
225
|
+
```
|
226
|
+
|
227
|
+
Host the files in a directory on http://localhost:8000/:
|
228
|
+
|
229
|
+
```shell
|
230
|
+
$ ronin-web server --root .
|
231
|
+
```
|
232
|
+
|
233
|
+
Mount a specific file at a specific HTTP path:
|
234
|
+
|
235
|
+
```shell
|
236
|
+
$ ronin-web server --dir /dir/index.html:./index.html
|
237
|
+
```
|
238
|
+
|
239
|
+
Mount a specific directory at a specific HTTP path:
|
240
|
+
|
241
|
+
```shell
|
242
|
+
$ ronin-web server --dir /dir:.
|
243
|
+
```
|
244
|
+
|
245
|
+
Add a redirect to the web server:
|
246
|
+
|
247
|
+
```shell
|
248
|
+
$ ronin-web server --redirect /redirect:https://example.com/
|
249
|
+
```
|
250
|
+
|
251
|
+
Start a HTTP reverse proxy that rewrites HTTP responses on http://localhost:8080:
|
252
|
+
|
253
|
+
```shell
|
254
|
+
$ ronin-web reverse-proxy --rewrite-response Example:Hax
|
255
|
+
```
|
256
|
+
|
257
|
+
Test the reverse proxy:
|
258
|
+
|
259
|
+
```shell
|
260
|
+
$ curl -H "Host: example.com" http://localhost:8080/
|
261
|
+
```
|
262
|
+
|
263
|
+
Generate a new Ruby script for parsing HTML/XML:
|
264
|
+
|
265
|
+
```shell
|
266
|
+
$ ronin-web new nokogiri parse.rb
|
267
|
+
erb nokogiri.rb.erb parse.rb
|
268
|
+
chmod parse.rb
|
269
|
+
```
|
270
|
+
|
271
|
+
Generate a new web spider script:
|
272
|
+
|
273
|
+
```shell
|
274
|
+
$ ronin-web new spider --host=www.example.com spider.rb
|
275
|
+
erb spider.rb.erb spider.rb
|
276
|
+
chmod spider.rb
|
277
|
+
```
|
278
|
+
|
279
|
+
Generate a new web server script:
|
280
|
+
|
281
|
+
```shell
|
282
|
+
$ ronin-web new server server.rb
|
283
|
+
erb server.rb.erb server.rb
|
284
|
+
chmod server.rb
|
285
|
+
```
|
286
|
+
|
287
|
+
Generate a new web app:
|
288
|
+
|
289
|
+
```shell
|
290
|
+
$ ronin-web new webapp app
|
291
|
+
mkdir app
|
292
|
+
mkdir app/lib
|
293
|
+
mkdir app/views
|
294
|
+
mkdir app/public
|
295
|
+
erb .ruby-version.erb app/.ruby-version
|
296
|
+
cp Gemfile app
|
297
|
+
erb app.rb.erb app/app.rb
|
298
|
+
cp config.ru app
|
299
|
+
```
|
300
|
+
|
60
301
|
## Examples
|
61
302
|
|
62
303
|
Get a web-page:
|
data/Rakefile
CHANGED
data/bin/ronin-web
CHANGED
@@ -1,17 +1,16 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
2
3
|
|
3
4
|
require 'rubygems'
|
4
5
|
|
5
6
|
root = File.expand_path(File.join(File.dirname(__FILE__),'..'))
|
6
7
|
if File.directory?(File.join(root,'.git'))
|
7
8
|
Dir.chdir(root) do
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
exit -1
|
14
|
-
end
|
9
|
+
require 'bundler/setup'
|
10
|
+
rescue LoadError => e
|
11
|
+
warn e.message
|
12
|
+
warn "Run `gem install bundler` to install Bundler"
|
13
|
+
exit(-1)
|
15
14
|
end
|
16
15
|
end
|
17
16
|
|
data/data/new/webapp/Gemfile
CHANGED
@@ -2,5 +2,5 @@ source 'https://rubygems.org'
|
|
2
2
|
|
3
3
|
gem 'puma', '~> 6.0'
|
4
4
|
|
5
|
-
# gem 'ronin-support', '~> 1.0'
|
6
|
-
gem 'ronin-web-server', '~> 0.1'
|
5
|
+
# gem 'ronin-support', '~> 1.0' # , github: 'ronin-rb/ronin-support'
|
6
|
+
gem 'ronin-web-server', '~> 0.1' # , github: 'ronin-rb/ronin-web-server'
|
data/data/new/webapp/config.ru
CHANGED
data/gemspec.yml
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
#
|
2
3
|
# ronin-web - A collection of useful web helper methods and commands.
|
3
4
|
#
|
@@ -24,6 +25,9 @@ require 'ronin/core/cli/command'
|
|
24
25
|
module Ronin
|
25
26
|
module Web
|
26
27
|
class CLI
|
28
|
+
#
|
29
|
+
# Base class for all `ronin-web` commands.
|
30
|
+
#
|
27
31
|
class Command < Core::CLI::Command
|
28
32
|
|
29
33
|
man_dir File.join(ROOT,'man')
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
#
|
2
3
|
# ronin-web - A collection of useful web helper methods and commands.
|
3
4
|
#
|
@@ -42,7 +43,7 @@ module Ronin
|
|
42
43
|
# ## Options
|
43
44
|
#
|
44
45
|
# -h, --help Print help information
|
45
|
-
#
|
46
|
+
#
|
46
47
|
class Diff < Command
|
47
48
|
|
48
49
|
include CommandKit::Terminal
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
#
|
2
3
|
# ronin-web - A collection of useful web helper methods and commands.
|
3
4
|
#
|
@@ -56,7 +57,7 @@ module Ronin
|
|
56
57
|
class Html < Command
|
57
58
|
|
58
59
|
usage '[options] {URL | FILE} [XPATH | CSS-path]'
|
59
|
-
|
60
|
+
|
60
61
|
option :xpath, short: '-X',
|
61
62
|
value: {type: String, usage: 'XPATH'},
|
62
63
|
desc: 'XPath query' do |xpath|
|
@@ -136,7 +137,7 @@ module Ronin
|
|
136
137
|
def run(source,query=@query)
|
137
138
|
unless query
|
138
139
|
print_error "must specify --xpath, --css-path, or an XPath/CSS-path argument"
|
139
|
-
exit
|
140
|
+
exit(-1)
|
140
141
|
end
|
141
142
|
|
142
143
|
doc = Nokogiri::HTML(read(source))
|
@@ -104,10 +104,9 @@ module Ronin
|
|
104
104
|
required: false
|
105
105
|
},
|
106
106
|
desc: 'Spiders a domain' do |value|
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
107
|
+
@entry_point_method = :domain
|
108
|
+
@entry_point_argument = value
|
109
|
+
end
|
111
110
|
|
112
111
|
option :site, equals: true,
|
113
112
|
value: {
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
#
|
2
3
|
# ronin-web - A collection of useful web helper methods and commands.
|
3
4
|
#
|
@@ -181,19 +182,19 @@ module Ronin
|
|
181
182
|
# @return [(Regexp, String), (String, String)]
|
182
183
|
#
|
183
184
|
def parse_rewrite_rule(value)
|
184
|
-
if
|
185
|
+
if (index = value.rindex('/:'))
|
185
186
|
regexp = Regexp.new(value[1...index])
|
186
|
-
replace = value[index+2..]
|
187
|
+
replace = value[(index + 2)..]
|
187
188
|
|
188
189
|
return [regexp, replace]
|
189
190
|
elsif (index = value.rindex(':'))
|
190
191
|
string = value[0...index]
|
191
|
-
replace = value[(index+1)..]
|
192
|
+
replace = value[(index + 1)..]
|
192
193
|
|
193
194
|
return [string, replace]
|
194
195
|
end
|
195
196
|
end
|
196
|
-
|
197
|
+
|
197
198
|
#
|
198
199
|
# Rewrites a request or response body.
|
199
200
|
#
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
#
|
2
3
|
# ronin-web - A collection of useful web helper methods and commands.
|
3
4
|
#
|
@@ -42,7 +43,7 @@ module Ronin
|
|
42
43
|
# -r, --root DIR Root directory to serve
|
43
44
|
# -R, --redirect /PATH:URL Registers a 302 Found redirect at the given PATH
|
44
45
|
# -h, --help Print help information
|
45
|
-
#
|
46
|
+
#
|
46
47
|
class Server < Command
|
47
48
|
|
48
49
|
include Core::CLI::Logging
|
@@ -57,7 +58,7 @@ module Ronin
|
|
57
58
|
default: 'localhost'
|
58
59
|
},
|
59
60
|
desc: 'Host name or IP to bind to' do |host|
|
60
|
-
App.
|
61
|
+
App.bind = host
|
61
62
|
end
|
62
63
|
|
63
64
|
option :port, short: '-p',
|
@@ -143,7 +144,7 @@ module Ronin
|
|
143
144
|
end
|
144
145
|
end
|
145
146
|
|
146
|
-
log_info "Starting web server listening on #{App.
|
147
|
+
log_info "Starting web server listening on #{App.bind}:#{App.port} ..."
|
147
148
|
begin
|
148
149
|
App.run!
|
149
150
|
rescue Errno::EADDRINUSE => error
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
#
|
2
3
|
# ronin-web - A collection of useful web helper methods and commands.
|
3
4
|
#
|
@@ -119,18 +120,18 @@ module Ronin
|
|
119
120
|
desc: 'Sets the read timeout'
|
120
121
|
|
121
122
|
option :ssl_timeout, value: {
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
123
|
+
type: Integer,
|
124
|
+
usage: 'SECS',
|
125
|
+
default: Spidr.ssl_timeout
|
126
|
+
},
|
127
|
+
desc: 'Sets the SSL connection timeout'
|
127
128
|
|
128
129
|
option :continue_timeout, value: {
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
130
|
+
type: Integer,
|
131
|
+
usage: 'SECS',
|
132
|
+
default: Spidr.continue_timeout
|
133
|
+
},
|
134
|
+
desc: 'Sets the continue timeout'
|
134
135
|
|
135
136
|
option :keep_alive_timeout, value: {
|
136
137
|
type: Integer,
|
@@ -168,14 +169,14 @@ module Ronin
|
|
168
169
|
end
|
169
170
|
|
170
171
|
option :user_agent, value: {
|
171
|
-
type:
|
172
|
+
type: String,
|
172
173
|
usage: 'USER-AGENT'
|
173
174
|
},
|
174
175
|
desc: 'Sets the User-Agent string'
|
175
176
|
|
176
177
|
option :user_agent_string, short: '-U',
|
177
178
|
value: {
|
178
|
-
type:
|
179
|
+
type: String,
|
179
180
|
usage: 'STRING'
|
180
181
|
},
|
181
182
|
desc: 'The User-Agent string to use' do |ua|
|
@@ -184,11 +185,9 @@ module Ronin
|
|
184
185
|
|
185
186
|
option :user_agent, short: '-u',
|
186
187
|
value: {
|
187
|
-
type:
|
188
|
-
|
189
|
-
|
190
|
-
}
|
191
|
-
]
|
188
|
+
type: Support::Network::HTTP::UserAgents::ALIASES.transform_keys { |key|
|
189
|
+
key.to_s.tr('_','-')
|
190
|
+
}
|
192
191
|
},
|
193
192
|
desc: 'The User-Agent to use' do |name|
|
194
193
|
@user_agent = name
|
@@ -196,34 +195,34 @@ module Ronin
|
|
196
195
|
|
197
196
|
option :referer, short: '-R',
|
198
197
|
value: {
|
199
|
-
type:
|
198
|
+
type: String,
|
200
199
|
usage: 'URL'
|
201
200
|
},
|
202
201
|
desc: 'Sets the Referer URL'
|
203
202
|
|
204
203
|
option :delay, short: '-d',
|
205
204
|
value: {
|
206
|
-
type:
|
205
|
+
type: Numeric,
|
207
206
|
usage: 'SECS'
|
208
207
|
},
|
209
208
|
desc: 'Sets the delay in seconds between each request'
|
210
209
|
|
211
210
|
option :limit, short: '-l',
|
212
211
|
value: {
|
213
|
-
type:
|
212
|
+
type: Integer,
|
214
213
|
usage: 'COUNT'
|
215
214
|
},
|
216
215
|
desc: 'Only spiders up to COUNT pages'
|
217
216
|
|
218
217
|
option :max_depth, short: '-d',
|
219
218
|
value: {
|
220
|
-
type:
|
221
|
-
usage: 'DEPTH'
|
219
|
+
type: Integer,
|
220
|
+
usage: 'DEPTH'
|
222
221
|
},
|
223
222
|
desc: 'Only spiders up to max depth'
|
224
223
|
|
225
224
|
option :enqueue, value: {
|
226
|
-
type:
|
225
|
+
type: String,
|
227
226
|
usage: 'URL'
|
228
227
|
},
|
229
228
|
desc: 'Adds the URL to the queue' do |url|
|
@@ -231,7 +230,7 @@ module Ronin
|
|
231
230
|
end
|
232
231
|
|
233
232
|
option :visited, value: {
|
234
|
-
type:
|
233
|
+
type: String,
|
235
234
|
usage: 'URL'
|
236
235
|
},
|
237
236
|
desc: 'Marks the URL as previously visited' do |url|
|
@@ -243,7 +242,7 @@ module Ronin
|
|
243
242
|
option :strip_query, desc: 'Enables/disables stripping the query component of every URL'
|
244
243
|
|
245
244
|
option :visit_host, value: {
|
246
|
-
type:
|
245
|
+
type: String,
|
247
246
|
usage: 'HOST'
|
248
247
|
},
|
249
248
|
desc: 'Visit URLs with the matching host name' do |host|
|
@@ -267,15 +266,15 @@ module Ronin
|
|
267
266
|
end
|
268
267
|
|
269
268
|
option :ignore_hosts_like, value: {
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
269
|
+
type: Regexp,
|
270
|
+
usage: '/REGEX/'
|
271
|
+
},
|
272
|
+
desc: 'Ignore the host names matching the REGEX' do |regex|
|
273
|
+
@ignore_hosts << regex
|
274
|
+
end
|
276
275
|
|
277
276
|
option :visit_port, value: {
|
278
|
-
type:
|
277
|
+
type: Integer,
|
279
278
|
usage: 'PORT'
|
280
279
|
},
|
281
280
|
desc: 'Visit URLs with the matching port number' do |port|
|
@@ -299,15 +298,15 @@ module Ronin
|
|
299
298
|
end
|
300
299
|
|
301
300
|
option :ignore_ports_like, value: {
|
302
|
-
|
303
|
-
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
|
301
|
+
type: Regexp,
|
302
|
+
usage: '/REGEX/'
|
303
|
+
},
|
304
|
+
desc: 'Ignore the port numbers matching the REGEXP' do |regex|
|
305
|
+
@ignore_ports << regex
|
306
|
+
end
|
308
307
|
|
309
308
|
option :visit_link, value: {
|
310
|
-
type:
|
309
|
+
type: String,
|
311
310
|
usage: 'URL'
|
312
311
|
},
|
313
312
|
desc: 'Visit the URL' do |link|
|
@@ -331,36 +330,36 @@ module Ronin
|
|
331
330
|
end
|
332
331
|
|
333
332
|
option :ignore_links_like, value: {
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
333
|
+
type: Regexp,
|
334
|
+
usage: '/REGEX/'
|
335
|
+
},
|
336
|
+
desc: 'Ignore URLs matching the REGEX' do |regex|
|
337
|
+
@ignore_links << regex
|
338
|
+
end
|
340
339
|
|
341
340
|
option :visit_ext, value: {
|
342
|
-
|
343
|
-
|
344
|
-
|
345
|
-
|
346
|
-
|
347
|
-
|
341
|
+
type: String,
|
342
|
+
usage: 'FILE_EXT'
|
343
|
+
},
|
344
|
+
desc: 'Visit URLs with the matching file ext' do |ext|
|
345
|
+
@visit_exts << ext
|
346
|
+
end
|
348
347
|
|
349
348
|
option :visit_exts_like, value: {
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
349
|
+
type: Regexp,
|
350
|
+
usage: '/REGEX/'
|
351
|
+
},
|
352
|
+
desc: 'Visit URLs with file exts that match the REGEX' do |regex|
|
353
|
+
@visit_exts << regex
|
354
|
+
end
|
356
355
|
|
357
356
|
option :ignore_ext, value: {
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
357
|
+
type: String,
|
358
|
+
usage: 'FILE_EXT'
|
359
|
+
},
|
360
|
+
desc: 'Ignore the URLs with the file ext' do |ext|
|
361
|
+
@ignore_exts << ext
|
362
|
+
end
|
364
363
|
|
365
364
|
option :ignore_exts_like, value: {
|
366
365
|
type: Regexp,
|
@@ -374,47 +373,47 @@ module Ronin
|
|
374
373
|
desc: 'Specifies whether to honor robots.txt'
|
375
374
|
|
376
375
|
option :host, value: {
|
377
|
-
type:
|
376
|
+
type: String,
|
378
377
|
usage: 'HOST'
|
379
378
|
},
|
380
379
|
desc: 'Spiders the specific HOST'
|
381
380
|
|
382
381
|
option :domain, value: {
|
383
|
-
type:
|
384
|
-
usage: 'DOMAIN'
|
382
|
+
type: String,
|
383
|
+
usage: 'DOMAIN'
|
385
384
|
},
|
386
385
|
desc: 'Spiders the whole domain'
|
387
386
|
|
388
387
|
option :site, value: {
|
389
|
-
type:
|
388
|
+
type: String,
|
390
389
|
usage: 'URL'
|
391
390
|
},
|
392
391
|
desc: 'Spiders the website, starting at the URL'
|
393
392
|
|
394
|
-
option :
|
393
|
+
option :print_status, desc: 'Print the status codes for each URL'
|
395
394
|
|
396
395
|
option :print_headers, desc: 'Print response headers for each URL'
|
397
396
|
|
398
397
|
option :print_header, value: {
|
399
|
-
type:
|
398
|
+
type: String,
|
400
399
|
usage: 'NAME'
|
401
400
|
},
|
402
401
|
desc: 'Prints a specific header'
|
403
402
|
|
404
403
|
option :history, value: {
|
405
|
-
type:
|
404
|
+
type: String,
|
406
405
|
usage: 'FILE'
|
407
406
|
},
|
408
407
|
desc: 'The history file'
|
409
408
|
|
410
409
|
option :archive, value: {
|
411
|
-
type:
|
410
|
+
type: String,
|
412
411
|
usage: 'DIR'
|
413
412
|
},
|
414
413
|
desc: 'Archive every visited page to the DIR'
|
415
414
|
|
416
415
|
option :git_archive, value: {
|
417
|
-
type:
|
416
|
+
type: String,
|
418
417
|
usage: 'DIR'
|
419
418
|
},
|
420
419
|
desc: 'Archive every visited page to the git repository'
|
@@ -588,6 +587,8 @@ module Ronin
|
|
588
587
|
end
|
589
588
|
end
|
590
589
|
|
590
|
+
# post-spidering tasks
|
591
|
+
|
591
592
|
if options[:git_archive]
|
592
593
|
archive.commit "Updated #{Time.now}"
|
593
594
|
end
|
@@ -699,6 +700,7 @@ module Ronin
|
|
699
700
|
#
|
700
701
|
def agent_kwargs
|
701
702
|
kwargs = {}
|
703
|
+
|
702
704
|
kwargs[:proxy] = options[:proxy] if options[:proxy]
|
703
705
|
|
704
706
|
unless @default_headers.empty?
|
data/lib/ronin/web/cli.rb
CHANGED
data/lib/ronin/web/html.rb
CHANGED
data/lib/ronin/web/mechanize.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
#
|
2
3
|
# ronin-web - A collection of useful web helper methods and commands.
|
3
4
|
#
|
@@ -31,7 +32,9 @@ module Ronin
|
|
31
32
|
class Mechanize < ::Mechanize
|
32
33
|
|
33
34
|
#
|
34
|
-
# Creates a new [Mechanize]
|
35
|
+
# Creates a new [Mechanize] Agent.
|
36
|
+
#
|
37
|
+
# [Mechanize]: https://github.com/sparklemotion/mechanize#readme
|
35
38
|
#
|
36
39
|
# @param [Network::HTTP::Proxy, Hash, String] proxy
|
37
40
|
# Proxy information.
|
@@ -56,6 +59,8 @@ module Ronin
|
|
56
59
|
user_agent: Support::Network::HTTP.user_agent)
|
57
60
|
super()
|
58
61
|
|
62
|
+
self.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
63
|
+
|
59
64
|
if proxy
|
60
65
|
proxy = URI(proxy)
|
61
66
|
|
data/lib/ronin/web/root.rb
CHANGED
data/lib/ronin/web/version.rb
CHANGED
data/lib/ronin/web/xml.rb
CHANGED
data/lib/ronin/web.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
#
|
2
3
|
# ronin-web - A collection of useful web helper methods and commands.
|
3
4
|
#
|
@@ -34,6 +35,9 @@ require 'nokogiri/diff'
|
|
34
35
|
require 'open_namespace'
|
35
36
|
|
36
37
|
module Ronin
|
38
|
+
#
|
39
|
+
# Top-level web helper methods.
|
40
|
+
#
|
37
41
|
module Web
|
38
42
|
include OpenNamespace
|
39
43
|
|
@@ -200,6 +204,7 @@ module Ronin
|
|
200
204
|
referer: nil,
|
201
205
|
cookie: nil,
|
202
206
|
**kwargs)
|
207
|
+
uri = URI(url)
|
203
208
|
options = {proxy: proxy, **kwargs}
|
204
209
|
|
205
210
|
if user_agent
|
@@ -214,7 +219,7 @@ module Ronin
|
|
214
219
|
options['Referer'] = referer if referer
|
215
220
|
options['Cookie'] = cookie if cookie
|
216
221
|
|
217
|
-
return
|
222
|
+
return uri.open(options)
|
218
223
|
end
|
219
224
|
|
220
225
|
#
|
@@ -344,7 +349,7 @@ module Ronin
|
|
344
349
|
# @api public
|
345
350
|
#
|
346
351
|
def self.post(url,query={},headers={},&block)
|
347
|
-
agent.post(url,query,headers
|
352
|
+
agent.post(url,query,headers,&block)
|
348
353
|
end
|
349
354
|
|
350
355
|
#
|
data/ronin-web.gemspec
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
#
|
1
|
+
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'yaml'
|
4
4
|
|
@@ -21,10 +21,10 @@ Gem::Specification.new do |gem|
|
|
21
21
|
gem.email = gemspec['email']
|
22
22
|
gem.homepage = gemspec['homepage']
|
23
23
|
|
24
|
-
glob =
|
24
|
+
glob = ->(patterns) { gem.files & Dir[*patterns] }
|
25
25
|
|
26
|
-
gem.files
|
27
|
-
gem.files
|
26
|
+
gem.files = `git ls-files`.split($/)
|
27
|
+
gem.files = glob[gemspec['files']] if gemspec['files']
|
28
28
|
gem.files += Array(gemspec['generated_files'])
|
29
29
|
# exclude test files from the packages gem
|
30
30
|
gem.files -= glob[gemspec['test_files'] || 'spec/{**/}*']
|
@@ -32,6 +32,7 @@ Gem::Specification.new do |gem|
|
|
32
32
|
gem.executables = gemspec.fetch('executables') do
|
33
33
|
glob['bin/*'].map { |path| File.basename(path) }
|
34
34
|
end
|
35
|
+
|
35
36
|
gem.default_executable = gem.executables.first if Gem::VERSION < '1.7.'
|
36
37
|
|
37
38
|
gem.extensions = glob[gemspec['extensions'] || 'ext/**/extconf.rb']
|
@@ -46,7 +47,7 @@ Gem::Specification.new do |gem|
|
|
46
47
|
gem.required_rubygems_version = gemspec['required_rubygems_version']
|
47
48
|
gem.post_install_message = gemspec['post_install_message']
|
48
49
|
|
49
|
-
split =
|
50
|
+
split = ->(string) { string.split(/,\s*/) }
|
50
51
|
|
51
52
|
if gemspec['dependencies']
|
52
53
|
gemspec['dependencies'].each do |name,versions|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ronin-web
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Postmodern
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-
|
11
|
+
date: 2023-04-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: nokogiri
|
@@ -101,6 +101,9 @@ dependencies:
|
|
101
101
|
- - "~>"
|
102
102
|
- !ruby/object:Gem::Version
|
103
103
|
version: '0.1'
|
104
|
+
- - ">="
|
105
|
+
- !ruby/object:Gem::Version
|
106
|
+
version: 0.1.1
|
104
107
|
type: :runtime
|
105
108
|
prerelease: false
|
106
109
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -108,6 +111,9 @@ dependencies:
|
|
108
111
|
- - "~>"
|
109
112
|
- !ruby/object:Gem::Version
|
110
113
|
version: '0.1'
|
114
|
+
- - ">="
|
115
|
+
- !ruby/object:Gem::Version
|
116
|
+
version: 0.1.1
|
111
117
|
- !ruby/object:Gem::Dependency
|
112
118
|
name: ronin-web-spider
|
113
119
|
requirement: !ruby/object:Gem::Requirement
|
@@ -181,6 +187,7 @@ files:
|
|
181
187
|
- ".gitignore"
|
182
188
|
- ".mailmap"
|
183
189
|
- ".rspec"
|
190
|
+
- ".rubocop.yml"
|
184
191
|
- ".ruby-version"
|
185
192
|
- ".yardopts"
|
186
193
|
- COPYING.txt
|