ronin-web 1.0.2 → 2.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ruby.yml +3 -2
- data/.gitignore +1 -0
- data/.rubocop.yml +5 -0
- data/.ruby-version +1 -1
- data/ChangeLog.md +46 -1
- data/Gemfile +25 -12
- data/README.md +257 -51
- data/Rakefile +9 -0
- data/data/completions/ronin-web +203 -0
- data/gemspec.yml +18 -5
- data/lib/ronin/web/cli/browser_options.rb +92 -0
- data/lib/ronin/web/cli/browser_shell.rb +448 -0
- data/lib/ronin/web/cli/command.rb +1 -1
- data/lib/ronin/web/cli/commands/browser.rb +373 -0
- data/lib/ronin/web/cli/commands/completion.rb +63 -0
- data/lib/ronin/web/cli/commands/diff.rb +60 -8
- data/lib/ronin/web/cli/commands/html.rb +21 -33
- data/lib/ronin/web/cli/commands/irb.rb +1 -1
- data/lib/ronin/web/cli/commands/new/{webapp.rb → app.rb} +8 -8
- data/lib/ronin/web/cli/commands/new/nokogiri.rb +4 -4
- data/lib/ronin/web/cli/commands/new/server.rb +1 -1
- data/lib/ronin/web/cli/commands/new/spider.rb +1 -1
- data/lib/ronin/web/cli/commands/new.rb +5 -3
- data/lib/ronin/web/cli/commands/reverse_proxy.rb +1 -1
- data/lib/ronin/web/cli/commands/screenshot.rb +186 -0
- data/lib/ronin/web/cli/commands/server.rb +1 -1
- data/lib/ronin/web/cli/commands/session_cookie.rb +265 -0
- data/lib/ronin/web/cli/commands/spider.rb +61 -467
- data/lib/ronin/web/cli/commands/user_agent.rb +177 -0
- data/lib/ronin/web/cli/commands/vulns.rb +463 -0
- data/lib/ronin/web/cli/commands/wordlist.rb +484 -0
- data/lib/ronin/web/cli/commands/xml.rb +149 -0
- data/lib/ronin/web/cli/js_shell.rb +69 -0
- data/lib/ronin/web/cli/ruby_shell.rb +1 -1
- data/lib/ronin/web/cli/spider_options.rb +919 -0
- data/lib/ronin/web/cli.rb +3 -1
- data/lib/ronin/web/html.rb +1 -1
- data/lib/ronin/web/root.rb +1 -1
- data/lib/ronin/web/version.rb +2 -2
- data/lib/ronin/web/xml.rb +1 -1
- data/lib/ronin/web.rb +4 -364
- data/man/ronin-web-browser.1 +92 -0
- data/man/ronin-web-browser.1.md +96 -0
- data/man/ronin-web-completion.1 +76 -0
- data/man/ronin-web-completion.1.md +78 -0
- data/man/ronin-web-diff.1 +14 -21
- data/man/ronin-web-diff.1.md +13 -6
- data/man/ronin-web-html.1 +30 -46
- data/man/ronin-web-html.1.md +27 -17
- data/man/ronin-web-irb.1 +9 -16
- data/man/ronin-web-irb.1.md +6 -2
- data/man/ronin-web-new-app.1.md +39 -0
- data/man/ronin-web-new-nokogiri.1 +9 -20
- data/man/ronin-web-new-nokogiri.1.md +5 -5
- data/man/ronin-web-new-server.1 +11 -23
- data/man/ronin-web-new-server.1.md +5 -5
- data/man/ronin-web-new-spider.1 +44 -88
- data/man/ronin-web-new-spider.1.md +37 -37
- data/man/ronin-web-new.1 +18 -30
- data/man/ronin-web-new.1.md +15 -11
- data/man/ronin-web-reverse-proxy.1 +33 -38
- data/man/ronin-web-reverse-proxy.1.md +20 -14
- data/man/ronin-web-screenshot.1 +56 -0
- data/man/ronin-web-screenshot.1.md +56 -0
- data/man/ronin-web-server.1 +15 -29
- data/man/ronin-web-server.1.md +13 -9
- data/man/ronin-web-session-cookie.1 +38 -0
- data/man/ronin-web-session-cookie.1.md +41 -0
- data/man/ronin-web-spider.1 +121 -130
- data/man/ronin-web-spider.1.md +115 -66
- data/man/ronin-web-user-agent.1 +44 -0
- data/man/ronin-web-user-agent.1.md +46 -0
- data/man/ronin-web-vulns.1 +175 -0
- data/man/ronin-web-vulns.1.md +177 -0
- data/man/ronin-web-wordlist.1 +258 -0
- data/man/ronin-web-wordlist.1.md +263 -0
- data/man/ronin-web-xml.1 +43 -0
- data/man/ronin-web-xml.1.md +46 -0
- data/man/ronin-web.1 +67 -18
- data/man/ronin-web.1.md +55 -4
- data/scripts/setup +58 -0
- metadata +121 -30
- data/lib/ronin/web/mechanize.rb +0 -84
- data/man/ronin-web-new-webapp.1.md +0 -39
- /data/data/new/{webapp → app}/.gitignore +0 -0
- /data/data/new/{webapp → app}/.ruby-version.erb +0 -0
- /data/data/new/{webapp → app}/Dockerfile.erb +0 -0
- /data/data/new/{webapp → app}/Gemfile +0 -0
- /data/data/new/{webapp → app}/app.rb.erb +0 -0
- /data/data/new/{webapp → app}/config.ru +0 -0
- /data/data/new/{webapp → app}/docker-compose.yml.erb +0 -0
data/man/ronin-web-spider.1
CHANGED
@@ -1,229 +1,220 @@
|
|
1
|
-
.\" Generated by kramdown-man 0.1
|
1
|
+
.\" Generated by kramdown-man 1.0.1
|
2
2
|
.\" https://github.com/postmodern/kramdown-man#readme
|
3
3
|
.TH ronin-web-spider 1 "2022-01-01" Ronin Web "User Manuals"
|
4
|
-
.
|
4
|
+
.SH NAME
|
5
|
+
.PP
|
6
|
+
ronin\-web\-spider \- Spiders a website
|
5
7
|
.SH SYNOPSIS
|
6
|
-
.
|
7
|
-
|
8
|
-
\fBronin-web-spider\fR \[lB]\fIoptions\fP\[rB] \[lC]\fB--host\fR \fIHOST\fP \[or] \fB--domain\fR \fIDOMAIN\fP \[or] \fB--site\fR \fIURL\fP\[rC]
|
9
|
-
.LP
|
8
|
+
.PP
|
9
|
+
\fBronin\-web spider\fR \[lB]\fIoptions\fP\[rB] \[lC]\fB\-\-host\fR \fIHOST\fP \[or] \fB\-\-domain\fR \fIDOMAIN\fP \[or] \fB\-\-site\fR \fIURL\fP\[rC]
|
10
10
|
.SH DESCRIPTION
|
11
|
-
.LP
|
12
11
|
.PP
|
13
12
|
Spiders a website\.
|
14
|
-
.LP
|
15
13
|
.SH OPTIONS
|
16
|
-
.LP
|
17
14
|
.TP
|
18
|
-
\fB
|
15
|
+
\fB\-\-host\fR \fIHOST\fP
|
16
|
+
Spiders the specific \fIHOST\fP\.
|
17
|
+
.TP
|
18
|
+
\fB\-\-domain\fR \fIDOMAIN\fP
|
19
|
+
Spiders the whole \fIDOMAIN\fP\.
|
20
|
+
.TP
|
21
|
+
\fB\-\-site\fR \fIURL\fP
|
22
|
+
Spiders the website, starting at the \fIURL\fP\.
|
23
|
+
.TP
|
24
|
+
\fB\-\-open\-timeout\fR \fISECS\fP
|
19
25
|
Sets the connection open timeout\.
|
20
|
-
.LP
|
21
26
|
.TP
|
22
|
-
\fB
|
27
|
+
\fB\-\-read\-timeout\fR \fISECS\fP
|
23
28
|
Sets the read timeout\.
|
24
|
-
.LP
|
25
29
|
.TP
|
26
|
-
\fB
|
30
|
+
\fB\-\-ssl\-timeout\fR \fISECS\fP
|
27
31
|
Sets the SSL connection timeout\.
|
28
|
-
.LP
|
29
32
|
.TP
|
30
|
-
\fB
|
33
|
+
\fB\-\-continue\-timeout\fR \fISECS\fP
|
31
34
|
Sets the continue timeout\.
|
32
|
-
.LP
|
33
35
|
.TP
|
34
|
-
\fB
|
36
|
+
\fB\-\-keep\-alive\-timeout\fR \fISECS\fP
|
35
37
|
Sets the connection keep alive timeout\.
|
36
|
-
.LP
|
37
38
|
.TP
|
38
|
-
\fB
|
39
|
+
\fB\-P\fR, \fB\-\-proxy\fR \fIPROXY\fP
|
39
40
|
Sets the proxy to use\.
|
40
|
-
.LP
|
41
41
|
.TP
|
42
|
-
\fB
|
42
|
+
\fB\-H\fR, \fB\-\-header\fR \[lq]\fINAME\fP: \fIVALUE\fP\[rq]
|
43
43
|
Sets a default header\.
|
44
|
-
.LP
|
45
44
|
.TP
|
46
|
-
\fB
|
45
|
+
\fB\-\-host\-header\fR \fINAME\fP\[eq]\fIVALUE\fP
|
47
46
|
Sets a default header\.
|
48
|
-
.LP
|
49
|
-
.HP
|
50
|
-
\fB-u\fR, \fB--user-agent\fR chrome\-linux\[or]chrome\-macos\[or]chrome\-windows\[or]chrome\-iphone\[or]chrome\-ipad\[or]chrome\-android\[or]firefox\-linux\[or]firefox\-macos\[or]firefox\-windows\[or]firefox\-iphone\[or]firefox\-ipad\[or]firefox\-android\[or]safari\-macos\[or]safari\-iphone\[or]safari\-ipad\[or]edge
|
51
|
-
The \fBUser-Agent\fR to use\.
|
52
|
-
.LP
|
53
47
|
.TP
|
54
|
-
\fB
|
55
|
-
The
|
56
|
-
.LP
|
48
|
+
\fB\-u\fR, \fB\-\-user\-agent\fR chrome\-linux\[or]chrome\-macos\[or]chrome\-windows\[or]chrome\-iphone\[or]chrome\-ipad\[or]chrome\-android\[or]firefox\-linux\[or]firefox\-macos\[or]firefox\-windows\[or]firefox\-iphone\[or]firefox\-ipad\[or]firefox\-android\[or]safari\-macos\[or]safari\-iphone\[or]safari\-ipad\[or]edge
|
49
|
+
The \fBUser\-Agent\fR to use\.
|
57
50
|
.TP
|
58
|
-
\fB
|
51
|
+
\fB\-U\fR, \fB\-\-user\-agent\-string\fR \fISTRING\fP
|
52
|
+
The raw \fBUser\-Agent\fR string to use\.
|
53
|
+
.TP
|
54
|
+
\fB\-R\fR, \fB\-\-referer\fR \fIURL\fP
|
59
55
|
Sets the \fBReferer\fR URL\.
|
60
|
-
.LP
|
61
56
|
.TP
|
62
|
-
\fB
|
57
|
+
\fB\-\-delay\fR \fISECS\fP
|
63
58
|
Sets the delay in seconds between each request\.
|
64
|
-
.LP
|
65
59
|
.TP
|
66
|
-
\fB
|
60
|
+
\fB\-l\fR, \fB\-\-limit\fR \fICOUNT\fP
|
67
61
|
Only spiders up to \fICOUNT\fP pages\.
|
68
|
-
.LP
|
69
62
|
.TP
|
70
|
-
\fB
|
63
|
+
\fB\-d\fR, \fB\-\-max\-depth\fR \fIDEPTH\fP
|
71
64
|
Only spiders up to max depth\.
|
72
|
-
.LP
|
73
65
|
.TP
|
74
|
-
\fB
|
66
|
+
\fB\-\-enqueue\fR \fIURL\fP
|
75
67
|
Adds the URL to the queue\.
|
76
|
-
.LP
|
77
68
|
.TP
|
78
|
-
\fB
|
69
|
+
\fB\-\-visited\fR \fIURL\fP
|
79
70
|
Marks the URL as previously visited\.
|
80
|
-
.LP
|
81
71
|
.TP
|
82
|
-
\fB
|
72
|
+
\fB\-\-strip\-fragments\fR
|
83
73
|
Enables\[sl]disables stripping the fragment component of every URL\.
|
84
|
-
.LP
|
85
74
|
.TP
|
86
|
-
\fB
|
75
|
+
\fB\-\-strip\-query\fR
|
87
76
|
Enables\[sl]disables stripping the query component of every URL\.
|
88
|
-
.LP
|
89
77
|
.TP
|
90
|
-
\fB
|
78
|
+
\fB\-\-visit\-scheme\fR \fISCHEME\fP
|
79
|
+
Visit URLs with the URI scheme\.
|
80
|
+
.TP
|
81
|
+
\fB\-\-visit\-schemes\-like\fR \fB\[sl]\fR\fIREGEX\fP\fB\[sl]\fR
|
82
|
+
Visit URLs with URI schemes that match the \fIREGEX\fP\.
|
83
|
+
.TP
|
84
|
+
\fB\-\-ignore\-scheme\fR \fISCHEME\fP
|
85
|
+
Ignore URLs with the URI scheme\.
|
86
|
+
.TP
|
87
|
+
\fB\-\-ignore\-schemes\-like\fR \fB\[sl]\fR\fIREGEX\fP\fB\[sl]\fR
|
88
|
+
Ignore URLs with URI schemes matching the \fIREGEX\fP\.
|
89
|
+
.TP
|
90
|
+
\fB\-\-visit\-host\fR \fIHOST\fP
|
91
91
|
Visit URLs with the matching host name\.
|
92
|
-
.
|
93
|
-
|
94
|
-
\fB--visit-hosts-like\fR \[sl]\fIREGEX\fP\[sl]
|
92
|
+
.TP
|
93
|
+
\fB\-\-visit\-hosts\-like\fR \fB\[sl]\fR\fIREGEX\fP\fB\[sl]\fR
|
95
94
|
Visit URLs with hostnames that match the \fIREGEX\fP\.
|
96
|
-
.LP
|
97
95
|
.TP
|
98
|
-
\fB
|
96
|
+
\fB\-\-ignore\-host\fR \fIHOST\fP
|
99
97
|
Ignore the host name\.
|
100
|
-
.
|
101
|
-
|
102
|
-
\fB--ignore-hosts-like\fR \[sl]\fIREGEX\fP\[sl]
|
98
|
+
.TP
|
99
|
+
\fB\-\-ignore\-hosts\-like\fR \fB\[sl]\fR\fIREGEX\fP\fB\[sl]\fR
|
103
100
|
Ignore the host names matching the \fIREGEX\fP\.
|
104
|
-
.LP
|
105
101
|
.TP
|
106
|
-
\fB
|
102
|
+
\fB\-\-visit\-port\fR \fIPORT\fP
|
107
103
|
Visit URLs with the matching port number\.
|
108
|
-
.
|
109
|
-
|
110
|
-
\fB--visit-ports-like\fR \[sl]\fIREGEX\fP\[sl]
|
104
|
+
.TP
|
105
|
+
\fB\-\-visit\-ports\-like\fR \fB\[sl]\fR\fIREGEX\fP\fB\[sl]\fR
|
111
106
|
Visit URLs with port numbers that match the \fIREGEX\fP\.
|
112
|
-
.LP
|
113
107
|
.TP
|
114
|
-
\fB
|
108
|
+
\fB\-\-ignore\-port\fR \fIPORT\fP
|
115
109
|
Ignore the port number\.
|
116
|
-
.
|
117
|
-
|
118
|
-
\fB--ignore-ports-like\fR \[sl]\fIREGEX\fP\[sl]
|
110
|
+
.TP
|
111
|
+
\fB\-\-ignore\-ports\-like\fR \fB\[sl]\fR\fIREGEX\fP\fB\[sl]\fR
|
119
112
|
Ignore the port numbers matching the \fIREGEXP\fP\.
|
120
|
-
.LP
|
121
113
|
.TP
|
122
|
-
\fB
|
114
|
+
\fB\-\-visit\-link\fR \fIURL\fP
|
123
115
|
Visit the \fIURL\fP\.
|
124
|
-
.
|
125
|
-
|
126
|
-
\fB--visit-links-like\fR \[sl]\fIREGEX\fP\[sl]
|
116
|
+
.TP
|
117
|
+
\fB\-\-visit\-links\-like\fR \fB\[sl]\fR\fIREGEX\fP\fB\[sl]\fR
|
127
118
|
Visit URLs that match the \fIREGEX\fP\.
|
128
|
-
.LP
|
129
119
|
.TP
|
130
|
-
\fB
|
120
|
+
\fB\-\-ignore\-link\fR \fIURL\fP
|
131
121
|
Ignore the \fIURL\fP\.
|
132
|
-
.
|
133
|
-
|
134
|
-
\fB--ignore-links-like\fR \[sl]\fIREGEX\fP\[sl]
|
122
|
+
.TP
|
123
|
+
\fB\-\-ignore\-links\-like\fR \fB\[sl]\fR\fIREGEX\fP\fB\[sl]\fR
|
135
124
|
Ignore URLs matching the \fIREGEX\fP\.
|
136
|
-
.LP
|
137
125
|
.TP
|
138
|
-
\fB
|
126
|
+
\fB\-\-visit\-ext\fR \fIFILE\[ru]EXT\fP
|
139
127
|
Visit URLs with the matching file ext\.
|
140
|
-
.
|
141
|
-
|
142
|
-
\fB--visit-exts-like\fR \[sl]\fIREGEX\fP\[sl]
|
128
|
+
.TP
|
129
|
+
\fB\-\-visit\-exts\-like\fR \fB\[sl]\fR\fIREGEX\fP\fB\[sl]\fR
|
143
130
|
Visit URLs with file exts that match the \fIREGEX\fP\.
|
144
|
-
.LP
|
145
131
|
.TP
|
146
|
-
\fB
|
132
|
+
\fB\-\-ignore\-ext\fR \fIFILE\[ru]EXT\fP
|
147
133
|
Ignore the URLs with the file ext\.
|
148
|
-
.LP
|
149
|
-
.HP
|
150
|
-
\fB--ignore-exts-like\fR \[sl]\fIREGEX\fP\[sl]
|
151
|
-
Ignore URLs with file exts matching the REGEX\.
|
152
|
-
.LP
|
153
|
-
.TP
|
154
|
-
\fB-r\fR, \fB--robots\fR
|
155
|
-
Specifies whether to honor \fBrobots.txt\fR\.
|
156
|
-
.LP
|
157
|
-
.TP
|
158
|
-
\fB--host\fR \fIHOST\fP
|
159
|
-
Spiders the specific \fIHOST\fP\.
|
160
|
-
.LP
|
161
134
|
.TP
|
162
|
-
\fB
|
163
|
-
|
164
|
-
.LP
|
135
|
+
\fB\-\-ignore\-exts\-like\fR \fB\[sl]\fR\fIREGEX\fP\fB\[sl]\fR
|
136
|
+
Ignore URLs with file exts matching the REGEX\.
|
165
137
|
.TP
|
166
|
-
\fB
|
167
|
-
|
168
|
-
.LP
|
138
|
+
\fB\-r\fR, \fB\-\-robots\fR
|
139
|
+
Specifies whether to honor \fBrobots\.txt\fR\.
|
169
140
|
.TP
|
170
|
-
\fB
|
141
|
+
\fB\-\-print\-status\fR
|
171
142
|
Print the status codes for each URL\.
|
172
|
-
.LP
|
173
143
|
.TP
|
174
|
-
\fB
|
144
|
+
\fB\-\-print\-headers\fR
|
175
145
|
Print response headers for each URL\.
|
176
|
-
.LP
|
177
146
|
.TP
|
178
|
-
\fB
|
147
|
+
\fB\-\-print\-header\fR \fINAME\fP
|
179
148
|
Prints a specific header\.
|
180
|
-
.LP
|
181
149
|
.TP
|
182
|
-
\fB
|
150
|
+
\fB\-\-history\fR \fIFILE\fP
|
183
151
|
Sets the history file to write every visited URL to\.
|
184
|
-
.LP
|
185
152
|
.TP
|
186
|
-
\fB
|
153
|
+
\fB\-\-archive\fR \fIDIR\fP
|
187
154
|
Archive every visited page to the \fIDIR\fP\.
|
188
|
-
.LP
|
189
155
|
.TP
|
190
|
-
\fB
|
156
|
+
\fB\-\-git\-archive\fR \fIDIR\fP
|
191
157
|
Archive every visited page to the git repository\.
|
192
|
-
.LP
|
193
158
|
.TP
|
194
|
-
\fB
|
159
|
+
\fB\-X\fR, \fB\-\-xpath\fR \fIXPATH\fP
|
195
160
|
Evaluates the XPath on each HTML page\.
|
196
|
-
.LP
|
197
161
|
.TP
|
198
|
-
\fB
|
162
|
+
\fB\-C\fR, \fB\-\-css\-path\fR \fIXPATH\fP
|
199
163
|
Evaluates the CSS\-path on each HTML page\.
|
200
|
-
.LP
|
201
164
|
.TP
|
202
|
-
\fB
|
165
|
+
\fB\-\-print\-hosts\fR
|
166
|
+
Print all discovered hostnames\.
|
167
|
+
.TP
|
168
|
+
\fB\-\-print\-certs\fR
|
169
|
+
Print all encountered SSL\[sl]TLS certificates\.
|
170
|
+
.TP
|
171
|
+
\fB\-\-save\-certs\fR
|
172
|
+
Saves all encountered SSL\[sl]TLS certificates\.
|
173
|
+
.TP
|
174
|
+
\fB\-\-print\-js\-strings\fR
|
175
|
+
Print all JavaScript strings\.
|
176
|
+
.TP
|
177
|
+
\fB\-\-print\-js\-url\-strings\fR
|
178
|
+
Print URL strings found in JavaScript\.
|
179
|
+
.TP
|
180
|
+
\fB\-\-print\-js\-path\-strings\fR
|
181
|
+
Print path strings found in JavaScript\.
|
182
|
+
.TP
|
183
|
+
\fB\-\-print\-js\-absolute\-path\-strings\fR
|
184
|
+
Only print absolute path strings found in JavaScript\.
|
185
|
+
.TP
|
186
|
+
\fB\-\-print\-js\-relative\-path\-strings\fR
|
187
|
+
Only print relative path strings found in JavaScript\.
|
188
|
+
.TP
|
189
|
+
\fB\-\-print\-html\-comments\fR
|
190
|
+
Print HTML comments\.
|
191
|
+
.TP
|
192
|
+
\fB\-\-print\-js\-comments\fR
|
193
|
+
Print JavaScript comments\.
|
194
|
+
.TP
|
195
|
+
\fB\-\-print\-comments\fR
|
196
|
+
Print all HTML and JavaScript comments\.
|
197
|
+
.TP
|
198
|
+
\fB\-v\fR, \fB\-\-verbose\fR
|
203
199
|
Enables verbose output\.
|
204
|
-
.LP
|
205
200
|
.TP
|
206
|
-
\fB
|
201
|
+
\fB\-h\fR, \fB\-\-help\fR
|
207
202
|
Print help information\.
|
208
|
-
.LP
|
209
203
|
.SH ENVIRONMENT
|
210
|
-
.LP
|
211
204
|
.TP
|
212
205
|
\fIHTTP\[ru]PROXY\fP
|
213
206
|
Sets the global HTTP proxy\.
|
214
|
-
.LP
|
215
207
|
.TP
|
216
208
|
\fIRONIN\[ru]HTTP\[ru]PROXY\fP
|
217
209
|
Sets the HTTP proxy for Ronin\.
|
218
|
-
.LP
|
219
210
|
.SH AUTHOR
|
220
|
-
.LP
|
221
211
|
.PP
|
222
212
|
Postmodern
|
223
213
|
.MT postmodern\.mod3\[at]gmail\.com
|
224
214
|
.ME
|
225
|
-
.LP
|
226
215
|
.SH SEE ALSO
|
227
|
-
.LP
|
228
216
|
.PP
|
229
|
-
ronin\-web\-server
|
217
|
+
.BR ronin\-web\-server (1)
|
218
|
+
.BR ronin\-web\-proxy (1)
|
219
|
+
.BR ronin\-web\-diff (1)
|
220
|
+
.BR ronin\-web\-new\-spider (1)
|
data/man/ronin-web-spider.1.md
CHANGED
@@ -1,8 +1,12 @@
|
|
1
1
|
# ronin-web-spider 1 "2022-01-01" Ronin Web "User Manuals"
|
2
2
|
|
3
|
+
## NAME
|
4
|
+
|
5
|
+
ronin-web-spider - Spiders a website
|
6
|
+
|
3
7
|
## SYNOPSIS
|
4
8
|
|
5
|
-
`ronin-web
|
9
|
+
`ronin-web spider` [*options*] {`--host` *HOST* \| `--domain` *DOMAIN* \| `--site` *URL*}
|
6
10
|
|
7
11
|
## DESCRIPTION
|
8
12
|
|
@@ -10,157 +14,202 @@ Spiders a website.
|
|
10
14
|
|
11
15
|
## OPTIONS
|
12
16
|
|
17
|
+
`--host` *HOST*
|
18
|
+
: Spiders the specific *HOST*.
|
19
|
+
|
20
|
+
`--domain` *DOMAIN*
|
21
|
+
: Spiders the whole *DOMAIN*.
|
22
|
+
|
23
|
+
`--site` *URL*
|
24
|
+
: Spiders the website, starting at the *URL*.
|
25
|
+
|
13
26
|
`--open-timeout` *SECS*
|
14
|
-
|
27
|
+
: Sets the connection open timeout.
|
15
28
|
|
16
29
|
`--read-timeout` *SECS*
|
17
|
-
|
30
|
+
: Sets the read timeout.
|
18
31
|
|
19
32
|
`--ssl-timeout` *SECS*
|
20
|
-
|
33
|
+
: Sets the SSL connection timeout.
|
21
34
|
|
22
35
|
`--continue-timeout` *SECS*
|
23
|
-
|
36
|
+
: Sets the continue timeout.
|
24
37
|
|
25
38
|
`--keep-alive-timeout` *SECS*
|
26
|
-
|
39
|
+
: Sets the connection keep alive timeout.
|
27
40
|
|
28
41
|
`-P`, `--proxy` *PROXY*
|
29
|
-
|
42
|
+
: Sets the proxy to use.
|
30
43
|
|
31
44
|
`-H`, `--header` "*NAME*: *VALUE*"
|
32
|
-
|
45
|
+
: Sets a default header.
|
33
46
|
|
34
47
|
`--host-header` *NAME*=*VALUE*
|
35
|
-
|
48
|
+
: Sets a default header.
|
36
49
|
|
37
50
|
`-u`, `--user-agent` chrome-linux|chrome-macos|chrome-windows|chrome-iphone|chrome-ipad|chrome-android|firefox-linux|firefox-macos|firefox-windows|firefox-iphone|firefox-ipad|firefox-android|safari-macos|safari-iphone|safari-ipad|edge
|
38
|
-
|
51
|
+
: The `User-Agent` to use.
|
39
52
|
|
40
53
|
`-U`, `--user-agent-string` *STRING*
|
41
|
-
|
54
|
+
: The raw `User-Agent` string to use.
|
42
55
|
|
43
56
|
`-R`, `--referer` *URL*
|
44
|
-
|
57
|
+
: Sets the `Referer` URL.
|
45
58
|
|
46
59
|
`--delay` *SECS*
|
47
|
-
|
60
|
+
: Sets the delay in seconds between each request.
|
48
61
|
|
49
62
|
`-l`, `--limit` *COUNT*
|
50
|
-
|
63
|
+
: Only spiders up to *COUNT* pages.
|
51
64
|
|
52
65
|
`-d`, `--max-depth` *DEPTH*
|
53
|
-
|
66
|
+
: Only spiders up to max depth.
|
54
67
|
|
55
68
|
`--enqueue` *URL*
|
56
|
-
|
69
|
+
: Adds the URL to the queue.
|
57
70
|
|
58
71
|
`--visited` *URL*
|
59
|
-
|
72
|
+
: Marks the URL as previously visited.
|
60
73
|
|
61
74
|
`--strip-fragments`
|
62
|
-
|
75
|
+
: Enables/disables stripping the fragment component of every URL.
|
63
76
|
|
64
77
|
`--strip-query`
|
65
|
-
|
78
|
+
: Enables/disables stripping the query component of every URL.
|
79
|
+
|
80
|
+
`--visit-scheme` *SCHEME*
|
81
|
+
: Visit URLs with the URI scheme.
|
82
|
+
|
83
|
+
`--visit-schemes-like` `/`*REGEX*`/`
|
84
|
+
: Visit URLs with URI schemes that match the *REGEX*.
|
85
|
+
|
86
|
+
`--ignore-scheme` *SCHEME*
|
87
|
+
: Ignore URLs with the URI scheme.
|
88
|
+
|
89
|
+
`--ignore-schemes-like` `/`*REGEX*`/`
|
90
|
+
: Ignore URLs with URI schemes matching the *REGEX*.
|
66
91
|
|
67
92
|
`--visit-host` *HOST*
|
68
|
-
|
93
|
+
: Visit URLs with the matching host name.
|
69
94
|
|
70
|
-
`--visit-hosts-like`
|
71
|
-
|
95
|
+
`--visit-hosts-like` `/`*REGEX*`/`
|
96
|
+
: Visit URLs with hostnames that match the *REGEX*.
|
72
97
|
|
73
98
|
`--ignore-host` *HOST*
|
74
|
-
|
99
|
+
: Ignore the host name.
|
75
100
|
|
76
|
-
`--ignore-hosts-like`
|
77
|
-
|
101
|
+
`--ignore-hosts-like` `/`*REGEX*`/`
|
102
|
+
: Ignore the host names matching the *REGEX*.
|
78
103
|
|
79
104
|
`--visit-port` *PORT*
|
80
|
-
|
105
|
+
: Visit URLs with the matching port number.
|
81
106
|
|
82
|
-
`--visit-ports-like`
|
83
|
-
|
107
|
+
`--visit-ports-like` `/`*REGEX*`/`
|
108
|
+
: Visit URLs with port numbers that match the *REGEX*.
|
84
109
|
|
85
110
|
`--ignore-port` *PORT*
|
86
|
-
|
111
|
+
: Ignore the port number.
|
87
112
|
|
88
|
-
`--ignore-ports-like`
|
89
|
-
|
113
|
+
`--ignore-ports-like` `/`*REGEX*`/`
|
114
|
+
: Ignore the port numbers matching the *REGEXP*.
|
90
115
|
|
91
116
|
`--visit-link` *URL*
|
92
|
-
|
117
|
+
: Visit the *URL*.
|
93
118
|
|
94
|
-
`--visit-links-like`
|
95
|
-
|
119
|
+
`--visit-links-like` `/`*REGEX*`/`
|
120
|
+
: Visit URLs that match the *REGEX*.
|
96
121
|
|
97
122
|
`--ignore-link` *URL*
|
98
|
-
|
123
|
+
: Ignore the *URL*.
|
99
124
|
|
100
|
-
`--ignore-links-like`
|
101
|
-
|
125
|
+
`--ignore-links-like` `/`*REGEX*`/`
|
126
|
+
: Ignore URLs matching the *REGEX*.
|
102
127
|
|
103
128
|
`--visit-ext` *FILE_EXT*
|
104
|
-
|
129
|
+
: Visit URLs with the matching file ext.
|
105
130
|
|
106
|
-
`--visit-exts-like`
|
107
|
-
|
131
|
+
`--visit-exts-like` `/`*REGEX*`/`
|
132
|
+
: Visit URLs with file exts that match the *REGEX*.
|
108
133
|
|
109
134
|
`--ignore-ext` *FILE_EXT*
|
110
|
-
|
135
|
+
: Ignore the URLs with the file ext.
|
111
136
|
|
112
|
-
`--ignore-exts-like`
|
113
|
-
|
137
|
+
`--ignore-exts-like` `/`*REGEX*`/`
|
138
|
+
: Ignore URLs with file exts matching the REGEX.
|
114
139
|
|
115
140
|
`-r`, `--robots`
|
116
|
-
|
117
|
-
|
118
|
-
`--host` *HOST*
|
119
|
-
Spiders the specific *HOST*.
|
120
|
-
|
121
|
-
`--domain` *DOMAIN*
|
122
|
-
Spiders the whole *DOMAIN*.
|
123
|
-
|
124
|
-
`--site` *URL*
|
125
|
-
Spiders the website, starting at the *URL*.
|
141
|
+
: Specifies whether to honor `robots.txt`.
|
126
142
|
|
127
143
|
`--print-status`
|
128
|
-
|
144
|
+
: Print the status codes for each URL.
|
129
145
|
|
130
146
|
`--print-headers`
|
131
|
-
|
147
|
+
: Print response headers for each URL.
|
132
148
|
|
133
149
|
`--print-header` *NAME*
|
134
|
-
|
150
|
+
: Prints a specific header.
|
135
151
|
|
136
152
|
`--history` *FILE*
|
137
|
-
|
153
|
+
: Sets the history file to write every visited URL to.
|
138
154
|
|
139
155
|
`--archive` *DIR*
|
140
|
-
|
156
|
+
: Archive every visited page to the *DIR*.
|
141
157
|
|
142
158
|
`--git-archive` *DIR*
|
143
|
-
|
159
|
+
: Archive every visited page to the git repository.
|
144
160
|
|
145
161
|
`-X`, `--xpath` *XPATH*
|
146
|
-
|
162
|
+
: Evaluates the XPath on each HTML page.
|
147
163
|
|
148
164
|
`-C`, `--css-path` *XPATH*
|
149
|
-
|
165
|
+
: Evaluates the CSS-path on each HTML page.
|
166
|
+
|
167
|
+
`--print-hosts`
|
168
|
+
: Print all discovered hostnames.
|
169
|
+
|
170
|
+
`--print-certs`
|
171
|
+
: Print all encountered SSL/TLS certificates.
|
172
|
+
|
173
|
+
`--save-certs`
|
174
|
+
: Saves all encountered SSL/TLS certificates.
|
175
|
+
|
176
|
+
`--print-js-strings`
|
177
|
+
: Print all JavaScript strings.
|
178
|
+
|
179
|
+
`--print-js-url-strings`
|
180
|
+
: Print URL strings found in JavaScript.
|
181
|
+
|
182
|
+
`--print-js-path-strings`
|
183
|
+
: Print path strings found in JavaScript.
|
184
|
+
|
185
|
+
`--print-js-absolute-path-strings`
|
186
|
+
: Only print absolute path strings found in JavaScript.
|
187
|
+
|
188
|
+
`--print-js-relative-path-strings`
|
189
|
+
: Only print relative path strings found in JavaScript.
|
190
|
+
|
191
|
+
`--print-html-comments`
|
192
|
+
: Print HTML comments.
|
193
|
+
|
194
|
+
`--print-js-comments`
|
195
|
+
: Print JavaScript comments.
|
196
|
+
|
197
|
+
`--print-comments`
|
198
|
+
: Print all HTML and JavaScript comments.
|
150
199
|
|
151
200
|
`-v`, `--verbose`
|
152
|
-
|
201
|
+
: Enables verbose output.
|
153
202
|
|
154
203
|
`-h`, `--help`
|
155
|
-
|
204
|
+
: Print help information.
|
156
205
|
|
157
206
|
## ENVIRONMENT
|
158
207
|
|
159
208
|
*HTTP_PROXY*
|
160
|
-
|
209
|
+
: Sets the global HTTP proxy.
|
161
210
|
|
162
211
|
*RONIN_HTTP_PROXY*
|
163
|
-
|
212
|
+
: Sets the HTTP proxy for Ronin.
|
164
213
|
|
165
214
|
## AUTHOR
|
166
215
|
|
@@ -168,4 +217,4 @@ Postmodern <postmodern.mod3@gmail.com>
|
|
168
217
|
|
169
218
|
## SEE ALSO
|
170
219
|
|
171
|
-
ronin-web-server(1) ronin-web-proxy(1) ronin-web-diff(1) ronin-web-new-spider(1)
|
220
|
+
[ronin-web-server](ronin-web-server.1.md) [ronin-web-proxy](ronin-web-proxy.1.md) [ronin-web-diff](ronin-web-diff.1.md) [ronin-web-new-spider](ronin-web-new-spider.1.md)
|
@@ -0,0 +1,44 @@
|
|
1
|
+
.\" Generated by kramdown-man 1.0.1
|
2
|
+
.\" https://github.com/postmodern/kramdown-man#readme
|
3
|
+
.TH ronin-web-user-agent 1 "2023-03-01" Ronin "User Manuals"
|
4
|
+
.SH NAME
|
5
|
+
.PP
|
6
|
+
ronin\-web\-user\-agent \- Generates a random User\-Agent string
|
7
|
+
.SH SYNOPSIS
|
8
|
+
.PP
|
9
|
+
\fBronin\-web user\-agent\fR \[lB]\fIoptions\fP\[rB]
|
10
|
+
.SH DESCRIPTION
|
11
|
+
.PP
|
12
|
+
Generates a random HTTP \fBUser\-Agent\fR string\.
|
13
|
+
.SH OPTIONS
|
14
|
+
.TP
|
15
|
+
\fB\-B\fR, \fB\-\-browser\fR \fBchrome\fR\[or]\fBfirefox\fR
|
16
|
+
Selects the desired browser type for the \fBUser\-Agent\fR string\.
|
17
|
+
.TP
|
18
|
+
\fB\-\-chrome\-version\fR \fIVERSION\fP
|
19
|
+
Sets desired Chrome version\. Only takes effect when \fB\-\-browser chrome\fR is also
|
20
|
+
given\.
|
21
|
+
.TP
|
22
|
+
\fB\-\-firefox\-version\fR \fIVERSION\fP
|
23
|
+
Sets desired Firefox version\. Only takes effect when \fB\-\-browser chrome\fR is
|
24
|
+
also given\.
|
25
|
+
.TP
|
26
|
+
\fB\-D\fR, \fB\-\-linux\-distro\fR \fBubuntu\fR\[or]\fBfedora\fR\[or]\fBarch\fR\[or]\fBDISTRO\fR
|
27
|
+
Selects the desired Linux distro\.
|
28
|
+
.TP
|
29
|
+
\fB\-A\fR, \fB\-\-arch\fR \fBx86\-64\fR\[or]\fBx86\fR\[or]\fBi686\fR\[or]\fBaarch64\fR\[or]\fBarm64\fR\[or]\fBarm\fR
|
30
|
+
Selects the desired architecture\.
|
31
|
+
.TP
|
32
|
+
\fB\-O\fR, \fB\-\-os\fR \fBandroid\fR\[or]\fBlinux\fR\[or]\fBwindows\fR
|
33
|
+
Selects the desired OS\.
|
34
|
+
.TP
|
35
|
+
\fB\-\-os\-version\fR \fIVERSION\fP
|
36
|
+
Sets the desired OS version\.
|
37
|
+
.TP
|
38
|
+
\fB\-h\fR, \fB\-\-help\fR
|
39
|
+
Print help information\.
|
40
|
+
.SH AUTHOR
|
41
|
+
.PP
|
42
|
+
Postmodern
|
43
|
+
.MT postmodern\.mod3\[at]gmail\.com
|
44
|
+
.ME
|