ronin-web 1.0.2 → 2.0.0.rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ruby.yml +3 -2
- data/.gitignore +1 -0
- data/.rubocop.yml +5 -0
- data/ChangeLog.md +46 -1
- data/Gemfile +25 -12
- data/README.md +257 -51
- data/Rakefile +9 -0
- data/data/completions/ronin-web +203 -0
- data/gemspec.yml +18 -5
- data/lib/ronin/web/cli/browser_options.rb +92 -0
- data/lib/ronin/web/cli/browser_shell.rb +448 -0
- data/lib/ronin/web/cli/command.rb +1 -1
- data/lib/ronin/web/cli/commands/browser.rb +373 -0
- data/lib/ronin/web/cli/commands/completion.rb +63 -0
- data/lib/ronin/web/cli/commands/diff.rb +60 -8
- data/lib/ronin/web/cli/commands/html.rb +21 -33
- data/lib/ronin/web/cli/commands/irb.rb +1 -1
- data/lib/ronin/web/cli/commands/new/{webapp.rb → app.rb} +8 -8
- data/lib/ronin/web/cli/commands/new/nokogiri.rb +4 -4
- data/lib/ronin/web/cli/commands/new/server.rb +1 -1
- data/lib/ronin/web/cli/commands/new/spider.rb +1 -1
- data/lib/ronin/web/cli/commands/new.rb +5 -3
- data/lib/ronin/web/cli/commands/reverse_proxy.rb +1 -1
- data/lib/ronin/web/cli/commands/screenshot.rb +186 -0
- data/lib/ronin/web/cli/commands/server.rb +1 -1
- data/lib/ronin/web/cli/commands/session_cookie.rb +265 -0
- data/lib/ronin/web/cli/commands/spider.rb +61 -467
- data/lib/ronin/web/cli/commands/user_agent.rb +177 -0
- data/lib/ronin/web/cli/commands/vulns.rb +463 -0
- data/lib/ronin/web/cli/commands/wordlist.rb +484 -0
- data/lib/ronin/web/cli/commands/xml.rb +149 -0
- data/lib/ronin/web/cli/js_shell.rb +69 -0
- data/lib/ronin/web/cli/ruby_shell.rb +1 -1
- data/lib/ronin/web/cli/spider_options.rb +919 -0
- data/lib/ronin/web/cli.rb +3 -1
- data/lib/ronin/web/html.rb +1 -1
- data/lib/ronin/web/root.rb +1 -1
- data/lib/ronin/web/version.rb +2 -2
- data/lib/ronin/web/xml.rb +1 -1
- data/lib/ronin/web.rb +4 -364
- data/man/ronin-web-browser.1 +92 -0
- data/man/ronin-web-browser.1.md +96 -0
- data/man/ronin-web-completion.1 +76 -0
- data/man/ronin-web-completion.1.md +78 -0
- data/man/ronin-web-diff.1 +14 -21
- data/man/ronin-web-diff.1.md +13 -6
- data/man/ronin-web-html.1 +30 -46
- data/man/ronin-web-html.1.md +27 -17
- data/man/ronin-web-irb.1 +9 -16
- data/man/ronin-web-irb.1.md +6 -2
- data/man/ronin-web-new-app.1.md +39 -0
- data/man/ronin-web-new-nokogiri.1 +9 -20
- data/man/ronin-web-new-nokogiri.1.md +5 -5
- data/man/ronin-web-new-server.1 +11 -23
- data/man/ronin-web-new-server.1.md +5 -5
- data/man/ronin-web-new-spider.1 +44 -88
- data/man/ronin-web-new-spider.1.md +37 -37
- data/man/ronin-web-new.1 +18 -30
- data/man/ronin-web-new.1.md +15 -11
- data/man/ronin-web-reverse-proxy.1 +33 -38
- data/man/ronin-web-reverse-proxy.1.md +20 -14
- data/man/ronin-web-screenshot.1 +56 -0
- data/man/ronin-web-screenshot.1.md +56 -0
- data/man/ronin-web-server.1 +15 -29
- data/man/ronin-web-server.1.md +13 -9
- data/man/ronin-web-session-cookie.1 +38 -0
- data/man/ronin-web-session-cookie.1.md +41 -0
- data/man/ronin-web-spider.1 +121 -130
- data/man/ronin-web-spider.1.md +115 -66
- data/man/ronin-web-user-agent.1 +44 -0
- data/man/ronin-web-user-agent.1.md +46 -0
- data/man/ronin-web-vulns.1 +175 -0
- data/man/ronin-web-vulns.1.md +177 -0
- data/man/ronin-web-wordlist.1 +258 -0
- data/man/ronin-web-wordlist.1.md +263 -0
- data/man/ronin-web-xml.1 +43 -0
- data/man/ronin-web-xml.1.md +46 -0
- data/man/ronin-web.1 +67 -18
- data/man/ronin-web.1.md +55 -4
- data/scripts/setup +58 -0
- metadata +122 -31
- data/lib/ronin/web/mechanize.rb +0 -84
- data/man/ronin-web-new-webapp.1.md +0 -39
- /data/data/new/{webapp → app}/.gitignore +0 -0
- /data/data/new/{webapp → app}/.ruby-version.erb +0 -0
- /data/data/new/{webapp → app}/Dockerfile.erb +0 -0
- /data/data/new/{webapp → app}/Gemfile +0 -0
- /data/data/new/{webapp → app}/app.rb.erb +0 -0
- /data/data/new/{webapp → app}/config.ru +0 -0
- /data/data/new/{webapp → app}/docker-compose.yml.erb +0 -0
data/man/ronin-web-spider.1
CHANGED
@@ -1,229 +1,220 @@
|
|
1
|
-
.\" Generated by kramdown-man 0.1
|
1
|
+
.\" Generated by kramdown-man 1.0.1
|
2
2
|
.\" https://github.com/postmodern/kramdown-man#readme
|
3
3
|
.TH ronin-web-spider 1 "2022-01-01" Ronin Web "User Manuals"
|
4
|
-
.
|
4
|
+
.SH NAME
|
5
|
+
.PP
|
6
|
+
ronin\-web\-spider \- Spiders a website
|
5
7
|
.SH SYNOPSIS
|
6
|
-
.
|
7
|
-
|
8
|
-
\fBronin-web-spider\fR \[lB]\fIoptions\fP\[rB] \[lC]\fB--host\fR \fIHOST\fP \[or] \fB--domain\fR \fIDOMAIN\fP \[or] \fB--site\fR \fIURL\fP\[rC]
|
9
|
-
.LP
|
8
|
+
.PP
|
9
|
+
\fBronin\-web spider\fR \[lB]\fIoptions\fP\[rB] \[lC]\fB\-\-host\fR \fIHOST\fP \[or] \fB\-\-domain\fR \fIDOMAIN\fP \[or] \fB\-\-site\fR \fIURL\fP\[rC]
|
10
10
|
.SH DESCRIPTION
|
11
|
-
.LP
|
12
11
|
.PP
|
13
12
|
Spiders a website\.
|
14
|
-
.LP
|
15
13
|
.SH OPTIONS
|
16
|
-
.LP
|
17
14
|
.TP
|
18
|
-
\fB
|
15
|
+
\fB\-\-host\fR \fIHOST\fP
|
16
|
+
Spiders the specific \fIHOST\fP\.
|
17
|
+
.TP
|
18
|
+
\fB\-\-domain\fR \fIDOMAIN\fP
|
19
|
+
Spiders the whole \fIDOMAIN\fP\.
|
20
|
+
.TP
|
21
|
+
\fB\-\-site\fR \fIURL\fP
|
22
|
+
Spiders the website, starting at the \fIURL\fP\.
|
23
|
+
.TP
|
24
|
+
\fB\-\-open\-timeout\fR \fISECS\fP
|
19
25
|
Sets the connection open timeout\.
|
20
|
-
.LP
|
21
26
|
.TP
|
22
|
-
\fB
|
27
|
+
\fB\-\-read\-timeout\fR \fISECS\fP
|
23
28
|
Sets the read timeout\.
|
24
|
-
.LP
|
25
29
|
.TP
|
26
|
-
\fB
|
30
|
+
\fB\-\-ssl\-timeout\fR \fISECS\fP
|
27
31
|
Sets the SSL connection timeout\.
|
28
|
-
.LP
|
29
32
|
.TP
|
30
|
-
\fB
|
33
|
+
\fB\-\-continue\-timeout\fR \fISECS\fP
|
31
34
|
Sets the continue timeout\.
|
32
|
-
.LP
|
33
35
|
.TP
|
34
|
-
\fB
|
36
|
+
\fB\-\-keep\-alive\-timeout\fR \fISECS\fP
|
35
37
|
Sets the connection keep alive timeout\.
|
36
|
-
.LP
|
37
38
|
.TP
|
38
|
-
\fB
|
39
|
+
\fB\-P\fR, \fB\-\-proxy\fR \fIPROXY\fP
|
39
40
|
Sets the proxy to use\.
|
40
|
-
.LP
|
41
41
|
.TP
|
42
|
-
\fB
|
42
|
+
\fB\-H\fR, \fB\-\-header\fR \[lq]\fINAME\fP: \fIVALUE\fP\[rq]
|
43
43
|
Sets a default header\.
|
44
|
-
.LP
|
45
44
|
.TP
|
46
|
-
\fB
|
45
|
+
\fB\-\-host\-header\fR \fINAME\fP\[eq]\fIVALUE\fP
|
47
46
|
Sets a default header\.
|
48
|
-
.LP
|
49
|
-
.HP
|
50
|
-
\fB-u\fR, \fB--user-agent\fR chrome\-linux\[or]chrome\-macos\[or]chrome\-windows\[or]chrome\-iphone\[or]chrome\-ipad\[or]chrome\-android\[or]firefox\-linux\[or]firefox\-macos\[or]firefox\-windows\[or]firefox\-iphone\[or]firefox\-ipad\[or]firefox\-android\[or]safari\-macos\[or]safari\-iphone\[or]safari\-ipad\[or]edge
|
51
|
-
The \fBUser-Agent\fR to use\.
|
52
|
-
.LP
|
53
47
|
.TP
|
54
|
-
\fB
|
55
|
-
The
|
56
|
-
.LP
|
48
|
+
\fB\-u\fR, \fB\-\-user\-agent\fR chrome\-linux\[or]chrome\-macos\[or]chrome\-windows\[or]chrome\-iphone\[or]chrome\-ipad\[or]chrome\-android\[or]firefox\-linux\[or]firefox\-macos\[or]firefox\-windows\[or]firefox\-iphone\[or]firefox\-ipad\[or]firefox\-android\[or]safari\-macos\[or]safari\-iphone\[or]safari\-ipad\[or]edge
|
49
|
+
The \fBUser\-Agent\fR to use\.
|
57
50
|
.TP
|
58
|
-
\fB
|
51
|
+
\fB\-U\fR, \fB\-\-user\-agent\-string\fR \fISTRING\fP
|
52
|
+
The raw \fBUser\-Agent\fR string to use\.
|
53
|
+
.TP
|
54
|
+
\fB\-R\fR, \fB\-\-referer\fR \fIURL\fP
|
59
55
|
Sets the \fBReferer\fR URL\.
|
60
|
-
.LP
|
61
56
|
.TP
|
62
|
-
\fB
|
57
|
+
\fB\-\-delay\fR \fISECS\fP
|
63
58
|
Sets the delay in seconds between each request\.
|
64
|
-
.LP
|
65
59
|
.TP
|
66
|
-
\fB
|
60
|
+
\fB\-l\fR, \fB\-\-limit\fR \fICOUNT\fP
|
67
61
|
Only spiders up to \fICOUNT\fP pages\.
|
68
|
-
.LP
|
69
62
|
.TP
|
70
|
-
\fB
|
63
|
+
\fB\-d\fR, \fB\-\-max\-depth\fR \fIDEPTH\fP
|
71
64
|
Only spiders up to max depth\.
|
72
|
-
.LP
|
73
65
|
.TP
|
74
|
-
\fB
|
66
|
+
\fB\-\-enqueue\fR \fIURL\fP
|
75
67
|
Adds the URL to the queue\.
|
76
|
-
.LP
|
77
68
|
.TP
|
78
|
-
\fB
|
69
|
+
\fB\-\-visited\fR \fIURL\fP
|
79
70
|
Marks the URL as previously visited\.
|
80
|
-
.LP
|
81
71
|
.TP
|
82
|
-
\fB
|
72
|
+
\fB\-\-strip\-fragments\fR
|
83
73
|
Enables\[sl]disables stripping the fragment component of every URL\.
|
84
|
-
.LP
|
85
74
|
.TP
|
86
|
-
\fB
|
75
|
+
\fB\-\-strip\-query\fR
|
87
76
|
Enables\[sl]disables stripping the query component of every URL\.
|
88
|
-
.LP
|
89
77
|
.TP
|
90
|
-
\fB
|
78
|
+
\fB\-\-visit\-scheme\fR \fISCHEME\fP
|
79
|
+
Visit URLs with the URI scheme\.
|
80
|
+
.TP
|
81
|
+
\fB\-\-visit\-schemes\-like\fR \fB\[sl]\fR\fIREGEX\fP\fB\[sl]\fR
|
82
|
+
Visit URLs with URI schemes that match the \fIREGEX\fP\.
|
83
|
+
.TP
|
84
|
+
\fB\-\-ignore\-scheme\fR \fISCHEME\fP
|
85
|
+
Ignore URLs with the URI scheme\.
|
86
|
+
.TP
|
87
|
+
\fB\-\-ignore\-schemes\-like\fR \fB\[sl]\fR\fIREGEX\fP\fB\[sl]\fR
|
88
|
+
Ignore URLs with URI schemes matching the \fIREGEX\fP\.
|
89
|
+
.TP
|
90
|
+
\fB\-\-visit\-host\fR \fIHOST\fP
|
91
91
|
Visit URLs with the matching host name\.
|
92
|
-
.
|
93
|
-
|
94
|
-
\fB--visit-hosts-like\fR \[sl]\fIREGEX\fP\[sl]
|
92
|
+
.TP
|
93
|
+
\fB\-\-visit\-hosts\-like\fR \fB\[sl]\fR\fIREGEX\fP\fB\[sl]\fR
|
95
94
|
Visit URLs with hostnames that match the \fIREGEX\fP\.
|
96
|
-
.LP
|
97
95
|
.TP
|
98
|
-
\fB
|
96
|
+
\fB\-\-ignore\-host\fR \fIHOST\fP
|
99
97
|
Ignore the host name\.
|
100
|
-
.
|
101
|
-
|
102
|
-
\fB--ignore-hosts-like\fR \[sl]\fIREGEX\fP\[sl]
|
98
|
+
.TP
|
99
|
+
\fB\-\-ignore\-hosts\-like\fR \fB\[sl]\fR\fIREGEX\fP\fB\[sl]\fR
|
103
100
|
Ignore the host names matching the \fIREGEX\fP\.
|
104
|
-
.LP
|
105
101
|
.TP
|
106
|
-
\fB
|
102
|
+
\fB\-\-visit\-port\fR \fIPORT\fP
|
107
103
|
Visit URLs with the matching port number\.
|
108
|
-
.
|
109
|
-
|
110
|
-
\fB--visit-ports-like\fR \[sl]\fIREGEX\fP\[sl]
|
104
|
+
.TP
|
105
|
+
\fB\-\-visit\-ports\-like\fR \fB\[sl]\fR\fIREGEX\fP\fB\[sl]\fR
|
111
106
|
Visit URLs with port numbers that match the \fIREGEX\fP\.
|
112
|
-
.LP
|
113
107
|
.TP
|
114
|
-
\fB
|
108
|
+
\fB\-\-ignore\-port\fR \fIPORT\fP
|
115
109
|
Ignore the port number\.
|
116
|
-
.
|
117
|
-
|
118
|
-
\fB--ignore-ports-like\fR \[sl]\fIREGEX\fP\[sl]
|
110
|
+
.TP
|
111
|
+
\fB\-\-ignore\-ports\-like\fR \fB\[sl]\fR\fIREGEX\fP\fB\[sl]\fR
|
119
112
|
Ignore the port numbers matching the \fIREGEXP\fP\.
|
120
|
-
.LP
|
121
113
|
.TP
|
122
|
-
\fB
|
114
|
+
\fB\-\-visit\-link\fR \fIURL\fP
|
123
115
|
Visit the \fIURL\fP\.
|
124
|
-
.
|
125
|
-
|
126
|
-
\fB--visit-links-like\fR \[sl]\fIREGEX\fP\[sl]
|
116
|
+
.TP
|
117
|
+
\fB\-\-visit\-links\-like\fR \fB\[sl]\fR\fIREGEX\fP\fB\[sl]\fR
|
127
118
|
Visit URLs that match the \fIREGEX\fP\.
|
128
|
-
.LP
|
129
119
|
.TP
|
130
|
-
\fB
|
120
|
+
\fB\-\-ignore\-link\fR \fIURL\fP
|
131
121
|
Ignore the \fIURL\fP\.
|
132
|
-
.
|
133
|
-
|
134
|
-
\fB--ignore-links-like\fR \[sl]\fIREGEX\fP\[sl]
|
122
|
+
.TP
|
123
|
+
\fB\-\-ignore\-links\-like\fR \fB\[sl]\fR\fIREGEX\fP\fB\[sl]\fR
|
135
124
|
Ignore URLs matching the \fIREGEX\fP\.
|
136
|
-
.LP
|
137
125
|
.TP
|
138
|
-
\fB
|
126
|
+
\fB\-\-visit\-ext\fR \fIFILE\[ru]EXT\fP
|
139
127
|
Visit URLs with the matching file ext\.
|
140
|
-
.
|
141
|
-
|
142
|
-
\fB--visit-exts-like\fR \[sl]\fIREGEX\fP\[sl]
|
128
|
+
.TP
|
129
|
+
\fB\-\-visit\-exts\-like\fR \fB\[sl]\fR\fIREGEX\fP\fB\[sl]\fR
|
143
130
|
Visit URLs with file exts that match the \fIREGEX\fP\.
|
144
|
-
.LP
|
145
131
|
.TP
|
146
|
-
\fB
|
132
|
+
\fB\-\-ignore\-ext\fR \fIFILE\[ru]EXT\fP
|
147
133
|
Ignore the URLs with the file ext\.
|
148
|
-
.LP
|
149
|
-
.HP
|
150
|
-
\fB--ignore-exts-like\fR \[sl]\fIREGEX\fP\[sl]
|
151
|
-
Ignore URLs with file exts matching the REGEX\.
|
152
|
-
.LP
|
153
|
-
.TP
|
154
|
-
\fB-r\fR, \fB--robots\fR
|
155
|
-
Specifies whether to honor \fBrobots.txt\fR\.
|
156
|
-
.LP
|
157
|
-
.TP
|
158
|
-
\fB--host\fR \fIHOST\fP
|
159
|
-
Spiders the specific \fIHOST\fP\.
|
160
|
-
.LP
|
161
134
|
.TP
|
162
|
-
\fB
|
163
|
-
|
164
|
-
.LP
|
135
|
+
\fB\-\-ignore\-exts\-like\fR \fB\[sl]\fR\fIREGEX\fP\fB\[sl]\fR
|
136
|
+
Ignore URLs with file exts matching the REGEX\.
|
165
137
|
.TP
|
166
|
-
\fB
|
167
|
-
|
168
|
-
.LP
|
138
|
+
\fB\-r\fR, \fB\-\-robots\fR
|
139
|
+
Specifies whether to honor \fBrobots\.txt\fR\.
|
169
140
|
.TP
|
170
|
-
\fB
|
141
|
+
\fB\-\-print\-status\fR
|
171
142
|
Print the status codes for each URL\.
|
172
|
-
.LP
|
173
143
|
.TP
|
174
|
-
\fB
|
144
|
+
\fB\-\-print\-headers\fR
|
175
145
|
Print response headers for each URL\.
|
176
|
-
.LP
|
177
146
|
.TP
|
178
|
-
\fB
|
147
|
+
\fB\-\-print\-header\fR \fINAME\fP
|
179
148
|
Prints a specific header\.
|
180
|
-
.LP
|
181
149
|
.TP
|
182
|
-
\fB
|
150
|
+
\fB\-\-history\fR \fIFILE\fP
|
183
151
|
Sets the history file to write every visited URL to\.
|
184
|
-
.LP
|
185
152
|
.TP
|
186
|
-
\fB
|
153
|
+
\fB\-\-archive\fR \fIDIR\fP
|
187
154
|
Archive every visited page to the \fIDIR\fP\.
|
188
|
-
.LP
|
189
155
|
.TP
|
190
|
-
\fB
|
156
|
+
\fB\-\-git\-archive\fR \fIDIR\fP
|
191
157
|
Archive every visited page to the git repository\.
|
192
|
-
.LP
|
193
158
|
.TP
|
194
|
-
\fB
|
159
|
+
\fB\-X\fR, \fB\-\-xpath\fR \fIXPATH\fP
|
195
160
|
Evaluates the XPath on each HTML page\.
|
196
|
-
.LP
|
197
161
|
.TP
|
198
|
-
\fB
|
162
|
+
\fB\-C\fR, \fB\-\-css\-path\fR \fIXPATH\fP
|
199
163
|
Evaluates the CSS\-path on each HTML page\.
|
200
|
-
.LP
|
201
164
|
.TP
|
202
|
-
\fB
|
165
|
+
\fB\-\-print\-hosts\fR
|
166
|
+
Print all discovered hostnames\.
|
167
|
+
.TP
|
168
|
+
\fB\-\-print\-certs\fR
|
169
|
+
Print all encountered SSL\[sl]TLS certificates\.
|
170
|
+
.TP
|
171
|
+
\fB\-\-save\-certs\fR
|
172
|
+
Saves all encountered SSL\[sl]TLS certificates\.
|
173
|
+
.TP
|
174
|
+
\fB\-\-print\-js\-strings\fR
|
175
|
+
Print all JavaScript strings\.
|
176
|
+
.TP
|
177
|
+
\fB\-\-print\-js\-url\-strings\fR
|
178
|
+
Print URL strings found in JavaScript\.
|
179
|
+
.TP
|
180
|
+
\fB\-\-print\-js\-path\-strings\fR
|
181
|
+
Print path strings found in JavaScript\.
|
182
|
+
.TP
|
183
|
+
\fB\-\-print\-js\-absolute\-path\-strings\fR
|
184
|
+
Only print absolute path strings found in JavaScript\.
|
185
|
+
.TP
|
186
|
+
\fB\-\-print\-js\-relative\-path\-strings\fR
|
187
|
+
Only print relative path strings found in JavaScript\.
|
188
|
+
.TP
|
189
|
+
\fB\-\-print\-html\-comments\fR
|
190
|
+
Print HTML comments\.
|
191
|
+
.TP
|
192
|
+
\fB\-\-print\-js\-comments\fR
|
193
|
+
Print JavaScript comments\.
|
194
|
+
.TP
|
195
|
+
\fB\-\-print\-comments\fR
|
196
|
+
Print all HTML and JavaScript comments\.
|
197
|
+
.TP
|
198
|
+
\fB\-v\fR, \fB\-\-verbose\fR
|
203
199
|
Enables verbose output\.
|
204
|
-
.LP
|
205
200
|
.TP
|
206
|
-
\fB
|
201
|
+
\fB\-h\fR, \fB\-\-help\fR
|
207
202
|
Print help information\.
|
208
|
-
.LP
|
209
203
|
.SH ENVIRONMENT
|
210
|
-
.LP
|
211
204
|
.TP
|
212
205
|
\fIHTTP\[ru]PROXY\fP
|
213
206
|
Sets the global HTTP proxy\.
|
214
|
-
.LP
|
215
207
|
.TP
|
216
208
|
\fIRONIN\[ru]HTTP\[ru]PROXY\fP
|
217
209
|
Sets the HTTP proxy for Ronin\.
|
218
|
-
.LP
|
219
210
|
.SH AUTHOR
|
220
|
-
.LP
|
221
211
|
.PP
|
222
212
|
Postmodern
|
223
213
|
.MT postmodern\.mod3\[at]gmail\.com
|
224
214
|
.ME
|
225
|
-
.LP
|
226
215
|
.SH SEE ALSO
|
227
|
-
.LP
|
228
216
|
.PP
|
229
|
-
ronin\-web\-server
|
217
|
+
.BR ronin\-web\-server (1)
|
218
|
+
.BR ronin\-web\-proxy (1)
|
219
|
+
.BR ronin\-web\-diff (1)
|
220
|
+
.BR ronin\-web\-new\-spider (1)
|
data/man/ronin-web-spider.1.md
CHANGED
@@ -1,8 +1,12 @@
|
|
1
1
|
# ronin-web-spider 1 "2022-01-01" Ronin Web "User Manuals"
|
2
2
|
|
3
|
+
## NAME
|
4
|
+
|
5
|
+
ronin-web-spider - Spiders a website
|
6
|
+
|
3
7
|
## SYNOPSIS
|
4
8
|
|
5
|
-
`ronin-web
|
9
|
+
`ronin-web spider` [*options*] {`--host` *HOST* \| `--domain` *DOMAIN* \| `--site` *URL*}
|
6
10
|
|
7
11
|
## DESCRIPTION
|
8
12
|
|
@@ -10,157 +14,202 @@ Spiders a website.
|
|
10
14
|
|
11
15
|
## OPTIONS
|
12
16
|
|
17
|
+
`--host` *HOST*
|
18
|
+
: Spiders the specific *HOST*.
|
19
|
+
|
20
|
+
`--domain` *DOMAIN*
|
21
|
+
: Spiders the whole *DOMAIN*.
|
22
|
+
|
23
|
+
`--site` *URL*
|
24
|
+
: Spiders the website, starting at the *URL*.
|
25
|
+
|
13
26
|
`--open-timeout` *SECS*
|
14
|
-
|
27
|
+
: Sets the connection open timeout.
|
15
28
|
|
16
29
|
`--read-timeout` *SECS*
|
17
|
-
|
30
|
+
: Sets the read timeout.
|
18
31
|
|
19
32
|
`--ssl-timeout` *SECS*
|
20
|
-
|
33
|
+
: Sets the SSL connection timeout.
|
21
34
|
|
22
35
|
`--continue-timeout` *SECS*
|
23
|
-
|
36
|
+
: Sets the continue timeout.
|
24
37
|
|
25
38
|
`--keep-alive-timeout` *SECS*
|
26
|
-
|
39
|
+
: Sets the connection keep alive timeout.
|
27
40
|
|
28
41
|
`-P`, `--proxy` *PROXY*
|
29
|
-
|
42
|
+
: Sets the proxy to use.
|
30
43
|
|
31
44
|
`-H`, `--header` "*NAME*: *VALUE*"
|
32
|
-
|
45
|
+
: Sets a default header.
|
33
46
|
|
34
47
|
`--host-header` *NAME*=*VALUE*
|
35
|
-
|
48
|
+
: Sets a default header.
|
36
49
|
|
37
50
|
`-u`, `--user-agent` chrome-linux|chrome-macos|chrome-windows|chrome-iphone|chrome-ipad|chrome-android|firefox-linux|firefox-macos|firefox-windows|firefox-iphone|firefox-ipad|firefox-android|safari-macos|safari-iphone|safari-ipad|edge
|
38
|
-
|
51
|
+
: The `User-Agent` to use.
|
39
52
|
|
40
53
|
`-U`, `--user-agent-string` *STRING*
|
41
|
-
|
54
|
+
: The raw `User-Agent` string to use.
|
42
55
|
|
43
56
|
`-R`, `--referer` *URL*
|
44
|
-
|
57
|
+
: Sets the `Referer` URL.
|
45
58
|
|
46
59
|
`--delay` *SECS*
|
47
|
-
|
60
|
+
: Sets the delay in seconds between each request.
|
48
61
|
|
49
62
|
`-l`, `--limit` *COUNT*
|
50
|
-
|
63
|
+
: Only spiders up to *COUNT* pages.
|
51
64
|
|
52
65
|
`-d`, `--max-depth` *DEPTH*
|
53
|
-
|
66
|
+
: Only spiders up to max depth.
|
54
67
|
|
55
68
|
`--enqueue` *URL*
|
56
|
-
|
69
|
+
: Adds the URL to the queue.
|
57
70
|
|
58
71
|
`--visited` *URL*
|
59
|
-
|
72
|
+
: Marks the URL as previously visited.
|
60
73
|
|
61
74
|
`--strip-fragments`
|
62
|
-
|
75
|
+
: Enables/disables stripping the fragment component of every URL.
|
63
76
|
|
64
77
|
`--strip-query`
|
65
|
-
|
78
|
+
: Enables/disables stripping the query component of every URL.
|
79
|
+
|
80
|
+
`--visit-scheme` *SCHEME*
|
81
|
+
: Visit URLs with the URI scheme.
|
82
|
+
|
83
|
+
`--visit-schemes-like` `/`*REGEX*`/`
|
84
|
+
: Visit URLs with URI schemes that match the *REGEX*.
|
85
|
+
|
86
|
+
`--ignore-scheme` *SCHEME*
|
87
|
+
: Ignore URLs with the URI scheme.
|
88
|
+
|
89
|
+
`--ignore-schemes-like` `/`*REGEX*`/`
|
90
|
+
: Ignore URLs with URI schemes matching the *REGEX*.
|
66
91
|
|
67
92
|
`--visit-host` *HOST*
|
68
|
-
|
93
|
+
: Visit URLs with the matching host name.
|
69
94
|
|
70
|
-
`--visit-hosts-like`
|
71
|
-
|
95
|
+
`--visit-hosts-like` `/`*REGEX*`/`
|
96
|
+
: Visit URLs with hostnames that match the *REGEX*.
|
72
97
|
|
73
98
|
`--ignore-host` *HOST*
|
74
|
-
|
99
|
+
: Ignore the host name.
|
75
100
|
|
76
|
-
`--ignore-hosts-like`
|
77
|
-
|
101
|
+
`--ignore-hosts-like` `/`*REGEX*`/`
|
102
|
+
: Ignore the host names matching the *REGEX*.
|
78
103
|
|
79
104
|
`--visit-port` *PORT*
|
80
|
-
|
105
|
+
: Visit URLs with the matching port number.
|
81
106
|
|
82
|
-
`--visit-ports-like`
|
83
|
-
|
107
|
+
`--visit-ports-like` `/`*REGEX*`/`
|
108
|
+
: Visit URLs with port numbers that match the *REGEX*.
|
84
109
|
|
85
110
|
`--ignore-port` *PORT*
|
86
|
-
|
111
|
+
: Ignore the port number.
|
87
112
|
|
88
|
-
`--ignore-ports-like`
|
89
|
-
|
113
|
+
`--ignore-ports-like` `/`*REGEX*`/`
|
114
|
+
: Ignore the port numbers matching the *REGEXP*.
|
90
115
|
|
91
116
|
`--visit-link` *URL*
|
92
|
-
|
117
|
+
: Visit the *URL*.
|
93
118
|
|
94
|
-
`--visit-links-like`
|
95
|
-
|
119
|
+
`--visit-links-like` `/`*REGEX*`/`
|
120
|
+
: Visit URLs that match the *REGEX*.
|
96
121
|
|
97
122
|
`--ignore-link` *URL*
|
98
|
-
|
123
|
+
: Ignore the *URL*.
|
99
124
|
|
100
|
-
`--ignore-links-like`
|
101
|
-
|
125
|
+
`--ignore-links-like` `/`*REGEX*`/`
|
126
|
+
: Ignore URLs matching the *REGEX*.
|
102
127
|
|
103
128
|
`--visit-ext` *FILE_EXT*
|
104
|
-
|
129
|
+
: Visit URLs with the matching file ext.
|
105
130
|
|
106
|
-
`--visit-exts-like`
|
107
|
-
|
131
|
+
`--visit-exts-like` `/`*REGEX*`/`
|
132
|
+
: Visit URLs with file exts that match the *REGEX*.
|
108
133
|
|
109
134
|
`--ignore-ext` *FILE_EXT*
|
110
|
-
|
135
|
+
: Ignore the URLs with the file ext.
|
111
136
|
|
112
|
-
`--ignore-exts-like`
|
113
|
-
|
137
|
+
`--ignore-exts-like` `/`*REGEX*`/`
|
138
|
+
: Ignore URLs with file exts matching the REGEX.
|
114
139
|
|
115
140
|
`-r`, `--robots`
|
116
|
-
|
117
|
-
|
118
|
-
`--host` *HOST*
|
119
|
-
Spiders the specific *HOST*.
|
120
|
-
|
121
|
-
`--domain` *DOMAIN*
|
122
|
-
Spiders the whole *DOMAIN*.
|
123
|
-
|
124
|
-
`--site` *URL*
|
125
|
-
Spiders the website, starting at the *URL*.
|
141
|
+
: Specifies whether to honor `robots.txt`.
|
126
142
|
|
127
143
|
`--print-status`
|
128
|
-
|
144
|
+
: Print the status codes for each URL.
|
129
145
|
|
130
146
|
`--print-headers`
|
131
|
-
|
147
|
+
: Print response headers for each URL.
|
132
148
|
|
133
149
|
`--print-header` *NAME*
|
134
|
-
|
150
|
+
: Prints a specific header.
|
135
151
|
|
136
152
|
`--history` *FILE*
|
137
|
-
|
153
|
+
: Sets the history file to write every visited URL to.
|
138
154
|
|
139
155
|
`--archive` *DIR*
|
140
|
-
|
156
|
+
: Archive every visited page to the *DIR*.
|
141
157
|
|
142
158
|
`--git-archive` *DIR*
|
143
|
-
|
159
|
+
: Archive every visited page to the git repository.
|
144
160
|
|
145
161
|
`-X`, `--xpath` *XPATH*
|
146
|
-
|
162
|
+
: Evaluates the XPath on each HTML page.
|
147
163
|
|
148
164
|
`-C`, `--css-path` *XPATH*
|
149
|
-
|
165
|
+
: Evaluates the CSS-path on each HTML page.
|
166
|
+
|
167
|
+
`--print-hosts`
|
168
|
+
: Print all discovered hostnames.
|
169
|
+
|
170
|
+
`--print-certs`
|
171
|
+
: Print all encountered SSL/TLS certificates.
|
172
|
+
|
173
|
+
`--save-certs`
|
174
|
+
: Saves all encountered SSL/TLS certificates.
|
175
|
+
|
176
|
+
`--print-js-strings`
|
177
|
+
: Print all JavaScript strings.
|
178
|
+
|
179
|
+
`--print-js-url-strings`
|
180
|
+
: Print URL strings found in JavaScript.
|
181
|
+
|
182
|
+
`--print-js-path-strings`
|
183
|
+
: Print path strings found in JavaScript.
|
184
|
+
|
185
|
+
`--print-js-absolute-path-strings`
|
186
|
+
: Only print absolute path strings found in JavaScript.
|
187
|
+
|
188
|
+
`--print-js-relative-path-strings`
|
189
|
+
: Only print relative path strings found in JavaScript.
|
190
|
+
|
191
|
+
`--print-html-comments`
|
192
|
+
: Print HTML comments.
|
193
|
+
|
194
|
+
`--print-js-comments`
|
195
|
+
: Print JavaScript comments.
|
196
|
+
|
197
|
+
`--print-comments`
|
198
|
+
: Print all HTML and JavaScript comments.
|
150
199
|
|
151
200
|
`-v`, `--verbose`
|
152
|
-
|
201
|
+
: Enables verbose output.
|
153
202
|
|
154
203
|
`-h`, `--help`
|
155
|
-
|
204
|
+
: Print help information.
|
156
205
|
|
157
206
|
## ENVIRONMENT
|
158
207
|
|
159
208
|
*HTTP_PROXY*
|
160
|
-
|
209
|
+
: Sets the global HTTP proxy.
|
161
210
|
|
162
211
|
*RONIN_HTTP_PROXY*
|
163
|
-
|
212
|
+
: Sets the HTTP proxy for Ronin.
|
164
213
|
|
165
214
|
## AUTHOR
|
166
215
|
|
@@ -168,4 +217,4 @@ Postmodern <postmodern.mod3@gmail.com>
|
|
168
217
|
|
169
218
|
## SEE ALSO
|
170
219
|
|
171
|
-
ronin-web-server(1) ronin-web-proxy(1) ronin-web-diff(1) ronin-web-new-spider(1)
|
220
|
+
[ronin-web-server](ronin-web-server.1.md) [ronin-web-proxy](ronin-web-proxy.1.md) [ronin-web-diff](ronin-web-diff.1.md) [ronin-web-new-spider](ronin-web-new-spider.1.md)
|
@@ -0,0 +1,44 @@
|
|
1
|
+
.\" Generated by kramdown-man 1.0.1
|
2
|
+
.\" https://github.com/postmodern/kramdown-man#readme
|
3
|
+
.TH ronin-web-user-agent 1 "2023-03-01" Ronin "User Manuals"
|
4
|
+
.SH NAME
|
5
|
+
.PP
|
6
|
+
ronin\-web\-user\-agent \- Generates a random User\-Agent string
|
7
|
+
.SH SYNOPSIS
|
8
|
+
.PP
|
9
|
+
\fBronin\-web user\-agent\fR \[lB]\fIoptions\fP\[rB]
|
10
|
+
.SH DESCRIPTION
|
11
|
+
.PP
|
12
|
+
Generates a random HTTP \fBUser\-Agent\fR string\.
|
13
|
+
.SH OPTIONS
|
14
|
+
.TP
|
15
|
+
\fB\-B\fR, \fB\-\-browser\fR \fBchrome\fR\[or]\fBfirefox\fR
|
16
|
+
Selects the desired browser type for the \fBUser\-Agent\fR string\.
|
17
|
+
.TP
|
18
|
+
\fB\-\-chrome\-version\fR \fIVERSION\fP
|
19
|
+
Sets desired Chrome version\. Only takes effect when \fB\-\-browser chrome\fR is also
|
20
|
+
given\.
|
21
|
+
.TP
|
22
|
+
\fB\-\-firefox\-version\fR \fIVERSION\fP
|
23
|
+
Sets desired Firefox version\. Only takes effect when \fB\-\-browser chrome\fR is
|
24
|
+
also given\.
|
25
|
+
.TP
|
26
|
+
\fB\-D\fR, \fB\-\-linux\-distro\fR \fBubuntu\fR\[or]\fBfedora\fR\[or]\fBarch\fR\[or]\fBDISTRO\fR
|
27
|
+
Selects the desired Linux distro\.
|
28
|
+
.TP
|
29
|
+
\fB\-A\fR, \fB\-\-arch\fR \fBx86\-64\fR\[or]\fBx86\fR\[or]\fBi686\fR\[or]\fBaarch64\fR\[or]\fBarm64\fR\[or]\fBarm\fR
|
30
|
+
Selects the desired architecture\.
|
31
|
+
.TP
|
32
|
+
\fB\-O\fR, \fB\-\-os\fR \fBandroid\fR\[or]\fBlinux\fR\[or]\fBwindows\fR
|
33
|
+
Selects the desired OS\.
|
34
|
+
.TP
|
35
|
+
\fB\-\-os\-version\fR \fIVERSION\fP
|
36
|
+
Sets the desired OS version\.
|
37
|
+
.TP
|
38
|
+
\fB\-h\fR, \fB\-\-help\fR
|
39
|
+
Print help information\.
|
40
|
+
.SH AUTHOR
|
41
|
+
.PP
|
42
|
+
Postmodern
|
43
|
+
.MT postmodern\.mod3\[at]gmail\.com
|
44
|
+
.ME
|