ronin-web 0.3.0.pre2 → 1.0.0.beta1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.editorconfig +11 -0
- data/.github/workflows/ruby.yml +31 -0
- data/.gitignore +13 -0
- data/.mailmap +1 -0
- data/.ruby-version +1 -0
- data/COPYING.txt +3 -3
- data/ChangeLog.md +115 -67
- data/Gemfile +42 -37
- data/README.md +159 -144
- data/Rakefile +16 -7
- data/bin/ronin-web +10 -18
- data/data/new/nokogiri.rb.erb +12 -0
- data/data/new/server.rb.erb +22 -0
- data/data/new/spider.rb.erb +26 -0
- data/data/new/webapp/.gitignore +15 -0
- data/data/new/webapp/.ruby-version.erb +1 -0
- data/data/new/webapp/Dockerfile.erb +11 -0
- data/data/new/webapp/Gemfile +6 -0
- data/data/new/webapp/app.rb.erb +15 -0
- data/data/new/webapp/config.ru +4 -0
- data/data/new/webapp/docker-compose.yml.erb +9 -0
- data/gemspec.yml +32 -17
- data/lib/ronin/web/cli/command.rb +36 -0
- data/lib/ronin/web/cli/commands/diff.rb +106 -0
- data/lib/ronin/web/cli/commands/html.rb +174 -0
- data/lib/ronin/web/cli/commands/irb.rb +56 -0
- data/lib/ronin/web/cli/commands/new/nokogiri.rb +85 -0
- data/lib/ronin/web/cli/commands/new/server.rb +96 -0
- data/lib/ronin/web/cli/commands/new/spider.rb +315 -0
- data/lib/ronin/web/cli/commands/new/webapp.rb +123 -0
- data/lib/ronin/web/cli/commands/new.rb +64 -0
- data/lib/ronin/web/cli/commands/reverse_proxy.rb +215 -0
- data/lib/ronin/web/cli/commands/server.rb +155 -0
- data/lib/ronin/web/cli/commands/spider.rb +822 -0
- data/lib/ronin/web/cli/ruby_shell.rb +50 -0
- data/lib/ronin/web/cli.rb +44 -0
- data/lib/ronin/web/html.rb +85 -0
- data/lib/ronin/web/mechanize.rb +79 -0
- data/lib/ronin/web/root.rb +27 -0
- data/lib/ronin/web/version.rb +7 -10
- data/lib/ronin/web/xml.rb +85 -0
- data/lib/ronin/web.rb +372 -13
- data/man/ronin-web-diff.1 +41 -0
- data/man/ronin-web-diff.1.md +30 -0
- data/man/ronin-web-html.1 +89 -0
- data/man/ronin-web-html.1.md +66 -0
- data/man/ronin-web-irb.1 +31 -0
- data/man/ronin-web-irb.1.md +22 -0
- data/man/ronin-web-new-nokogiri.1 +41 -0
- data/man/ronin-web-new-nokogiri.1.md +30 -0
- data/man/ronin-web-new-server.1 +45 -0
- data/man/ronin-web-new-server.1.md +33 -0
- data/man/ronin-web-new-spider.1 +173 -0
- data/man/ronin-web-new-spider.1.md +129 -0
- data/man/ronin-web-new-webapp.1 +53 -0
- data/man/ronin-web-new-webapp.1.md +39 -0
- data/man/ronin-web-new.1 +59 -0
- data/man/ronin-web-new.1.md +44 -0
- data/man/ronin-web-reverse-proxy.1 +63 -0
- data/man/ronin-web-reverse-proxy.1.md +47 -0
- data/man/ronin-web-server.1 +59 -0
- data/man/ronin-web-server.1.md +43 -0
- data/man/ronin-web-spider.1 +225 -0
- data/man/ronin-web-spider.1.md +168 -0
- data/man/ronin-web.1 +41 -0
- data/man/ronin-web.1.md +30 -0
- data/ronin-web.gemspec +57 -11
- data/spec/cli/ruby_shell_spec.rb +14 -0
- data/spec/html_spec.rb +43 -0
- data/spec/mechanize_spec.rb +72 -0
- data/spec/spec_helper.rb +5 -3
- data/spec/web_spec.rb +97 -0
- data/spec/xml_spec.rb +42 -0
- metadata +237 -217
- data/.gemtest +0 -0
- data/lib/ronin/network/mixins/web.rb +0 -256
- data/lib/ronin/web/extensions/nokogiri/xml/attr.rb +0 -47
- data/lib/ronin/web/extensions/nokogiri/xml/document.rb +0 -48
- data/lib/ronin/web/extensions/nokogiri/xml/element.rb +0 -57
- data/lib/ronin/web/extensions/nokogiri/xml/node.rb +0 -86
- data/lib/ronin/web/extensions/nokogiri/xml/text.rb +0 -47
- data/lib/ronin/web/extensions/nokogiri/xml.rb +0 -27
- data/lib/ronin/web/extensions/nokogiri.rb +0 -23
- data/lib/ronin/web/extensions.rb +0 -23
- data/lib/ronin/web/middleware/base.rb +0 -144
- data/lib/ronin/web/middleware/directories.rb +0 -179
- data/lib/ronin/web/middleware/files.rb +0 -144
- data/lib/ronin/web/middleware/filters/campaign_filter.rb +0 -77
- data/lib/ronin/web/middleware/filters/ip_filter.rb +0 -73
- data/lib/ronin/web/middleware/filters/path_filter.rb +0 -73
- data/lib/ronin/web/middleware/filters/referer_filter.rb +0 -71
- data/lib/ronin/web/middleware/filters/user_agent_filter.rb +0 -71
- data/lib/ronin/web/middleware/filters/vhost_filter.rb +0 -71
- data/lib/ronin/web/middleware/filters.rb +0 -28
- data/lib/ronin/web/middleware/helpers.rb +0 -145
- data/lib/ronin/web/middleware/proxy.rb +0 -265
- data/lib/ronin/web/middleware/proxy_request.rb +0 -262
- data/lib/ronin/web/middleware/request.rb +0 -79
- data/lib/ronin/web/middleware/response.rb +0 -33
- data/lib/ronin/web/middleware/router.rb +0 -167
- data/lib/ronin/web/middleware/rule.rb +0 -103
- data/lib/ronin/web/middleware.rb +0 -27
- data/lib/ronin/web/proxy/app.rb +0 -32
- data/lib/ronin/web/proxy/base.rb +0 -46
- data/lib/ronin/web/proxy/web.rb +0 -46
- data/lib/ronin/web/proxy.rb +0 -25
- data/lib/ronin/web/server/app.rb +0 -32
- data/lib/ronin/web/server/base.rb +0 -461
- data/lib/ronin/web/server/web.rb +0 -66
- data/lib/ronin/web/server.rb +0 -25
- data/lib/ronin/web/spider.rb +0 -115
- data/lib/ronin/web/web.rb +0 -573
- data/spec/helpers/output.rb +0 -3
- data/spec/web/extensions/nokogiri_spec.rb +0 -38
- data/spec/web/helpers/rack_app.rb +0 -31
- data/spec/web/helpers/root/test1/index.html +0 -1
- data/spec/web/helpers/root/test1/test1.txt +0 -1
- data/spec/web/helpers/root/test1.txt +0 -1
- data/spec/web/helpers/root/test2/test2.txt +0 -1
- data/spec/web/helpers/root/test2.txt +0 -1
- data/spec/web/helpers/root/test3/test3.txt +0 -1
- data/spec/web/helpers/root/test3.txt +0 -1
- data/spec/web/helpers/root.rb +0 -15
- data/spec/web/middleware/directories_spec.rb +0 -86
- data/spec/web/middleware/files_spec.rb +0 -57
- data/spec/web/middleware/filters/campaign_filter_spec.rb +0 -30
- data/spec/web/middleware/filters/ip_filter_spec.rb +0 -25
- data/spec/web/middleware/filters/path_filter_spec.rb +0 -29
- data/spec/web/middleware/filters/referer_filter_spec.rb +0 -25
- data/spec/web/middleware/filters/user_agent_filter_spec.rb +0 -25
- data/spec/web/middleware/filters/vhost_filter_spec.rb +0 -23
- data/spec/web/middleware/proxy_spec.rb +0 -67
- data/spec/web/middleware/response_spec.rb +0 -20
- data/spec/web/middleware/router_spec.rb +0 -65
- data/spec/web/middleware/rule_spec.rb +0 -37
- data/spec/web/proxy/base_spec.rb +0 -8
- data/spec/web/server/base_spec.rb +0 -77
- data/spec/web/server/classes/public1/static1.txt +0 -1
- data/spec/web/server/classes/public2/static2.txt +0 -1
- data/spec/web/server/classes/sub_app.rb +0 -13
- data/spec/web/server/classes/test_app.rb +0 -20
- data/spec/web/web_spec.rb +0 -157
@@ -0,0 +1,47 @@
|
|
1
|
+
# ronin-web-proxy 1 "2022-01-01" Ronin "User Manuals"
|
2
|
+
|
3
|
+
## SYNOPSIS
|
4
|
+
|
5
|
+
`ronin-web proxy` [*options*] [`--host` *HOST*] [`--port` *PORT*]
|
6
|
+
|
7
|
+
## DESCRIPTION
|
8
|
+
|
9
|
+
Starts a HTTP proxy server.
|
10
|
+
|
11
|
+
## OPTIONS
|
12
|
+
|
13
|
+
`-H`, `--host` *HOST*
|
14
|
+
The host that the proxy server will listen on. Defaults to `localhost`.
|
15
|
+
|
16
|
+
`-p`, `--port` *PORT*
|
17
|
+
The port that the proxy server will listen on. Default to `8080`.
|
18
|
+
|
19
|
+
`-b`, `--show-body`
|
20
|
+
Controls whether to display the request/response bodies or not.
|
21
|
+
|
22
|
+
`--rewrite-requests` {*STRING*:*REPLACE*|/*REGEXP*/:*REPLACE*}
|
23
|
+
Rewrites all request bodies by replacing the *STRING* or *REGEXP* with the
|
24
|
+
given *REPLACE* string.
|
25
|
+
|
26
|
+
`--rewrite-responses` {*STRING*:*REPLACE*|/*REGEXP*/:*REPLACE*}
|
27
|
+
Rewrites all response bodies by replacing the *STRING* or *REGEXP* with the
|
28
|
+
given *REPLACE* string.
|
29
|
+
|
30
|
+
`-h`, `--help`
|
31
|
+
prints help information.
|
32
|
+
|
33
|
+
## EXAMPLES
|
34
|
+
|
35
|
+
`sudo ronin-web proxy --host 0.0.0.0 --port 80`
|
36
|
+
Listen on the external interface on port 80.
|
37
|
+
|
38
|
+
`ronin-web proxy --rewrite-responses https:http`
|
39
|
+
Replace every occurrence of `https` with `http` in the response bodies.
|
40
|
+
|
41
|
+
## AUTHOR
|
42
|
+
|
43
|
+
Postmodern <postmodern.mod3@gmail.com>
|
44
|
+
|
45
|
+
## SEE ALSO
|
46
|
+
|
47
|
+
ronin-web-server(1)
|
@@ -0,0 +1,59 @@
|
|
1
|
+
.\" Generated by kramdown-man 0.1.8
|
2
|
+
.\" https://github.com/postmodern/kramdown-man#readme
|
3
|
+
.TH ronin-web-server 1 "2022-01-01" Ronin Web "User Manuals"
|
4
|
+
.LP
|
5
|
+
.SH SYNOPSIS
|
6
|
+
.LP
|
7
|
+
.HP
|
8
|
+
\fBronin-web-server\fR \[lB]\fIoptions\fP\[rB]
|
9
|
+
.LP
|
10
|
+
.SH DESCRIPTION
|
11
|
+
.LP
|
12
|
+
.PP
|
13
|
+
Starts a web server\.
|
14
|
+
.LP
|
15
|
+
.SH OPTIONS
|
16
|
+
.LP
|
17
|
+
.TP
|
18
|
+
\fB-H\fR, \fB--host\fR \fIHOST\fP
|
19
|
+
Host name or IP to bind to\. Defaults to \fBlocalhost\fR\.
|
20
|
+
.LP
|
21
|
+
.TP
|
22
|
+
\fB-p\fR, \fB--port\fR \fIPORT\fP
|
23
|
+
Port number to listen on\. Defaults to \fB8000\fR\.
|
24
|
+
.LP
|
25
|
+
.TP
|
26
|
+
\fB-A\fR, \fB--basic-auth\fR \fIUSER\fP:\fIPASSWORD\fP
|
27
|
+
Sets up Basic\-Authentication with the given \fIUSER\fP and \fIPASSWORD\fP\.
|
28
|
+
.LP
|
29
|
+
.TP
|
30
|
+
\fB-d\fR, \fB--dir\fR \[sl]\fIPATH\fP:\fIDIR\fP
|
31
|
+
Mounts a directory to the given \fIPATH\fP\.
|
32
|
+
.LP
|
33
|
+
.TP
|
34
|
+
\fB-f\fR, \fB--file\fR \[sl]\fIPATH\fP:\fIFILE\fP
|
35
|
+
Mounts a file to the given \fIPATH\fP\.
|
36
|
+
.LP
|
37
|
+
.TP
|
38
|
+
\fB-r\fR, \fB--root\fR \fIDIR\fP
|
39
|
+
Root directory to serve\.
|
40
|
+
.LP
|
41
|
+
.TP
|
42
|
+
\fB-R\fR, \fB--redirect\fR \[sl]\fIPATH\fP:\fIURL\fP
|
43
|
+
Registers a \fB302 Found\fR redirect at the given \fIPATH\fP
|
44
|
+
.LP
|
45
|
+
.TP
|
46
|
+
\fB-h\fR, \fB--help\fR
|
47
|
+
Print help information
|
48
|
+
.LP
|
49
|
+
.SH AUTHOR
|
50
|
+
.LP
|
51
|
+
.PP
|
52
|
+
Postmodern
|
53
|
+
.MT postmodern\.mod3\[at]gmail\.com
|
54
|
+
.ME
|
55
|
+
.LP
|
56
|
+
.SH SEE ALSO
|
57
|
+
.LP
|
58
|
+
.PP
|
59
|
+
ronin\-web\-proxy(1)
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# ronin-web-server 1 "2022-01-01" Ronin Web "User Manuals"
|
2
|
+
|
3
|
+
## SYNOPSIS
|
4
|
+
|
5
|
+
`ronin-web-server` [*options*]
|
6
|
+
|
7
|
+
## DESCRIPTION
|
8
|
+
|
9
|
+
Starts a web server.
|
10
|
+
|
11
|
+
## OPTIONS
|
12
|
+
|
13
|
+
`-H`, `--host` *HOST*
|
14
|
+
Host name or IP to bind to. Defaults to `localhost`.
|
15
|
+
|
16
|
+
`-p`, `--port` *PORT*
|
17
|
+
Port number to listen on. Defaults to `8000`.
|
18
|
+
|
19
|
+
`-A`, `--basic-auth` *USER*:*PASSWORD*
|
20
|
+
Sets up Basic-Authentication with the given *USER* and *PASSWORD*.
|
21
|
+
|
22
|
+
`-d`, `--dir` /*PATH*:*DIR*
|
23
|
+
Mounts a directory to the given *PATH*.
|
24
|
+
|
25
|
+
`-f`, `--file` /*PATH*:*FILE*
|
26
|
+
Mounts a file to the given *PATH*.
|
27
|
+
|
28
|
+
`-r`, `--root` *DIR*
|
29
|
+
Root directory to serve.
|
30
|
+
|
31
|
+
`-R`, `--redirect` /*PATH*:*URL*
|
32
|
+
Registers a `302 Found` redirect at the given *PATH*
|
33
|
+
|
34
|
+
`-h`, `--help`
|
35
|
+
Print help information
|
36
|
+
|
37
|
+
## AUTHOR
|
38
|
+
|
39
|
+
Postmodern <postmodern.mod3@gmail.com>
|
40
|
+
|
41
|
+
## SEE ALSO
|
42
|
+
|
43
|
+
ronin-web-proxy(1)
|
@@ -0,0 +1,225 @@
|
|
1
|
+
.\" Generated by kramdown-man 0.1.8
|
2
|
+
.\" https://github.com/postmodern/kramdown-man#readme
|
3
|
+
.TH ronin-web-spider 1 "2022-01-01" Ronin Web "User Manuals"
|
4
|
+
.LP
|
5
|
+
.SH SYNOPSIS
|
6
|
+
.LP
|
7
|
+
.HP
|
8
|
+
\fBronin-web-spider\fR \[lB]\fIoptions\fP\[rB] \[lC]\fB--host\fR \fIHOST\fP \[or] \fB--domain\fR \fIDOMAIN\fP \[or] \fB--site\fR \fIURL\fP\[rC]
|
9
|
+
.LP
|
10
|
+
.SH DESCRIPTION
|
11
|
+
.LP
|
12
|
+
.PP
|
13
|
+
Spiders a website\.
|
14
|
+
.LP
|
15
|
+
.SH OPTIONS
|
16
|
+
.LP
|
17
|
+
.TP
|
18
|
+
\fB--open-timeout\fR \fISECS\fP
|
19
|
+
Sets the connection open timeout\.
|
20
|
+
.LP
|
21
|
+
.TP
|
22
|
+
\fB--read-timeout\fR \fISECS\fP
|
23
|
+
Sets the read timeout\.
|
24
|
+
.LP
|
25
|
+
.TP
|
26
|
+
\fB--ssl-timeout\fR \fISECS\fP
|
27
|
+
Sets the SSL connection timeout\.
|
28
|
+
.LP
|
29
|
+
.TP
|
30
|
+
\fB--continue-timeout\fR \fISECS\fP
|
31
|
+
Sets the continue timeout\.
|
32
|
+
.LP
|
33
|
+
.TP
|
34
|
+
\fB--keep-alive-timeout\fR \fISECS\fP
|
35
|
+
Sets the connection keep alive timeout\.
|
36
|
+
.LP
|
37
|
+
.TP
|
38
|
+
\fB-P\fR, \fB--proxy\fR \fIPROXY\fP
|
39
|
+
Sets the proxy to use\.
|
40
|
+
.LP
|
41
|
+
.TP
|
42
|
+
\fB-H\fR, \fB--header\fR \[lq]\fINAME\fP: \fIVALUE\fP\[rq]
|
43
|
+
Sets a default header\.
|
44
|
+
.LP
|
45
|
+
.TP
|
46
|
+
\fB--host-header\fR \fINAME\fP\[eq]\fIVALUE\fP
|
47
|
+
Sets a default header\.
|
48
|
+
.LP
|
49
|
+
.HP
|
50
|
+
\fB-u\fR, \fB--user-agent\fR chrome\[ru]linux\[or]chrome\[ru]macos\[or]chrome\[ru]windows\[or]chrome\[ru]iphone\[or]chrome\[ru]ipad\[or]chrome\[ru]android\[or]firefox\[ru]linux\[or]firefox\[ru]macos\[or]firefox\[ru]windows\[or]firefox\[ru]iphone\[or]firefox\[ru]ipad\[or]firefox\[ru]android\[or]safari\[ru]macos\[or]safari\[ru]iphone\[or]safari\[ru]ipad\[or]edge
|
51
|
+
The \fBUser-Agent\fR to use\.
|
52
|
+
.LP
|
53
|
+
.TP
|
54
|
+
\fB-U\fR, \fB--user-agent-string\fR \fISTRING\fP
|
55
|
+
The raw \fBUser-Agent\fR string to use\.
|
56
|
+
.LP
|
57
|
+
.TP
|
58
|
+
\fB-R\fR, \fB--referer\fR \fIURL\fP
|
59
|
+
Sets the \fBReferer\fR URL\.
|
60
|
+
.LP
|
61
|
+
.TP
|
62
|
+
\fB--delay\fR \fISECS\fP
|
63
|
+
Sets the delay in seconds between each request\.
|
64
|
+
.LP
|
65
|
+
.TP
|
66
|
+
\fB-l\fR, \fB--limit\fR \fICOUNT\fP
|
67
|
+
Only spiders up to \fICOUNT\fP pages\.
|
68
|
+
.LP
|
69
|
+
.TP
|
70
|
+
\fB-d\fR, \fB--max-depth\fR \fIDEPTH\fP
|
71
|
+
Only spiders up to max depth\.
|
72
|
+
.LP
|
73
|
+
.TP
|
74
|
+
\fB--enqueue\fR \fIURL\fP
|
75
|
+
Adds the URL to the queue\.
|
76
|
+
.LP
|
77
|
+
.TP
|
78
|
+
\fB--visited\fR \fIURL\fP
|
79
|
+
Marks the URL as previously visited\.
|
80
|
+
.LP
|
81
|
+
.TP
|
82
|
+
\fB--strip-fragments\fR
|
83
|
+
Enables\[sl]disables stripping the fragment component of every URL\.
|
84
|
+
.LP
|
85
|
+
.TP
|
86
|
+
\fB--strip-query\fR
|
87
|
+
Enables\[sl]disables stripping the query component of every URL\.
|
88
|
+
.LP
|
89
|
+
.TP
|
90
|
+
\fB--visit-host\fR \fIHOST\fP
|
91
|
+
Visit URLs with the matching host name\.
|
92
|
+
.LP
|
93
|
+
.HP
|
94
|
+
\fB--visit-hosts-like\fR \[sl]\fIREGEX\fP\[sl]
|
95
|
+
Visit URLs with hostnames that match the \fIREGEX\fP\.
|
96
|
+
.LP
|
97
|
+
.TP
|
98
|
+
\fB--ignore-host\fR \fIHOST\fP
|
99
|
+
Ignore the host name\.
|
100
|
+
.LP
|
101
|
+
.HP
|
102
|
+
\fB--ignore-hosts-like\fR \[sl]\fIREGEX\fP\[sl]
|
103
|
+
Ignore the host names matching the \fIREGEX\fP\.
|
104
|
+
.LP
|
105
|
+
.TP
|
106
|
+
\fB--visit-port\fR \fIPORT\fP
|
107
|
+
Visit URLs with the matching port number\.
|
108
|
+
.LP
|
109
|
+
.HP
|
110
|
+
\fB--visit-ports-like\fR \[sl]\fIREGEX\fP\[sl]
|
111
|
+
Visit URLs with port numbers that match the \fIREGEX\fP\.
|
112
|
+
.LP
|
113
|
+
.TP
|
114
|
+
\fB--ignore-port\fR \fIPORT\fP
|
115
|
+
Ignore the port number\.
|
116
|
+
.LP
|
117
|
+
.HP
|
118
|
+
\fB--ignore-ports-like\fR \[sl]\fIREGEX\fP\[sl]
|
119
|
+
Ignore the port numbers matching the \fIREGEXP\fP\.
|
120
|
+
.LP
|
121
|
+
.TP
|
122
|
+
\fB--visit-link\fR \fIURL\fP
|
123
|
+
Visit the \fIURL\fP\.
|
124
|
+
.LP
|
125
|
+
.HP
|
126
|
+
\fB--visit-links-like\fR \[sl]\fIREGEX\fP\[sl]
|
127
|
+
Visit URLs that match the \fIREGEX\fP\.
|
128
|
+
.LP
|
129
|
+
.TP
|
130
|
+
\fB--ignore-link\fR \fIURL\fP
|
131
|
+
Ignore the \fIURL\fP\.
|
132
|
+
.LP
|
133
|
+
.HP
|
134
|
+
\fB--ignore-links-like\fR \[sl]\fIREGEX\fP\[sl]
|
135
|
+
Ignore URLs matching the \fIREGEX\fP\.
|
136
|
+
.LP
|
137
|
+
.TP
|
138
|
+
\fB--visit-ext\fR \fIFILE\[ru]EXT\fP
|
139
|
+
Visit URLs with the matching file ext\.
|
140
|
+
.LP
|
141
|
+
.HP
|
142
|
+
\fB--visit-exts-like\fR \[sl]\fIREGEX\fP\[sl]
|
143
|
+
Visit URLs with file exts that match the \fIREGEX\fP\.
|
144
|
+
.LP
|
145
|
+
.TP
|
146
|
+
\fB--ignore-ext\fR \fIFILE\[ru]EXT\fP
|
147
|
+
Ignore the URLs with the file ext\.
|
148
|
+
.LP
|
149
|
+
.HP
|
150
|
+
\fB--ignore-exts-like\fR \[sl]\fIREGEX\fP\[sl]
|
151
|
+
Ignore URLs with file exts matching the REGEX\.
|
152
|
+
.LP
|
153
|
+
.TP
|
154
|
+
\fB-r\fR, \fB--robots\fR
|
155
|
+
Specifies whether to honor \fBrobots.txt\fR\.
|
156
|
+
.LP
|
157
|
+
.TP
|
158
|
+
\fB--host\fR \fIHOST\fP
|
159
|
+
Spiders the specific \fIHOST\fP\.
|
160
|
+
.LP
|
161
|
+
.TP
|
162
|
+
\fB--domain\fR \fIDOMAIN\fP
|
163
|
+
Spiders the whole \fIDOMAIN\fP\.
|
164
|
+
.LP
|
165
|
+
.TP
|
166
|
+
\fB--site\fR \fIURL\fP
|
167
|
+
Spiders the website, starting at the \fIURL\fP\.
|
168
|
+
.LP
|
169
|
+
.TP
|
170
|
+
\fB--print-status\fR
|
171
|
+
Print the status codes for each URL\.
|
172
|
+
.LP
|
173
|
+
.TP
|
174
|
+
\fB--print-headers\fR
|
175
|
+
Print response headers for each URL\.
|
176
|
+
.LP
|
177
|
+
.TP
|
178
|
+
\fB--print-header\fR \fINAME\fP
|
179
|
+
Prints a specific header\.
|
180
|
+
.LP
|
181
|
+
.TP
|
182
|
+
\fB--archive\fR \fIDIR\fP
|
183
|
+
Archive every visited page to the \fIDIR\fP\.
|
184
|
+
.LP
|
185
|
+
.TP
|
186
|
+
\fB--git-archive\fR \fIDIR\fP
|
187
|
+
Archive every visited page to the git repository\.
|
188
|
+
.LP
|
189
|
+
.TP
|
190
|
+
\fB-X\fR, \fB--xpath\fR \fIXPATH\fP
|
191
|
+
Evaluates the XPath on each HTML page\.
|
192
|
+
.LP
|
193
|
+
.TP
|
194
|
+
\fB-C\fR, \fB--css-path\fR \fIXPATH\fP
|
195
|
+
Evaluates the CSS\-path on each HTML page\.
|
196
|
+
.LP
|
197
|
+
.TP
|
198
|
+
\fB-v\fR, \fB--verbose\fR
|
199
|
+
Enables verbose output\.
|
200
|
+
.LP
|
201
|
+
.TP
|
202
|
+
\fB-h\fR, \fB--help\fR
|
203
|
+
Print help information\.
|
204
|
+
.LP
|
205
|
+
.SH ENVIRONMENT
|
206
|
+
.LP
|
207
|
+
.TP
|
208
|
+
\fIHTTP\[ru]PROXY\fP
|
209
|
+
Sets the global HTTP proxy\.
|
210
|
+
.LP
|
211
|
+
.TP
|
212
|
+
\fIRONIN\[ru]HTTP\[ru]PROXY\fP
|
213
|
+
Sets the HTTP proxy for Ronin\.
|
214
|
+
.LP
|
215
|
+
.SH AUTHOR
|
216
|
+
.LP
|
217
|
+
.PP
|
218
|
+
Postmodern
|
219
|
+
.MT postmodern\.mod3\[at]gmail\.com
|
220
|
+
.ME
|
221
|
+
.LP
|
222
|
+
.SH SEE ALSO
|
223
|
+
.LP
|
224
|
+
.PP
|
225
|
+
ronin\-web\-server(1) ronin\-web\-proxy(1) ronin\-web\-diff(1) ronin\-web\-new\-spider(1)
|
@@ -0,0 +1,168 @@
|
|
1
|
+
# ronin-web-spider 1 "2022-01-01" Ronin Web "User Manuals"
|
2
|
+
|
3
|
+
## SYNOPSIS
|
4
|
+
|
5
|
+
`ronin-web-spider` [*options*] {`--host` *HOST* \| `--domain` *DOMAIN* \| `--site` *URL*}
|
6
|
+
|
7
|
+
## DESCRIPTION
|
8
|
+
|
9
|
+
Spiders a website.
|
10
|
+
|
11
|
+
## OPTIONS
|
12
|
+
|
13
|
+
`--open-timeout` *SECS*
|
14
|
+
Sets the connection open timeout.
|
15
|
+
|
16
|
+
`--read-timeout` *SECS*
|
17
|
+
Sets the read timeout.
|
18
|
+
|
19
|
+
`--ssl-timeout` *SECS*
|
20
|
+
Sets the SSL connection timeout.
|
21
|
+
|
22
|
+
`--continue-timeout` *SECS*
|
23
|
+
Sets the continue timeout.
|
24
|
+
|
25
|
+
`--keep-alive-timeout` *SECS*
|
26
|
+
Sets the connection keep alive timeout.
|
27
|
+
|
28
|
+
`-P`, `--proxy` *PROXY*
|
29
|
+
Sets the proxy to use.
|
30
|
+
|
31
|
+
`-H`, `--header` "*NAME*: *VALUE*"
|
32
|
+
Sets a default header.
|
33
|
+
|
34
|
+
`--host-header` *NAME*=*VALUE*
|
35
|
+
Sets a default header.
|
36
|
+
|
37
|
+
`-u`, `--user-agent` chrome_linux|chrome_macos|chrome_windows|chrome_iphone|chrome_ipad|chrome_android|firefox_linux|firefox_macos|firefox_windows|firefox_iphone|firefox_ipad|firefox_android|safari_macos|safari_iphone|safari_ipad|edge
|
38
|
+
The `User-Agent` to use.
|
39
|
+
|
40
|
+
`-U`, `--user-agent-string` *STRING*
|
41
|
+
The raw `User-Agent` string to use.
|
42
|
+
|
43
|
+
`-R`, `--referer` *URL*
|
44
|
+
Sets the `Referer` URL.
|
45
|
+
|
46
|
+
`--delay` *SECS*
|
47
|
+
Sets the delay in seconds between each request.
|
48
|
+
|
49
|
+
`-l`, `--limit` *COUNT*
|
50
|
+
Only spiders up to *COUNT* pages.
|
51
|
+
|
52
|
+
`-d`, `--max-depth` *DEPTH*
|
53
|
+
Only spiders up to max depth.
|
54
|
+
|
55
|
+
`--enqueue` *URL*
|
56
|
+
Adds the URL to the queue.
|
57
|
+
|
58
|
+
`--visited` *URL*
|
59
|
+
Marks the URL as previously visited.
|
60
|
+
|
61
|
+
`--strip-fragments`
|
62
|
+
Enables/disables stripping the fragment component of every URL.
|
63
|
+
|
64
|
+
`--strip-query`
|
65
|
+
Enables/disables stripping the query component of every URL.
|
66
|
+
|
67
|
+
`--visit-host` *HOST*
|
68
|
+
Visit URLs with the matching host name.
|
69
|
+
|
70
|
+
`--visit-hosts-like` /*REGEX*/
|
71
|
+
Visit URLs with hostnames that match the *REGEX*.
|
72
|
+
|
73
|
+
`--ignore-host` *HOST*
|
74
|
+
Ignore the host name.
|
75
|
+
|
76
|
+
`--ignore-hosts-like` /*REGEX*/
|
77
|
+
Ignore the host names matching the *REGEX*.
|
78
|
+
|
79
|
+
`--visit-port` *PORT*
|
80
|
+
Visit URLs with the matching port number.
|
81
|
+
|
82
|
+
`--visit-ports-like` /*REGEX*/
|
83
|
+
Visit URLs with port numbers that match the *REGEX*.
|
84
|
+
|
85
|
+
`--ignore-port` *PORT*
|
86
|
+
Ignore the port number.
|
87
|
+
|
88
|
+
`--ignore-ports-like` /*REGEX*/
|
89
|
+
Ignore the port numbers matching the *REGEXP*.
|
90
|
+
|
91
|
+
`--visit-link` *URL*
|
92
|
+
Visit the *URL*.
|
93
|
+
|
94
|
+
`--visit-links-like` /*REGEX*/
|
95
|
+
Visit URLs that match the *REGEX*.
|
96
|
+
|
97
|
+
`--ignore-link` *URL*
|
98
|
+
Ignore the *URL*.
|
99
|
+
|
100
|
+
`--ignore-links-like` /*REGEX*/
|
101
|
+
Ignore URLs matching the *REGEX*.
|
102
|
+
|
103
|
+
`--visit-ext` *FILE_EXT*
|
104
|
+
Visit URLs with the matching file ext.
|
105
|
+
|
106
|
+
`--visit-exts-like` /*REGEX*/
|
107
|
+
Visit URLs with file exts that match the *REGEX*.
|
108
|
+
|
109
|
+
`--ignore-ext` *FILE_EXT*
|
110
|
+
Ignore the URLs with the file ext.
|
111
|
+
|
112
|
+
`--ignore-exts-like` /*REGEX*/
|
113
|
+
Ignore URLs with file exts matching the REGEX.
|
114
|
+
|
115
|
+
`-r`, `--robots`
|
116
|
+
Specifies whether to honor `robots.txt`.
|
117
|
+
|
118
|
+
`--host` *HOST*
|
119
|
+
Spiders the specific *HOST*.
|
120
|
+
|
121
|
+
`--domain` *DOMAIN*
|
122
|
+
Spiders the whole *DOMAIN*.
|
123
|
+
|
124
|
+
`--site` *URL*
|
125
|
+
Spiders the website, starting at the *URL*.
|
126
|
+
|
127
|
+
`--print-status`
|
128
|
+
Print the status codes for each URL.
|
129
|
+
|
130
|
+
`--print-headers`
|
131
|
+
Print response headers for each URL.
|
132
|
+
|
133
|
+
`--print-header` *NAME*
|
134
|
+
Prints a specific header.
|
135
|
+
|
136
|
+
`--archive` *DIR*
|
137
|
+
Archive every visited page to the *DIR*.
|
138
|
+
|
139
|
+
`--git-archive` *DIR*
|
140
|
+
Archive every visited page to the git repository.
|
141
|
+
|
142
|
+
`-X`, `--xpath` *XPATH*
|
143
|
+
Evaluates the XPath on each HTML page.
|
144
|
+
|
145
|
+
`-C`, `--css-path` *XPATH*
|
146
|
+
Evaluates the CSS-path on each HTML page.
|
147
|
+
|
148
|
+
`-v`, `--verbose`
|
149
|
+
Enables verbose output.
|
150
|
+
|
151
|
+
`-h`, `--help`
|
152
|
+
Print help information.
|
153
|
+
|
154
|
+
## ENVIRONMENT
|
155
|
+
|
156
|
+
*HTTP_PROXY*
|
157
|
+
Sets the global HTTP proxy.
|
158
|
+
|
159
|
+
*RONIN_HTTP_PROXY*
|
160
|
+
Sets the HTTP proxy for Ronin.
|
161
|
+
|
162
|
+
## AUTHOR
|
163
|
+
|
164
|
+
Postmodern <postmodern.mod3@gmail.com>
|
165
|
+
|
166
|
+
## SEE ALSO
|
167
|
+
|
168
|
+
ronin-web-server(1) ronin-web-proxy(1) ronin-web-diff(1) ronin-web-new-spider(1)
|
data/man/ronin-web.1
ADDED
@@ -0,0 +1,41 @@
|
|
1
|
+
.\" Generated by kramdown-man 0.1.8
|
2
|
+
.\" https://github.com/postmodern/kramdown-man#readme
|
3
|
+
.TH ronin-web 1 "2022-01-01" Ronin Web "User Manuals"
|
4
|
+
.LP
|
5
|
+
.SH SYNOPSIS
|
6
|
+
.LP
|
7
|
+
.HP
|
8
|
+
\fBronin-web\fR \[lB]\fIoptions\fP\[rB] \[lB]\fICOMMAND\fP \[lB]\fIARGS\fP\[rB]\[rB]
|
9
|
+
.LP
|
10
|
+
.SH DESCRIPTION
|
11
|
+
.LP
|
12
|
+
.PP
|
13
|
+
A collection of useful web commands\.
|
14
|
+
.LP
|
15
|
+
.SH ARGUMENTS
|
16
|
+
.LP
|
17
|
+
.TP
|
18
|
+
\fICOMMAND\fP
|
19
|
+
The sub\-command to execute\.
|
20
|
+
.LP
|
21
|
+
.TP
|
22
|
+
\fIARGS\fP
|
23
|
+
Additional arguments for the sub\-command\.
|
24
|
+
.LP
|
25
|
+
.SH OPTIONS
|
26
|
+
.LP
|
27
|
+
.TP
|
28
|
+
\fB-h\fR, \fB--help\fR
|
29
|
+
Prints help information\.
|
30
|
+
.LP
|
31
|
+
.SH AUTHOR
|
32
|
+
.LP
|
33
|
+
.PP
|
34
|
+
Postmodern
|
35
|
+
.MT postmodern\.mod3\[at]gmail\.com
|
36
|
+
.ME
|
37
|
+
.LP
|
38
|
+
.SH SEE ALSO
|
39
|
+
.LP
|
40
|
+
.PP
|
41
|
+
ronin\-web\-diff(1) ronin\-web\-html(1) ronin\-web\-server(1) ronin\-web\-spider(1) ronin\-web\-proxy(1)
|
data/man/ronin-web.1.md
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
# ronin-web 1 "2022-01-01" Ronin Web "User Manuals"
|
2
|
+
|
3
|
+
## SYNOPSIS
|
4
|
+
|
5
|
+
`ronin-web` [*options*] [*COMMAND* [*ARGS*]]
|
6
|
+
|
7
|
+
## DESCRIPTION
|
8
|
+
|
9
|
+
A collection of useful web commands.
|
10
|
+
|
11
|
+
## ARGUMENTS
|
12
|
+
|
13
|
+
*COMMAND*
|
14
|
+
The sub-command to execute.
|
15
|
+
|
16
|
+
*ARGS*
|
17
|
+
Additional arguments for the sub-command.
|
18
|
+
|
19
|
+
## OPTIONS
|
20
|
+
|
21
|
+
`-h`, `--help`
|
22
|
+
Prints help information.
|
23
|
+
|
24
|
+
## AUTHOR
|
25
|
+
|
26
|
+
Postmodern <postmodern.mod3@gmail.com>
|
27
|
+
|
28
|
+
## SEE ALSO
|
29
|
+
|
30
|
+
ronin-web-diff(1) ronin-web-html(1) ronin-web-server(1) ronin-web-spider(1) ronin-web-proxy(1)
|
data/ronin-web.gemspec
CHANGED
@@ -1,15 +1,61 @@
|
|
1
|
-
#
|
1
|
+
# encoding: utf-8
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
|
3
|
+
require 'yaml'
|
4
|
+
|
5
|
+
Gem::Specification.new do |gem|
|
6
|
+
gemspec = YAML.load_file('gemspec.yml')
|
7
|
+
|
8
|
+
gem.name = gemspec.fetch('name')
|
9
|
+
gem.version = gemspec.fetch('version') do
|
10
|
+
lib_dir = File.join(File.dirname(__FILE__),'lib')
|
11
|
+
$LOAD_PATH << lib_dir unless $LOAD_PATH.include?(lib_dir)
|
12
|
+
|
13
|
+
require 'ronin/web/version'
|
14
|
+
Ronin::Web::VERSION
|
15
|
+
end
|
16
|
+
|
17
|
+
gem.summary = gemspec['summary']
|
18
|
+
gem.description = gemspec['description']
|
19
|
+
gem.licenses = Array(gemspec['license'])
|
20
|
+
gem.authors = Array(gemspec['authors'])
|
21
|
+
gem.email = gemspec['email']
|
22
|
+
gem.homepage = gemspec['homepage']
|
23
|
+
|
24
|
+
glob = lambda { |patterns| gem.files & Dir[*patterns] }
|
25
|
+
|
26
|
+
gem.files = `git ls-files`.split($/)
|
27
|
+
gem.files = glob[gemspec['files']] if gemspec['files']
|
28
|
+
gem.files += Array(gemspec['generated_files'])
|
29
|
+
|
30
|
+
gem.executables = gemspec.fetch('executables') do
|
31
|
+
glob['bin/*'].map { |path| File.basename(path) }
|
6
32
|
end
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
33
|
+
gem.default_executable = gem.executables.first if Gem::VERSION < '1.7.'
|
34
|
+
|
35
|
+
gem.extensions = glob[gemspec['extensions'] || 'ext/**/extconf.rb']
|
36
|
+
gem.test_files = glob[gemspec['test_files'] || 'spec/{**/}*_spec.rb']
|
37
|
+
gem.extra_rdoc_files = glob[gemspec['extra_doc_files'] || '*.{txt,md}']
|
38
|
+
|
39
|
+
gem.require_paths = Array(gemspec.fetch('require_paths') {
|
40
|
+
%w[ext lib].select { |dir| File.directory?(dir) }
|
41
|
+
})
|
42
|
+
|
43
|
+
gem.requirements = gemspec['requirements']
|
44
|
+
gem.required_ruby_version = gemspec['required_ruby_version']
|
45
|
+
gem.required_rubygems_version = gemspec['required_rubygems_version']
|
46
|
+
gem.post_install_message = gemspec['post_install_message']
|
47
|
+
|
48
|
+
split = lambda { |string| string.split(/,\s*/) }
|
49
|
+
|
50
|
+
if gemspec['dependencies']
|
51
|
+
gemspec['dependencies'].each do |name,versions|
|
52
|
+
gem.add_dependency(name,split[versions])
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
if gemspec['development_dependencies']
|
57
|
+
gemspec['development_dependencies'].each do |name,versions|
|
58
|
+
gem.add_development_dependency(name,split[versions])
|
59
|
+
end
|
14
60
|
end
|
15
61
|
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'ronin/web/cli/ruby_shell'
|
3
|
+
|
4
|
+
describe Ronin::Web::CLI::RubyShell do
|
5
|
+
describe "#initialize" do
|
6
|
+
it "must default #name to 'ronin-web'" do
|
7
|
+
expect(subject.name).to eq('ronin-web')
|
8
|
+
end
|
9
|
+
|
10
|
+
it "must default #context to Ronin::Web" do
|
11
|
+
expect(subject.context).to be(Ronin::Web)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|