ronin-web 0.3.0.rc1 → 1.0.0.beta1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (148) hide show
  1. checksums.yaml +7 -0
  2. data/.editorconfig +11 -0
  3. data/.github/workflows/ruby.yml +31 -0
  4. data/.gitignore +13 -0
  5. data/.mailmap +1 -0
  6. data/.ruby-version +1 -0
  7. data/COPYING.txt +3 -3
  8. data/ChangeLog.md +115 -70
  9. data/Gemfile +42 -37
  10. data/README.md +159 -145
  11. data/Rakefile +12 -3
  12. data/bin/ronin-web +9 -17
  13. data/data/new/nokogiri.rb.erb +12 -0
  14. data/data/new/server.rb.erb +22 -0
  15. data/data/new/spider.rb.erb +26 -0
  16. data/data/new/webapp/.gitignore +15 -0
  17. data/data/new/webapp/.ruby-version.erb +1 -0
  18. data/data/new/webapp/Dockerfile.erb +11 -0
  19. data/data/new/webapp/Gemfile +6 -0
  20. data/data/new/webapp/app.rb.erb +15 -0
  21. data/data/new/webapp/config.ru +4 -0
  22. data/data/new/webapp/docker-compose.yml.erb +9 -0
  23. data/gemspec.yml +32 -14
  24. data/lib/ronin/web/cli/command.rb +36 -0
  25. data/lib/ronin/web/cli/commands/diff.rb +106 -0
  26. data/lib/ronin/web/cli/commands/html.rb +174 -0
  27. data/lib/ronin/web/cli/commands/irb.rb +56 -0
  28. data/lib/ronin/web/cli/commands/new/nokogiri.rb +85 -0
  29. data/lib/ronin/web/cli/commands/new/server.rb +96 -0
  30. data/lib/ronin/web/cli/commands/new/spider.rb +315 -0
  31. data/lib/ronin/web/cli/commands/new/webapp.rb +123 -0
  32. data/lib/ronin/web/cli/commands/new.rb +64 -0
  33. data/lib/ronin/web/cli/commands/reverse_proxy.rb +215 -0
  34. data/lib/ronin/web/cli/commands/server.rb +155 -0
  35. data/lib/ronin/web/cli/commands/spider.rb +822 -0
  36. data/lib/ronin/web/cli/ruby_shell.rb +50 -0
  37. data/lib/ronin/web/cli.rb +44 -0
  38. data/lib/ronin/web/html.rb +85 -0
  39. data/lib/ronin/web/mechanize.rb +34 -36
  40. data/lib/ronin/web/root.rb +27 -0
  41. data/lib/ronin/web/version.rb +7 -10
  42. data/lib/ronin/web/xml.rb +85 -0
  43. data/lib/ronin/web.rb +372 -13
  44. data/man/ronin-web-diff.1 +41 -0
  45. data/man/ronin-web-diff.1.md +30 -0
  46. data/man/ronin-web-html.1 +89 -0
  47. data/man/ronin-web-html.1.md +66 -0
  48. data/man/ronin-web-irb.1 +31 -0
  49. data/man/ronin-web-irb.1.md +22 -0
  50. data/man/ronin-web-new-nokogiri.1 +41 -0
  51. data/man/ronin-web-new-nokogiri.1.md +30 -0
  52. data/man/ronin-web-new-server.1 +45 -0
  53. data/man/ronin-web-new-server.1.md +33 -0
  54. data/man/ronin-web-new-spider.1 +173 -0
  55. data/man/ronin-web-new-spider.1.md +129 -0
  56. data/man/ronin-web-new-webapp.1 +53 -0
  57. data/man/ronin-web-new-webapp.1.md +39 -0
  58. data/man/ronin-web-new.1 +59 -0
  59. data/man/ronin-web-new.1.md +44 -0
  60. data/man/ronin-web-reverse-proxy.1 +63 -0
  61. data/man/ronin-web-reverse-proxy.1.md +47 -0
  62. data/man/ronin-web-server.1 +59 -0
  63. data/man/ronin-web-server.1.md +43 -0
  64. data/man/ronin-web-spider.1 +225 -0
  65. data/man/ronin-web-spider.1.md +168 -0
  66. data/man/ronin-web.1 +41 -0
  67. data/man/ronin-web.1.md +30 -0
  68. data/ronin-web.gemspec +39 -109
  69. data/spec/cli/ruby_shell_spec.rb +14 -0
  70. data/spec/html_spec.rb +43 -0
  71. data/spec/mechanize_spec.rb +72 -0
  72. data/spec/spec_helper.rb +5 -3
  73. data/spec/web_spec.rb +97 -0
  74. data/spec/xml_spec.rb +42 -0
  75. metadata +236 -224
  76. data/.gemtest +0 -0
  77. data/data/ronin/web/user_agents.yml +0 -247
  78. data/lib/ronin/network/mixins/web.rb +0 -258
  79. data/lib/ronin/web/config.rb +0 -34
  80. data/lib/ronin/web/extensions/nokogiri/xml/attr.rb +0 -47
  81. data/lib/ronin/web/extensions/nokogiri/xml/document.rb +0 -48
  82. data/lib/ronin/web/extensions/nokogiri/xml/element.rb +0 -57
  83. data/lib/ronin/web/extensions/nokogiri/xml/node.rb +0 -86
  84. data/lib/ronin/web/extensions/nokogiri/xml/text.rb +0 -47
  85. data/lib/ronin/web/extensions/nokogiri/xml.rb +0 -27
  86. data/lib/ronin/web/extensions/nokogiri.rb +0 -23
  87. data/lib/ronin/web/extensions.rb +0 -23
  88. data/lib/ronin/web/middleware/base.rb +0 -144
  89. data/lib/ronin/web/middleware/directories.rb +0 -179
  90. data/lib/ronin/web/middleware/files.rb +0 -144
  91. data/lib/ronin/web/middleware/filters/campaign_filter.rb +0 -77
  92. data/lib/ronin/web/middleware/filters/ip_filter.rb +0 -73
  93. data/lib/ronin/web/middleware/filters/path_filter.rb +0 -73
  94. data/lib/ronin/web/middleware/filters/referer_filter.rb +0 -71
  95. data/lib/ronin/web/middleware/filters/user_agent_filter.rb +0 -71
  96. data/lib/ronin/web/middleware/filters/vhost_filter.rb +0 -71
  97. data/lib/ronin/web/middleware/filters.rb +0 -28
  98. data/lib/ronin/web/middleware/helpers.rb +0 -145
  99. data/lib/ronin/web/middleware/proxy.rb +0 -265
  100. data/lib/ronin/web/middleware/proxy_request.rb +0 -262
  101. data/lib/ronin/web/middleware/request.rb +0 -79
  102. data/lib/ronin/web/middleware/response.rb +0 -33
  103. data/lib/ronin/web/middleware/router.rb +0 -167
  104. data/lib/ronin/web/middleware/rule.rb +0 -103
  105. data/lib/ronin/web/middleware.rb +0 -27
  106. data/lib/ronin/web/proxy/app.rb +0 -32
  107. data/lib/ronin/web/proxy/base.rb +0 -46
  108. data/lib/ronin/web/proxy/web.rb +0 -46
  109. data/lib/ronin/web/proxy.rb +0 -25
  110. data/lib/ronin/web/server/app.rb +0 -32
  111. data/lib/ronin/web/server/base.rb +0 -461
  112. data/lib/ronin/web/server/web.rb +0 -66
  113. data/lib/ronin/web/server.rb +0 -25
  114. data/lib/ronin/web/spider.rb +0 -120
  115. data/lib/ronin/web/user_agents.rb +0 -196
  116. data/lib/ronin/web/web.rb +0 -560
  117. data/spec/helpers/output.rb +0 -3
  118. data/spec/web/extensions/nokogiri_spec.rb +0 -38
  119. data/spec/web/helpers/rack_app.rb +0 -24
  120. data/spec/web/helpers/root/test1/index.html +0 -1
  121. data/spec/web/helpers/root/test1/test1.txt +0 -1
  122. data/spec/web/helpers/root/test1.txt +0 -1
  123. data/spec/web/helpers/root/test2/test2.txt +0 -1
  124. data/spec/web/helpers/root/test2.txt +0 -1
  125. data/spec/web/helpers/root/test3/test3.txt +0 -1
  126. data/spec/web/helpers/root/test3.txt +0 -1
  127. data/spec/web/helpers/root.rb +0 -15
  128. data/spec/web/mechanize_spec.rb +0 -62
  129. data/spec/web/middleware/directories_spec.rb +0 -86
  130. data/spec/web/middleware/files_spec.rb +0 -57
  131. data/spec/web/middleware/filters/campaign_filter_spec.rb +0 -30
  132. data/spec/web/middleware/filters/ip_filter_spec.rb +0 -25
  133. data/spec/web/middleware/filters/path_filter_spec.rb +0 -29
  134. data/spec/web/middleware/filters/referer_filter_spec.rb +0 -25
  135. data/spec/web/middleware/filters/user_agent_filter_spec.rb +0 -25
  136. data/spec/web/middleware/filters/vhost_filter_spec.rb +0 -23
  137. data/spec/web/middleware/proxy_spec.rb +0 -67
  138. data/spec/web/middleware/response_spec.rb +0 -20
  139. data/spec/web/middleware/router_spec.rb +0 -65
  140. data/spec/web/middleware/rule_spec.rb +0 -37
  141. data/spec/web/proxy/base_spec.rb +0 -8
  142. data/spec/web/server/base_spec.rb +0 -77
  143. data/spec/web/server/classes/public1/static1.txt +0 -1
  144. data/spec/web/server/classes/public2/static2.txt +0 -1
  145. data/spec/web/server/classes/sub_app.rb +0 -13
  146. data/spec/web/server/classes/test_app.rb +0 -20
  147. data/spec/web/user_agents_spec.rb +0 -56
  148. data/spec/web/web_spec.rb +0 -101
@@ -0,0 +1,215 @@
1
+ #
2
+ # ronin-web - A collection of useful web helper methods and commands.
3
+ #
4
+ # Copyright (c) 2006-2022 Hal Brodigan (postmodern.mod3 at gmail.com)
5
+ #
6
+ # ronin-web is free software: you can redistribute it and/or modify
7
+ # it under the terms of the GNU General Public License as published by
8
+ # the Free Software Foundation, either version 3 of the License, or
9
+ # (at your option) any later version.
10
+ #
11
+ # ronin-web is distributed in the hope that it will be useful,
12
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
13
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
+ # GNU General Public License for more details.
15
+ #
16
+ # You should have received a copy of the GNU General Public License
17
+ # along with ronin-web. If not, see <https://www.gnu.org/licenses/>.
18
+ #
19
+
20
+ require 'ronin/web/cli/command'
21
+ require 'ronin/core/cli/logging'
22
+ require 'ronin/web/server/reverse_proxy'
23
+
24
+ module Ronin
25
+ module Web
26
+ class CLI
27
+ module Commands
28
+ #
29
+ # Starts a HTTP proxy server.
30
+ #
31
+ # ## Usage
32
+ #
33
+ # ronin-web reverse-proxy [options] [--host HOST] [--port PORT]
34
+ #
35
+ # ## Options
36
+ #
37
+ # -H, --host HOST Host to listen on (Default: localhost)
38
+ # -p, --port PORT Port to listen on (Default: 8080)
39
+ # -b, --show-body Print the request and response bodies
40
+ # --rewrite-requests /REGEXP/:REPLACE
41
+ # Rewrite request bodies
42
+ # --rewrite-responses /REGEXP/:REPLACE
43
+ # Rewrite response bodies
44
+ # -h, --help Print help information
45
+ #
46
+ # @api private
47
+ #
48
+ class Proxy < Command
49
+
50
+ include Core::CLI::Logging
51
+
52
+ command_name 'reverse-proxy'
53
+
54
+ usage '[options] [--host HOST] [--port PORT]'
55
+
56
+ option :host, short: '-H',
57
+ value: {
58
+ type: String,
59
+ usage: 'HOST',
60
+ default: 'localhost'
61
+ },
62
+ desc: 'Host to listen on'
63
+
64
+ option :port, short: '-p',
65
+ value: {
66
+ type: Integer,
67
+ usage: 'PORT',
68
+ default: 8080
69
+ },
70
+ desc: 'Port to listen on'
71
+
72
+ option :show_body, short: '-b',
73
+ desc: 'Print the request and response bodies'
74
+
75
+ option :rewrite_requests, value: {
76
+ type: String,
77
+ usage: '/REGEXP/:REPLACE'
78
+ },
79
+ desc: 'Rewrite request bodies' do |str|
80
+ @rewrite_requests << parse_rewrite_rule(str)
81
+ end
82
+
83
+ option :rewrite_responses, value: {
84
+ type: String,
85
+ usage: '/REGEXP/:REPLACE'
86
+ },
87
+ desc: 'Rewrite response bodies' do |str|
88
+ @rewrite_responses << parse_rewrite_rule(str)
89
+ end
90
+
91
+ description 'Starts a HTTP proxy server'
92
+
93
+ man_page 'ronin-web-reverse-proxy.1'
94
+
95
+ #
96
+ # Initializes the `reverse-proxy` command.
97
+ #
98
+ # @param [Hash{Symbol => Object}] kwargs
99
+ # Additional keyword arguments.
100
+ #
101
+ def initialize(**kwargs)
102
+ super(**kwargs)
103
+
104
+ @rewrite_requests = []
105
+ @rewrite_responses = []
106
+ end
107
+
108
+ #
109
+ # Runs the `ronin-web reverse-proxy` command.
110
+ #
111
+ def run
112
+ proxy = Ronin::Web::Server::ReverseProxy.new do |proxy|
113
+ proxy.on_request do |request|
114
+ puts "[#{request.ip} -> #{request.host_with_port}] #{request.request_method} #{request.url}"
115
+
116
+ request.headers.each do |name,value|
117
+ puts "> #{name}: #{value}"
118
+ end
119
+ puts
120
+
121
+ unless @rewrite_requests.empty?
122
+ request.body = rewrite_body(request.body,@rewrite_requests)
123
+ end
124
+
125
+ print_body(request.body) if options[:show_body]
126
+ end
127
+
128
+ proxy.on_response do |response|
129
+ puts "< HTTP/1.1 #{response.status}"
130
+
131
+ response.headers.each do |name,value|
132
+ puts "< #{name}: #{value}"
133
+ end
134
+ puts
135
+
136
+ unless @rewrite_responses.empty?
137
+ response.body = rewrite_body(response.body,@rewrite_responses)
138
+ end
139
+
140
+ print_body(response.body) if options[:show_body]
141
+ end
142
+ end
143
+
144
+ log_info "Starting proxy server on #{options[:host]}:#{options[:port]} ..."
145
+ proxy.run!(host: options[:host], port: options[:port])
146
+ log_info "shutting down ..."
147
+ end
148
+
149
+ #
150
+ # Prints a request or response body.
151
+ #
152
+ # @param [IO, StringIO, Array<String>, String] body
153
+ # The request/response body to print. May be a IO/StringIO object,
154
+ # an Array of Strings, or a String.
155
+ #
156
+ def print_body(body)
157
+ case body
158
+ when StringIO, IO
159
+ body.each_line do |line|
160
+ puts line
161
+ end
162
+
163
+ body.rewind
164
+ else
165
+ puts body
166
+ end
167
+ end
168
+
169
+ #
170
+ # Parses a rewrite rule.
171
+ #
172
+ # @param [String] value
173
+ #
174
+ # @return [(Regexp, String), (String, String)]
175
+ #
176
+ def parse_rewrite_rule(value)
177
+ if (index = value.rindex('/:'))
178
+ regexp = Regexp.new(value[1...index])
179
+ replace = value[index+2..]
180
+
181
+ return [regexp, replace]
182
+ elsif (index = value.rindex(':'))
183
+ string = value[0...index]
184
+ replace = value[(index+1)..]
185
+
186
+ return [string, replace]
187
+ end
188
+ end
189
+
190
+ #
191
+ # Rewrites a request or response body.
192
+ #
193
+ # @param [IO, StringIO, Array<String>, String] body
194
+ #
195
+ # @return [String]
196
+ #
197
+ def rewrite_body(body,rules)
198
+ body = case body
199
+ when StringIO, IO then body.read
200
+ when Array then body.join
201
+ else body.to_s
202
+ end
203
+
204
+ rules.each do |(pattern,replace)|
205
+ body.gsub!(pattern,replace)
206
+ end
207
+
208
+ return body
209
+ end
210
+
211
+ end
212
+ end
213
+ end
214
+ end
215
+ end
@@ -0,0 +1,155 @@
1
+ #
2
+ # ronin-web - A collection of useful web helper methods and commands.
3
+ #
4
+ # Copyright (c) 2006-2022 Hal Brodigan (postmodern.mod3 at gmail.com)
5
+ #
6
+ # ronin-web is free software: you can redistribute it and/or modify
7
+ # it under the terms of the GNU General Public License as published by
8
+ # the Free Software Foundation, either version 3 of the License, or
9
+ # (at your option) any later version.
10
+ #
11
+ # ronin-web is distributed in the hope that it will be useful,
12
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
13
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
+ # GNU General Public License for more details.
15
+ #
16
+ # You should have received a copy of the GNU General Public License
17
+ # along with ronin-web. If not, see <https://www.gnu.org/licenses/>.
18
+ #
19
+
20
+ require 'ronin/web/cli/command'
21
+ require 'ronin/core/cli/logging'
22
+ require 'ronin/web/server'
23
+
24
+ module Ronin
25
+ module Web
26
+ class CLI
27
+ module Commands
28
+ #
29
+ # Starts a web server.
30
+ #
31
+ # ## Usage
32
+ #
33
+ # ronin-web server [options]
34
+ #
35
+ # ## Options
36
+ #
37
+ # -H, --host HOST Host name or IP to bind to (Default: localhost)
38
+ # -p, --port PORT Port number to listen on (Default: 8000)
39
+ # -A, --basic-auth USER:PASSWORD Sets up Basic-Authentication
40
+ # -d, --dir /PATH:DIR Mounts a directory to the given PATH
41
+ # -f, --file /PATH:FILE Mounts a file to the given PATH
42
+ # -r, --root DIR Root directory to serve
43
+ # -R, --redirect /PATH:URL Registers a 302 Found redirect at the given PATH
44
+ # -h, --help Print help information
45
+ #
46
+ class Server < Command
47
+
48
+ include Core::CLI::Logging
49
+
50
+ class App < Ronin::Web::Server::Base
51
+ end
52
+
53
+ option :host, short: '-H',
54
+ value: {
55
+ type: String,
56
+ usage: 'HOST',
57
+ default: 'localhost'
58
+ },
59
+ desc: 'Host name or IP to bind to' do |host|
60
+ App.host = host
61
+ end
62
+
63
+ option :port, short: '-p',
64
+ value: {
65
+ type: Integer,
66
+ usage: 'PORT',
67
+ default: App.port
68
+ },
69
+ desc: 'Port number to listen on' do |port|
70
+ App.port = port
71
+ end
72
+
73
+ option :basic_auth, short: '-A',
74
+ value: {
75
+ type: String,
76
+ usage: 'USER:PASSWORD'
77
+ },
78
+ desc: 'Sets up Basic-Authentication' do |str|
79
+ auth_user, auth_password = str.split(':',2)
80
+
81
+ App.basic_auth(auth_user,auth_password)
82
+ end
83
+
84
+ option :dir, short: '-d',
85
+ value: {
86
+ type: String,
87
+ usage: '/PATH:DIR'
88
+ },
89
+ desc: 'Mounts a directory to the given PATH' do |str|
90
+ url_path, dir = str.split(':',2)
91
+
92
+ App.directory(url_path,dir)
93
+ end
94
+
95
+ option :file, short: '-f',
96
+ value: {
97
+ type: String,
98
+ usage: '/PATH:FILE'
99
+ },
100
+ desc: 'Mounts a file to the given PATH' do |str|
101
+ url_path, file = str.split(':',2)
102
+
103
+ App.file(url_path,file)
104
+ end
105
+
106
+ option :root, short: '-r',
107
+ value: {
108
+ type: String,
109
+ usage: 'DIR'
110
+ },
111
+ desc: 'Root directory to serve'
112
+
113
+ option :redirect, short: '-R',
114
+ value: {
115
+ type: String,
116
+ usage: '/PATH:URL'
117
+ },
118
+ desc: 'Registers a 302 Found redirect at the given PATH' do |str|
119
+ route, url = str.split(':',2)
120
+
121
+ App.redirect(route,url)
122
+ end
123
+
124
+ description 'Starts a web server'
125
+
126
+ man_page 'ronin-web-server.1'
127
+
128
+ #
129
+ # Runs the `ronin-web server` command.
130
+ #
131
+ def run
132
+ if options[:root]
133
+ App.public_dir = options[:root]
134
+ else
135
+ App.any('*') do
136
+ puts "#{request.request_method} #{request.path}"
137
+
138
+ request.headers.each do |name,value|
139
+ puts "#{name}: #{value}"
140
+ end
141
+
142
+ puts request.body.read
143
+ end
144
+ end
145
+
146
+ log_info "Starting web server listening on #{App.host}:#{App.port} ..."
147
+ App.run!
148
+ log_info "Shutting down ..."
149
+ end
150
+
151
+ end
152
+ end
153
+ end
154
+ end
155
+ end