ronin-web 0.3.0.rc1 → 1.0.0.beta2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (148) hide show
  1. checksums.yaml +7 -0
  2. data/.editorconfig +11 -0
  3. data/.github/workflows/ruby.yml +31 -0
  4. data/.gitignore +13 -0
  5. data/.mailmap +1 -0
  6. data/.ruby-version +1 -0
  7. data/COPYING.txt +3 -3
  8. data/ChangeLog.md +115 -70
  9. data/Gemfile +42 -37
  10. data/README.md +160 -145
  11. data/Rakefile +12 -3
  12. data/bin/ronin-web +9 -17
  13. data/data/new/nokogiri.rb.erb +12 -0
  14. data/data/new/server.rb.erb +22 -0
  15. data/data/new/spider.rb.erb +26 -0
  16. data/data/new/webapp/.gitignore +15 -0
  17. data/data/new/webapp/.ruby-version.erb +1 -0
  18. data/data/new/webapp/Dockerfile.erb +11 -0
  19. data/data/new/webapp/Gemfile +6 -0
  20. data/data/new/webapp/app.rb.erb +15 -0
  21. data/data/new/webapp/config.ru +4 -0
  22. data/data/new/webapp/docker-compose.yml.erb +9 -0
  23. data/gemspec.yml +38 -16
  24. data/lib/ronin/web/cli/command.rb +36 -0
  25. data/lib/ronin/web/cli/commands/diff.rb +106 -0
  26. data/lib/ronin/web/cli/commands/html.rb +174 -0
  27. data/lib/ronin/web/cli/commands/irb.rb +56 -0
  28. data/lib/ronin/web/cli/commands/new/nokogiri.rb +85 -0
  29. data/lib/ronin/web/cli/commands/new/server.rb +96 -0
  30. data/lib/ronin/web/cli/commands/new/spider.rb +315 -0
  31. data/lib/ronin/web/cli/commands/new/webapp.rb +123 -0
  32. data/lib/ronin/web/cli/commands/new.rb +64 -0
  33. data/lib/ronin/web/cli/commands/reverse_proxy.rb +215 -0
  34. data/lib/ronin/web/cli/commands/server.rb +155 -0
  35. data/lib/ronin/web/cli/commands/spider.rb +822 -0
  36. data/lib/ronin/web/cli/ruby_shell.rb +50 -0
  37. data/lib/ronin/web/cli.rb +49 -0
  38. data/lib/ronin/web/html.rb +85 -0
  39. data/lib/ronin/web/mechanize.rb +34 -36
  40. data/lib/ronin/web/root.rb +27 -0
  41. data/lib/ronin/web/version.rb +7 -10
  42. data/lib/ronin/web/xml.rb +85 -0
  43. data/lib/ronin/web.rb +372 -13
  44. data/man/ronin-web-diff.1 +41 -0
  45. data/man/ronin-web-diff.1.md +30 -0
  46. data/man/ronin-web-html.1 +89 -0
  47. data/man/ronin-web-html.1.md +66 -0
  48. data/man/ronin-web-irb.1 +31 -0
  49. data/man/ronin-web-irb.1.md +22 -0
  50. data/man/ronin-web-new-nokogiri.1 +41 -0
  51. data/man/ronin-web-new-nokogiri.1.md +30 -0
  52. data/man/ronin-web-new-server.1 +45 -0
  53. data/man/ronin-web-new-server.1.md +33 -0
  54. data/man/ronin-web-new-spider.1 +173 -0
  55. data/man/ronin-web-new-spider.1.md +129 -0
  56. data/man/ronin-web-new-webapp.1 +53 -0
  57. data/man/ronin-web-new-webapp.1.md +39 -0
  58. data/man/ronin-web-new.1 +59 -0
  59. data/man/ronin-web-new.1.md +44 -0
  60. data/man/ronin-web-reverse-proxy.1 +63 -0
  61. data/man/ronin-web-reverse-proxy.1.md +47 -0
  62. data/man/ronin-web-server.1 +59 -0
  63. data/man/ronin-web-server.1.md +43 -0
  64. data/man/ronin-web-spider.1 +225 -0
  65. data/man/ronin-web-spider.1.md +168 -0
  66. data/man/ronin-web.1 +41 -0
  67. data/man/ronin-web.1.md +30 -0
  68. data/ronin-web.gemspec +39 -109
  69. data/spec/cli/ruby_shell_spec.rb +14 -0
  70. data/spec/html_spec.rb +43 -0
  71. data/spec/mechanize_spec.rb +72 -0
  72. data/spec/spec_helper.rb +5 -3
  73. data/spec/web_spec.rb +97 -0
  74. data/spec/xml_spec.rb +42 -0
  75. metadata +236 -224
  76. data/.gemtest +0 -0
  77. data/data/ronin/web/user_agents.yml +0 -247
  78. data/lib/ronin/network/mixins/web.rb +0 -258
  79. data/lib/ronin/web/config.rb +0 -34
  80. data/lib/ronin/web/extensions/nokogiri/xml/attr.rb +0 -47
  81. data/lib/ronin/web/extensions/nokogiri/xml/document.rb +0 -48
  82. data/lib/ronin/web/extensions/nokogiri/xml/element.rb +0 -57
  83. data/lib/ronin/web/extensions/nokogiri/xml/node.rb +0 -86
  84. data/lib/ronin/web/extensions/nokogiri/xml/text.rb +0 -47
  85. data/lib/ronin/web/extensions/nokogiri/xml.rb +0 -27
  86. data/lib/ronin/web/extensions/nokogiri.rb +0 -23
  87. data/lib/ronin/web/extensions.rb +0 -23
  88. data/lib/ronin/web/middleware/base.rb +0 -144
  89. data/lib/ronin/web/middleware/directories.rb +0 -179
  90. data/lib/ronin/web/middleware/files.rb +0 -144
  91. data/lib/ronin/web/middleware/filters/campaign_filter.rb +0 -77
  92. data/lib/ronin/web/middleware/filters/ip_filter.rb +0 -73
  93. data/lib/ronin/web/middleware/filters/path_filter.rb +0 -73
  94. data/lib/ronin/web/middleware/filters/referer_filter.rb +0 -71
  95. data/lib/ronin/web/middleware/filters/user_agent_filter.rb +0 -71
  96. data/lib/ronin/web/middleware/filters/vhost_filter.rb +0 -71
  97. data/lib/ronin/web/middleware/filters.rb +0 -28
  98. data/lib/ronin/web/middleware/helpers.rb +0 -145
  99. data/lib/ronin/web/middleware/proxy.rb +0 -265
  100. data/lib/ronin/web/middleware/proxy_request.rb +0 -262
  101. data/lib/ronin/web/middleware/request.rb +0 -79
  102. data/lib/ronin/web/middleware/response.rb +0 -33
  103. data/lib/ronin/web/middleware/router.rb +0 -167
  104. data/lib/ronin/web/middleware/rule.rb +0 -103
  105. data/lib/ronin/web/middleware.rb +0 -27
  106. data/lib/ronin/web/proxy/app.rb +0 -32
  107. data/lib/ronin/web/proxy/base.rb +0 -46
  108. data/lib/ronin/web/proxy/web.rb +0 -46
  109. data/lib/ronin/web/proxy.rb +0 -25
  110. data/lib/ronin/web/server/app.rb +0 -32
  111. data/lib/ronin/web/server/base.rb +0 -461
  112. data/lib/ronin/web/server/web.rb +0 -66
  113. data/lib/ronin/web/server.rb +0 -25
  114. data/lib/ronin/web/spider.rb +0 -120
  115. data/lib/ronin/web/user_agents.rb +0 -196
  116. data/lib/ronin/web/web.rb +0 -560
  117. data/spec/helpers/output.rb +0 -3
  118. data/spec/web/extensions/nokogiri_spec.rb +0 -38
  119. data/spec/web/helpers/rack_app.rb +0 -24
  120. data/spec/web/helpers/root/test1/index.html +0 -1
  121. data/spec/web/helpers/root/test1/test1.txt +0 -1
  122. data/spec/web/helpers/root/test1.txt +0 -1
  123. data/spec/web/helpers/root/test2/test2.txt +0 -1
  124. data/spec/web/helpers/root/test2.txt +0 -1
  125. data/spec/web/helpers/root/test3/test3.txt +0 -1
  126. data/spec/web/helpers/root/test3.txt +0 -1
  127. data/spec/web/helpers/root.rb +0 -15
  128. data/spec/web/mechanize_spec.rb +0 -62
  129. data/spec/web/middleware/directories_spec.rb +0 -86
  130. data/spec/web/middleware/files_spec.rb +0 -57
  131. data/spec/web/middleware/filters/campaign_filter_spec.rb +0 -30
  132. data/spec/web/middleware/filters/ip_filter_spec.rb +0 -25
  133. data/spec/web/middleware/filters/path_filter_spec.rb +0 -29
  134. data/spec/web/middleware/filters/referer_filter_spec.rb +0 -25
  135. data/spec/web/middleware/filters/user_agent_filter_spec.rb +0 -25
  136. data/spec/web/middleware/filters/vhost_filter_spec.rb +0 -23
  137. data/spec/web/middleware/proxy_spec.rb +0 -67
  138. data/spec/web/middleware/response_spec.rb +0 -20
  139. data/spec/web/middleware/router_spec.rb +0 -65
  140. data/spec/web/middleware/rule_spec.rb +0 -37
  141. data/spec/web/proxy/base_spec.rb +0 -8
  142. data/spec/web/server/base_spec.rb +0 -77
  143. data/spec/web/server/classes/public1/static1.txt +0 -1
  144. data/spec/web/server/classes/public2/static2.txt +0 -1
  145. data/spec/web/server/classes/sub_app.rb +0 -13
  146. data/spec/web/server/classes/test_app.rb +0 -20
  147. data/spec/web/user_agents_spec.rb +0 -56
  148. data/spec/web/web_spec.rb +0 -101
@@ -0,0 +1,215 @@
1
+ #
2
+ # ronin-web - A collection of useful web helper methods and commands.
3
+ #
4
+ # Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
5
+ #
6
+ # ronin-web is free software: you can redistribute it and/or modify
7
+ # it under the terms of the GNU General Public License as published by
8
+ # the Free Software Foundation, either version 3 of the License, or
9
+ # (at your option) any later version.
10
+ #
11
+ # ronin-web is distributed in the hope that it will be useful,
12
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
13
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
+ # GNU General Public License for more details.
15
+ #
16
+ # You should have received a copy of the GNU General Public License
17
+ # along with ronin-web. If not, see <https://www.gnu.org/licenses/>.
18
+ #
19
+
20
+ require 'ronin/web/cli/command'
21
+ require 'ronin/core/cli/logging'
22
+ require 'ronin/web/server/reverse_proxy'
23
+
24
+ module Ronin
25
+ module Web
26
+ class CLI
27
+ module Commands
28
+ #
29
+ # Starts a HTTP proxy server.
30
+ #
31
+ # ## Usage
32
+ #
33
+ # ronin-web reverse-proxy [options] [--host HOST] [--port PORT]
34
+ #
35
+ # ## Options
36
+ #
37
+ # -H, --host HOST Host to listen on (Default: localhost)
38
+ # -p, --port PORT Port to listen on (Default: 8080)
39
+ # -b, --show-body Print the request and response bodies
40
+ # --rewrite-requests /REGEXP/:REPLACE
41
+ # Rewrite request bodies
42
+ # --rewrite-responses /REGEXP/:REPLACE
43
+ # Rewrite response bodies
44
+ # -h, --help Print help information
45
+ #
46
+ # @api private
47
+ #
48
+ class Proxy < Command
49
+
50
+ include Core::CLI::Logging
51
+
52
+ command_name 'reverse-proxy'
53
+
54
+ usage '[options] [--host HOST] [--port PORT]'
55
+
56
+ option :host, short: '-H',
57
+ value: {
58
+ type: String,
59
+ usage: 'HOST',
60
+ default: 'localhost'
61
+ },
62
+ desc: 'Host to listen on'
63
+
64
+ option :port, short: '-p',
65
+ value: {
66
+ type: Integer,
67
+ usage: 'PORT',
68
+ default: 8080
69
+ },
70
+ desc: 'Port to listen on'
71
+
72
+ option :show_body, short: '-b',
73
+ desc: 'Print the request and response bodies'
74
+
75
+ option :rewrite_requests, value: {
76
+ type: String,
77
+ usage: '/REGEXP/:REPLACE'
78
+ },
79
+ desc: 'Rewrite request bodies' do |str|
80
+ @rewrite_requests << parse_rewrite_rule(str)
81
+ end
82
+
83
+ option :rewrite_responses, value: {
84
+ type: String,
85
+ usage: '/REGEXP/:REPLACE'
86
+ },
87
+ desc: 'Rewrite response bodies' do |str|
88
+ @rewrite_responses << parse_rewrite_rule(str)
89
+ end
90
+
91
+ description 'Starts a HTTP proxy server'
92
+
93
+ man_page 'ronin-web-reverse-proxy.1'
94
+
95
+ #
96
+ # Initializes the `reverse-proxy` command.
97
+ #
98
+ # @param [Hash{Symbol => Object}] kwargs
99
+ # Additional keyword arguments.
100
+ #
101
+ def initialize(**kwargs)
102
+ super(**kwargs)
103
+
104
+ @rewrite_requests = []
105
+ @rewrite_responses = []
106
+ end
107
+
108
+ #
109
+ # Runs the `ronin-web reverse-proxy` command.
110
+ #
111
+ def run
112
+ proxy = Ronin::Web::Server::ReverseProxy.new do |proxy|
113
+ proxy.on_request do |request|
114
+ puts "[#{request.ip} -> #{request.host_with_port}] #{request.request_method} #{request.url}"
115
+
116
+ request.headers.each do |name,value|
117
+ puts "> #{name}: #{value}"
118
+ end
119
+ puts
120
+
121
+ unless @rewrite_requests.empty?
122
+ request.body = rewrite_body(request.body,@rewrite_requests)
123
+ end
124
+
125
+ print_body(request.body) if options[:show_body]
126
+ end
127
+
128
+ proxy.on_response do |response|
129
+ puts "< HTTP/1.1 #{response.status}"
130
+
131
+ response.headers.each do |name,value|
132
+ puts "< #{name}: #{value}"
133
+ end
134
+ puts
135
+
136
+ unless @rewrite_responses.empty?
137
+ response.body = rewrite_body(response.body,@rewrite_responses)
138
+ end
139
+
140
+ print_body(response.body) if options[:show_body]
141
+ end
142
+ end
143
+
144
+ log_info "Starting proxy server on #{options[:host]}:#{options[:port]} ..."
145
+ proxy.run!(host: options[:host], port: options[:port])
146
+ log_info "shutting down ..."
147
+ end
148
+
149
+ #
150
+ # Prints a request or response body.
151
+ #
152
+ # @param [IO, StringIO, Array<String>, String] body
153
+ # The request/response body to print. May be a IO/StringIO object,
154
+ # an Array of Strings, or a String.
155
+ #
156
+ def print_body(body)
157
+ case body
158
+ when StringIO, IO
159
+ body.each_line do |line|
160
+ puts line
161
+ end
162
+
163
+ body.rewind
164
+ else
165
+ puts body
166
+ end
167
+ end
168
+
169
+ #
170
+ # Parses a rewrite rule.
171
+ #
172
+ # @param [String] value
173
+ #
174
+ # @return [(Regexp, String), (String, String)]
175
+ #
176
+ def parse_rewrite_rule(value)
177
+ if (index = value.rindex('/:'))
178
+ regexp = Regexp.new(value[1...index])
179
+ replace = value[index+2..]
180
+
181
+ return [regexp, replace]
182
+ elsif (index = value.rindex(':'))
183
+ string = value[0...index]
184
+ replace = value[(index+1)..]
185
+
186
+ return [string, replace]
187
+ end
188
+ end
189
+
190
+ #
191
+ # Rewrites a request or response body.
192
+ #
193
+ # @param [IO, StringIO, Array<String>, String] body
194
+ #
195
+ # @return [String]
196
+ #
197
+ def rewrite_body(body,rules)
198
+ body = case body
199
+ when StringIO, IO then body.read
200
+ when Array then body.join
201
+ else body.to_s
202
+ end
203
+
204
+ rules.each do |(pattern,replace)|
205
+ body.gsub!(pattern,replace)
206
+ end
207
+
208
+ return body
209
+ end
210
+
211
+ end
212
+ end
213
+ end
214
+ end
215
+ end
@@ -0,0 +1,155 @@
1
+ #
2
+ # ronin-web - A collection of useful web helper methods and commands.
3
+ #
4
+ # Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
5
+ #
6
+ # ronin-web is free software: you can redistribute it and/or modify
7
+ # it under the terms of the GNU General Public License as published by
8
+ # the Free Software Foundation, either version 3 of the License, or
9
+ # (at your option) any later version.
10
+ #
11
+ # ronin-web is distributed in the hope that it will be useful,
12
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
13
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
+ # GNU General Public License for more details.
15
+ #
16
+ # You should have received a copy of the GNU General Public License
17
+ # along with ronin-web. If not, see <https://www.gnu.org/licenses/>.
18
+ #
19
+
20
+ require 'ronin/web/cli/command'
21
+ require 'ronin/core/cli/logging'
22
+ require 'ronin/web/server'
23
+
24
+ module Ronin
25
+ module Web
26
+ class CLI
27
+ module Commands
28
+ #
29
+ # Starts a web server.
30
+ #
31
+ # ## Usage
32
+ #
33
+ # ronin-web server [options]
34
+ #
35
+ # ## Options
36
+ #
37
+ # -H, --host HOST Host name or IP to bind to (Default: localhost)
38
+ # -p, --port PORT Port number to listen on (Default: 8000)
39
+ # -A, --basic-auth USER:PASSWORD Sets up Basic-Authentication
40
+ # -d, --dir /PATH:DIR Mounts a directory to the given PATH
41
+ # -f, --file /PATH:FILE Mounts a file to the given PATH
42
+ # -r, --root DIR Root directory to serve
43
+ # -R, --redirect /PATH:URL Registers a 302 Found redirect at the given PATH
44
+ # -h, --help Print help information
45
+ #
46
+ class Server < Command
47
+
48
+ include Core::CLI::Logging
49
+
50
+ class App < Ronin::Web::Server::Base
51
+ end
52
+
53
+ option :host, short: '-H',
54
+ value: {
55
+ type: String,
56
+ usage: 'HOST',
57
+ default: 'localhost'
58
+ },
59
+ desc: 'Host name or IP to bind to' do |host|
60
+ App.host = host
61
+ end
62
+
63
+ option :port, short: '-p',
64
+ value: {
65
+ type: Integer,
66
+ usage: 'PORT',
67
+ default: App.port
68
+ },
69
+ desc: 'Port number to listen on' do |port|
70
+ App.port = port
71
+ end
72
+
73
+ option :basic_auth, short: '-A',
74
+ value: {
75
+ type: String,
76
+ usage: 'USER:PASSWORD'
77
+ },
78
+ desc: 'Sets up Basic-Authentication' do |str|
79
+ auth_user, auth_password = str.split(':',2)
80
+
81
+ App.basic_auth(auth_user,auth_password)
82
+ end
83
+
84
+ option :dir, short: '-d',
85
+ value: {
86
+ type: String,
87
+ usage: '/PATH:DIR'
88
+ },
89
+ desc: 'Mounts a directory to the given PATH' do |str|
90
+ url_path, dir = str.split(':',2)
91
+
92
+ App.directory(url_path,dir)
93
+ end
94
+
95
+ option :file, short: '-f',
96
+ value: {
97
+ type: String,
98
+ usage: '/PATH:FILE'
99
+ },
100
+ desc: 'Mounts a file to the given PATH' do |str|
101
+ url_path, file = str.split(':',2)
102
+
103
+ App.file(url_path,file)
104
+ end
105
+
106
+ option :root, short: '-r',
107
+ value: {
108
+ type: String,
109
+ usage: 'DIR'
110
+ },
111
+ desc: 'Root directory to serve'
112
+
113
+ option :redirect, short: '-R',
114
+ value: {
115
+ type: String,
116
+ usage: '/PATH:URL'
117
+ },
118
+ desc: 'Registers a 302 Found redirect at the given PATH' do |str|
119
+ route, url = str.split(':',2)
120
+
121
+ App.redirect(route,url)
122
+ end
123
+
124
+ description 'Starts a web server'
125
+
126
+ man_page 'ronin-web-server.1'
127
+
128
+ #
129
+ # Runs the `ronin-web server` command.
130
+ #
131
+ def run
132
+ if options[:root]
133
+ App.public_dir = options[:root]
134
+ else
135
+ App.any('*') do
136
+ puts "#{request.request_method} #{request.path}"
137
+
138
+ request.headers.each do |name,value|
139
+ puts "#{name}: #{value}"
140
+ end
141
+
142
+ puts request.body.read
143
+ end
144
+ end
145
+
146
+ log_info "Starting web server listening on #{App.host}:#{App.port} ..."
147
+ App.run!
148
+ log_info "Shutting down ..."
149
+ end
150
+
151
+ end
152
+ end
153
+ end
154
+ end
155
+ end