ronin-web 0.3.0.pre2 → 1.0.0.beta1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.editorconfig +11 -0
- data/.github/workflows/ruby.yml +31 -0
- data/.gitignore +13 -0
- data/.mailmap +1 -0
- data/.ruby-version +1 -0
- data/COPYING.txt +3 -3
- data/ChangeLog.md +115 -67
- data/Gemfile +42 -37
- data/README.md +159 -144
- data/Rakefile +16 -7
- data/bin/ronin-web +10 -18
- data/data/new/nokogiri.rb.erb +12 -0
- data/data/new/server.rb.erb +22 -0
- data/data/new/spider.rb.erb +26 -0
- data/data/new/webapp/.gitignore +15 -0
- data/data/new/webapp/.ruby-version.erb +1 -0
- data/data/new/webapp/Dockerfile.erb +11 -0
- data/data/new/webapp/Gemfile +6 -0
- data/data/new/webapp/app.rb.erb +15 -0
- data/data/new/webapp/config.ru +4 -0
- data/data/new/webapp/docker-compose.yml.erb +9 -0
- data/gemspec.yml +32 -17
- data/lib/ronin/web/cli/command.rb +36 -0
- data/lib/ronin/web/cli/commands/diff.rb +106 -0
- data/lib/ronin/web/cli/commands/html.rb +174 -0
- data/lib/ronin/web/cli/commands/irb.rb +56 -0
- data/lib/ronin/web/cli/commands/new/nokogiri.rb +85 -0
- data/lib/ronin/web/cli/commands/new/server.rb +96 -0
- data/lib/ronin/web/cli/commands/new/spider.rb +315 -0
- data/lib/ronin/web/cli/commands/new/webapp.rb +123 -0
- data/lib/ronin/web/cli/commands/new.rb +64 -0
- data/lib/ronin/web/cli/commands/reverse_proxy.rb +215 -0
- data/lib/ronin/web/cli/commands/server.rb +155 -0
- data/lib/ronin/web/cli/commands/spider.rb +822 -0
- data/lib/ronin/web/cli/ruby_shell.rb +50 -0
- data/lib/ronin/web/cli.rb +44 -0
- data/lib/ronin/web/html.rb +85 -0
- data/lib/ronin/web/mechanize.rb +79 -0
- data/lib/ronin/web/root.rb +27 -0
- data/lib/ronin/web/version.rb +7 -10
- data/lib/ronin/web/xml.rb +85 -0
- data/lib/ronin/web.rb +372 -13
- data/man/ronin-web-diff.1 +41 -0
- data/man/ronin-web-diff.1.md +30 -0
- data/man/ronin-web-html.1 +89 -0
- data/man/ronin-web-html.1.md +66 -0
- data/man/ronin-web-irb.1 +31 -0
- data/man/ronin-web-irb.1.md +22 -0
- data/man/ronin-web-new-nokogiri.1 +41 -0
- data/man/ronin-web-new-nokogiri.1.md +30 -0
- data/man/ronin-web-new-server.1 +45 -0
- data/man/ronin-web-new-server.1.md +33 -0
- data/man/ronin-web-new-spider.1 +173 -0
- data/man/ronin-web-new-spider.1.md +129 -0
- data/man/ronin-web-new-webapp.1 +53 -0
- data/man/ronin-web-new-webapp.1.md +39 -0
- data/man/ronin-web-new.1 +59 -0
- data/man/ronin-web-new.1.md +44 -0
- data/man/ronin-web-reverse-proxy.1 +63 -0
- data/man/ronin-web-reverse-proxy.1.md +47 -0
- data/man/ronin-web-server.1 +59 -0
- data/man/ronin-web-server.1.md +43 -0
- data/man/ronin-web-spider.1 +225 -0
- data/man/ronin-web-spider.1.md +168 -0
- data/man/ronin-web.1 +41 -0
- data/man/ronin-web.1.md +30 -0
- data/ronin-web.gemspec +57 -11
- data/spec/cli/ruby_shell_spec.rb +14 -0
- data/spec/html_spec.rb +43 -0
- data/spec/mechanize_spec.rb +72 -0
- data/spec/spec_helper.rb +5 -3
- data/spec/web_spec.rb +97 -0
- data/spec/xml_spec.rb +42 -0
- metadata +237 -217
- data/.gemtest +0 -0
- data/lib/ronin/network/mixins/web.rb +0 -256
- data/lib/ronin/web/extensions/nokogiri/xml/attr.rb +0 -47
- data/lib/ronin/web/extensions/nokogiri/xml/document.rb +0 -48
- data/lib/ronin/web/extensions/nokogiri/xml/element.rb +0 -57
- data/lib/ronin/web/extensions/nokogiri/xml/node.rb +0 -86
- data/lib/ronin/web/extensions/nokogiri/xml/text.rb +0 -47
- data/lib/ronin/web/extensions/nokogiri/xml.rb +0 -27
- data/lib/ronin/web/extensions/nokogiri.rb +0 -23
- data/lib/ronin/web/extensions.rb +0 -23
- data/lib/ronin/web/middleware/base.rb +0 -144
- data/lib/ronin/web/middleware/directories.rb +0 -179
- data/lib/ronin/web/middleware/files.rb +0 -144
- data/lib/ronin/web/middleware/filters/campaign_filter.rb +0 -77
- data/lib/ronin/web/middleware/filters/ip_filter.rb +0 -73
- data/lib/ronin/web/middleware/filters/path_filter.rb +0 -73
- data/lib/ronin/web/middleware/filters/referer_filter.rb +0 -71
- data/lib/ronin/web/middleware/filters/user_agent_filter.rb +0 -71
- data/lib/ronin/web/middleware/filters/vhost_filter.rb +0 -71
- data/lib/ronin/web/middleware/filters.rb +0 -28
- data/lib/ronin/web/middleware/helpers.rb +0 -145
- data/lib/ronin/web/middleware/proxy.rb +0 -265
- data/lib/ronin/web/middleware/proxy_request.rb +0 -262
- data/lib/ronin/web/middleware/request.rb +0 -79
- data/lib/ronin/web/middleware/response.rb +0 -33
- data/lib/ronin/web/middleware/router.rb +0 -167
- data/lib/ronin/web/middleware/rule.rb +0 -103
- data/lib/ronin/web/middleware.rb +0 -27
- data/lib/ronin/web/proxy/app.rb +0 -32
- data/lib/ronin/web/proxy/base.rb +0 -46
- data/lib/ronin/web/proxy/web.rb +0 -46
- data/lib/ronin/web/proxy.rb +0 -25
- data/lib/ronin/web/server/app.rb +0 -32
- data/lib/ronin/web/server/base.rb +0 -461
- data/lib/ronin/web/server/web.rb +0 -66
- data/lib/ronin/web/server.rb +0 -25
- data/lib/ronin/web/spider.rb +0 -115
- data/lib/ronin/web/web.rb +0 -573
- data/spec/helpers/output.rb +0 -3
- data/spec/web/extensions/nokogiri_spec.rb +0 -38
- data/spec/web/helpers/rack_app.rb +0 -31
- data/spec/web/helpers/root/test1/index.html +0 -1
- data/spec/web/helpers/root/test1/test1.txt +0 -1
- data/spec/web/helpers/root/test1.txt +0 -1
- data/spec/web/helpers/root/test2/test2.txt +0 -1
- data/spec/web/helpers/root/test2.txt +0 -1
- data/spec/web/helpers/root/test3/test3.txt +0 -1
- data/spec/web/helpers/root/test3.txt +0 -1
- data/spec/web/helpers/root.rb +0 -15
- data/spec/web/middleware/directories_spec.rb +0 -86
- data/spec/web/middleware/files_spec.rb +0 -57
- data/spec/web/middleware/filters/campaign_filter_spec.rb +0 -30
- data/spec/web/middleware/filters/ip_filter_spec.rb +0 -25
- data/spec/web/middleware/filters/path_filter_spec.rb +0 -29
- data/spec/web/middleware/filters/referer_filter_spec.rb +0 -25
- data/spec/web/middleware/filters/user_agent_filter_spec.rb +0 -25
- data/spec/web/middleware/filters/vhost_filter_spec.rb +0 -23
- data/spec/web/middleware/proxy_spec.rb +0 -67
- data/spec/web/middleware/response_spec.rb +0 -20
- data/spec/web/middleware/router_spec.rb +0 -65
- data/spec/web/middleware/rule_spec.rb +0 -37
- data/spec/web/proxy/base_spec.rb +0 -8
- data/spec/web/server/base_spec.rb +0 -77
- data/spec/web/server/classes/public1/static1.txt +0 -1
- data/spec/web/server/classes/public2/static2.txt +0 -1
- data/spec/web/server/classes/sub_app.rb +0 -13
- data/spec/web/server/classes/test_app.rb +0 -20
- data/spec/web/web_spec.rb +0 -157
data/README.md
CHANGED
@@ -1,198 +1,213 @@
|
|
1
|
-
#
|
1
|
+
# ronin-web
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
* [Documentation](http://rubydoc.info/github/ronin-ruby/ronin-web/frames)
|
6
|
-
* [Mailing List](http://groups.google.com/group/ronin-ruby)
|
7
|
-
* irc.freenode.net #ronin
|
3
|
+
[![CI](https://github.com/ronin-rb/ronin-web/actions/workflows/ruby.yml/badge.svg)](https://github.com/ronin-rb/ronin-web/actions/workflows/ruby.yml)
|
4
|
+
[![Code Climate](https://codeclimate.com/github/ronin-rb/ronin-web.svg)](https://codeclimate.com/github/ronin-rb/ronin-web)
|
8
5
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
Ronin allows for the rapid development and distribution of code, exploits
|
16
|
-
or payloads over many common Source-Code-Management (SCM) systems.
|
17
|
-
|
18
|
-
### Ruby
|
6
|
+
* [Source](https://github.com/ronin-rb/ronin-web)
|
7
|
+
* [Issues](https://github.com/ronin-rb/ronin-web/issues)
|
8
|
+
* [Documentation](https://ronin-rb.dev/docs/ronin-web/frames)
|
9
|
+
* [Discord](https://discord.gg/6WAb3PsVX9) |
|
10
|
+
[Twitter](https://twitter.com/ronin_rb) |
|
11
|
+
[Mastodon](https://infosec.exchange/@ronin_rb)
|
19
12
|
|
20
|
-
|
21
|
-
ease. The Ruby environment contains a multitude of convenience methods
|
22
|
-
for working with data in Ruby, a Ruby Object Database, a customized Ruby
|
23
|
-
Console and an extendable command-line interface.
|
24
|
-
|
25
|
-
### Extend
|
26
|
-
|
27
|
-
Ronin's more specialized features are provided by additional Ronin
|
28
|
-
libraries, which users can choose to install. These libraries can allow
|
29
|
-
one to write and run Exploits and Payloads, scan for PHP vulnerabilities,
|
30
|
-
perform Google Dorks or run 3rd party scanners.
|
13
|
+
## Description
|
31
14
|
|
32
|
-
|
15
|
+
ronin-web is a collection of useful Web-related libraries, classes, and
|
16
|
+
CLI commands.
|
33
17
|
|
34
|
-
|
35
|
-
|
36
|
-
hosted on any SVN, Hg, Git or Rsync server. Ronin makes it easy to create,
|
37
|
-
install or update Overlays.
|
18
|
+
ronin-web is part of the [ronin-rb] project, a [Ruby] toolkit for security
|
19
|
+
research and development.
|
38
20
|
|
39
21
|
## Features
|
40
22
|
|
41
|
-
*
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
*
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
*
|
52
|
-
|
53
|
-
* Provides {Ronin::Web::Proxy}, a [Sinatra](http://sinatrarb.com/) based
|
54
|
-
Web Proxy.
|
23
|
+
* HTML/XML parsing/building (using [Nokogiri][nokogiri]).
|
24
|
+
* Also provides additional extensions to [Nokogiri][nokogiri] using
|
25
|
+
[nokogiri-ext].
|
26
|
+
* Supports diffing HTML/XML documents using [nokogiri-diff].
|
27
|
+
* Automated Web Browsing using [Mechanize][mechanize].
|
28
|
+
* Supports random `User-Agent` generation using [ronin-web-user_agents].
|
29
|
+
* Provides an easy to use [Sinatra][sintra] based web server using
|
30
|
+
[ronin-web-server].
|
31
|
+
* Provides an easy to use web spider using [ronin-web-spider].
|
32
|
+
* Provides a CLI for common web related tasks.
|
33
|
+
* Has 98% documentation coverage.
|
34
|
+
* Has 89% test coverage.
|
55
35
|
|
56
36
|
## Synopsis
|
57
37
|
|
58
|
-
|
38
|
+
```
|
39
|
+
Usage: ronin-web [options] [COMMAND [ARGS...]]
|
59
40
|
|
60
|
-
|
41
|
+
Options:
|
42
|
+
-h, --help Print help information
|
43
|
+
|
44
|
+
Arguments:
|
45
|
+
[COMMAND] The command name to run
|
46
|
+
[ARGS ...] Additional arguments for the command
|
47
|
+
|
48
|
+
Commands:
|
49
|
+
diff
|
50
|
+
help
|
51
|
+
html
|
52
|
+
irb
|
53
|
+
new
|
54
|
+
reverse-proxy
|
55
|
+
server
|
56
|
+
spider
|
57
|
+
```
|
61
58
|
|
62
59
|
## Examples
|
63
60
|
|
64
61
|
Get a web-page:
|
65
62
|
|
66
|
-
|
63
|
+
```ruby
|
64
|
+
Web.get('http://www.rubyinside.com/')
|
65
|
+
```
|
67
66
|
|
68
67
|
Get only the body of the web-page:
|
69
68
|
|
70
|
-
|
69
|
+
```ruby
|
70
|
+
Web.get_body('http://www.rubyinside.com/')
|
71
|
+
```
|
71
72
|
|
72
|
-
Get a [Mechanize agent]
|
73
|
+
Get a [Mechanize agent][mechanize]:
|
73
74
|
|
74
|
-
|
75
|
+
```ruby
|
76
|
+
agent = Web.agent
|
77
|
+
```
|
75
78
|
|
76
79
|
Parse HTML:
|
77
80
|
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
81
|
+
```ruby
|
82
|
+
HTML.parse(open('some_file.html'))
|
83
|
+
# => <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">
|
84
|
+
# <html>
|
85
|
+
# <head>
|
86
|
+
# <script type="text/javascript" src="redirect.js"></script>
|
87
|
+
# </head>
|
88
|
+
# </html>
|
89
|
+
```
|
85
90
|
|
86
91
|
Build a HTML document:
|
87
92
|
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
puts doc.to_html
|
97
|
-
# <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">
|
98
|
-
# <html><head><script src="redirect.js" type="text/javascript"></script></head></html>
|
93
|
+
```ruby
|
94
|
+
doc = HTML.build do
|
95
|
+
html {
|
96
|
+
head {
|
97
|
+
script(:type => 'text/javascript', :src => 'redirect.js')
|
98
|
+
}
|
99
|
+
}
|
100
|
+
end
|
99
101
|
|
100
|
-
|
102
|
+
puts doc.to_html
|
103
|
+
# <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">
|
104
|
+
# <html><head><script src="redirect.js" type="text/javascript"></script></head></html>
|
105
|
+
```
|
101
106
|
|
102
|
-
|
103
|
-
# => <?xml version="1.0"?>
|
104
|
-
# <users>
|
105
|
-
# <user>
|
106
|
-
# <name>admin</name>
|
107
|
-
# <password>0mni</password>
|
108
|
-
# </user>
|
109
|
-
# </users>
|
107
|
+
Parse XML:
|
110
108
|
|
109
|
+
```ruby
|
110
|
+
XML.parse(some_text)
|
111
|
+
# => <?xml version="1.0"?>
|
112
|
+
# <users>
|
113
|
+
# <user>
|
114
|
+
# <name>admin</name>
|
115
|
+
# <password>0mni</password>
|
116
|
+
# </user>
|
117
|
+
# </users>
|
118
|
+
```
|
111
119
|
|
112
120
|
Build a XML document:
|
113
121
|
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
Web::Spider.host('www.example.com') do |spider|
|
139
|
-
spider.every_url do |url|
|
140
|
-
# ...
|
141
|
-
end
|
142
|
-
|
143
|
-
spider.every_page do |page|
|
144
|
-
# ...
|
145
|
-
end
|
146
|
-
end
|
147
|
-
|
148
|
-
Serve files via a Web Server:
|
149
|
-
|
150
|
-
require 'ronin/web/server'
|
151
|
-
|
152
|
-
Web.server do
|
153
|
-
file '/opensearch.xml', '/tmp/test.xml'
|
154
|
-
directory '/download/', '/tmp/download/'
|
155
|
-
end
|
156
|
-
|
157
|
-
Web.server.get '/test' do
|
158
|
-
'Test 1 2 1 2'
|
159
|
-
end
|
122
|
+
```ruby
|
123
|
+
doc = XML.build do
|
124
|
+
playlist {
|
125
|
+
mp3 {
|
126
|
+
file { text('02 THE WAIT.mp3') }
|
127
|
+
artist { text('Evil Nine') }
|
128
|
+
track { text('The Wait feat David Autokratz') }
|
129
|
+
duration { text('1000000000') }
|
130
|
+
}
|
131
|
+
}
|
132
|
+
end
|
133
|
+
|
134
|
+
puts doc.to_xml
|
135
|
+
# <?xml version="1.0"?>
|
136
|
+
# <playlist>
|
137
|
+
# <mp3>
|
138
|
+
# <file>02 THE WAIT.mp3</file>
|
139
|
+
# <artist>Evil Nine</artist>
|
140
|
+
# <track>The Wait feat David Autokratz</track>
|
141
|
+
# <duration>1000000000</duration>
|
142
|
+
# </mp3>
|
143
|
+
# </playlist>
|
144
|
+
```
|
160
145
|
|
161
146
|
## Requirements
|
162
147
|
|
163
|
-
* [
|
164
|
-
|
165
|
-
* [
|
166
|
-
* [
|
167
|
-
* [
|
168
|
-
* [
|
169
|
-
* [
|
170
|
-
* [
|
171
|
-
* [ronin-support]
|
172
|
-
* [ronin]
|
148
|
+
* [Ruby] >= 3.0.0
|
149
|
+
* [nokogiri] ~> 1.4
|
150
|
+
* [libxml2]
|
151
|
+
* [libxslt1]
|
152
|
+
* [nokogiri-ext] ~> 0.1
|
153
|
+
* [nokogiri-diff] ~> 0.2
|
154
|
+
* [mechanize] ~> 2.0
|
155
|
+
* [open_namespace] ~> 0.4
|
156
|
+
* [ronin-support] ~> 1.0
|
157
|
+
* [ronin-web-server] ~> 0.1
|
158
|
+
* [ronin-web-spider] ~> 0.1
|
159
|
+
* [ronin-web-user_agents] ~> 0.1
|
160
|
+
* [ronin-core] ~> 0.1
|
173
161
|
|
174
162
|
## Install
|
175
163
|
|
176
|
-
|
164
|
+
```shell
|
165
|
+
$ gem install ronin-web
|
166
|
+
```
|
177
167
|
|
178
|
-
##
|
168
|
+
## Development
|
169
|
+
|
170
|
+
1. [Fork It!](https://github.com/ronin-rb/ronin-web/fork)
|
171
|
+
2. Clone It!
|
172
|
+
3. `cd ronin-web`
|
173
|
+
4. `bundle install`
|
174
|
+
5. `git checkout -b my_feature`
|
175
|
+
6. Code It!
|
176
|
+
7. `bundle exec rake spec`
|
177
|
+
8. `git push origin my_feature`
|
179
178
|
|
180
|
-
|
181
|
-
scraping and spidering functionality.
|
179
|
+
## License
|
182
180
|
|
183
|
-
|
181
|
+
ronin-web - A collection of useful web helper methods and commands.
|
184
182
|
|
185
|
-
|
183
|
+
Copyright (c) 2006-2022 Hal Brodigan (postmodern.mod3 at gmail.com)
|
186
184
|
|
187
|
-
|
185
|
+
ronin-web is free software: you can redistribute it and/or modify
|
188
186
|
it under the terms of the GNU General Public License as published by
|
189
187
|
the Free Software Foundation, either version 3 of the License, or
|
190
188
|
(at your option) any later version.
|
191
189
|
|
192
|
-
|
190
|
+
ronin-web is distributed in the hope that it will be useful,
|
193
191
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
194
192
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
195
193
|
GNU General Public License for more details.
|
196
194
|
|
197
195
|
You should have received a copy of the GNU General Public License
|
198
|
-
along with
|
196
|
+
along with ronin-web. If not, see <https://www.gnu.org/licenses/>.
|
197
|
+
|
198
|
+
[ronin-rb]: https://ronin-rb.dev
|
199
|
+
[Ruby]: https://www.ruby-lang.org
|
200
|
+
|
201
|
+
[nokogiri]: https://nokogiri.org/
|
202
|
+
[nokogiri-ext]: https://github.com/postmodern/nokogiri-ext#readme
|
203
|
+
[nokogiri-diff]: https://github.com/postmodern/nokogiri-diff#readme
|
204
|
+
[libxml2]: http://xmlsoft.org/
|
205
|
+
[libxslt1]: http://xmlsoft.org/XSLT/
|
206
|
+
[mechanize]: https://github.com/sparklemotion/mechanize#readme
|
207
|
+
[open_namespace]: https://github.com/postmodern/open_namespace#readme
|
208
|
+
[ronin-support]: https://github.com/ronin-rb/ronin-support#readme
|
209
|
+
[ronin-core]: https://github.com/ronin-rb/ronin-core#readme
|
210
|
+
[ronin-web-server]: https://github.com/ronin-rb/ronin-web-server#readme
|
211
|
+
[ronin-web-spider]: https://github.com/ronin-rb/ronin-web-spider#readme
|
212
|
+
[ronin-web-user_agents]: https://github.com/ronin-rb/ronin-web-user_agents#readme
|
213
|
+
[ronin]: https://github.com/ronin-rb/ronin#readme
|
data/Rakefile
CHANGED
@@ -3,28 +3,37 @@ require 'rubygems'
|
|
3
3
|
begin
|
4
4
|
require 'bundler'
|
5
5
|
rescue LoadError => e
|
6
|
-
|
7
|
-
|
6
|
+
warn e.message
|
7
|
+
warn "Run `gem install bundler` to install Bundler."
|
8
8
|
exit e.status_code
|
9
9
|
end
|
10
10
|
|
11
11
|
begin
|
12
12
|
Bundler.setup(:development)
|
13
13
|
rescue Bundler::BundlerError => e
|
14
|
-
|
15
|
-
|
14
|
+
warn e.message
|
15
|
+
warn "Run `bundle install` to install missing gems"
|
16
16
|
exit e.status_code
|
17
17
|
end
|
18
18
|
|
19
19
|
require 'rake'
|
20
20
|
|
21
|
-
require '
|
22
|
-
|
21
|
+
require 'rubygems/tasks'
|
22
|
+
Gem::Tasks.new(sign: {checksum: true, pgp: true})
|
23
23
|
|
24
24
|
require 'rspec/core/rake_task'
|
25
25
|
RSpec::Core::RakeTask.new
|
26
|
-
task :test
|
26
|
+
task :test => :spec
|
27
27
|
task :default => :spec
|
28
28
|
|
29
|
+
namespace :spec do
|
30
|
+
RSpec::Core::RakeTask.new(:network) do |t|
|
31
|
+
t.rspec_opts = '--tag network'
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
29
35
|
require 'yard'
|
30
36
|
YARD::Rake::YardocTask.new
|
37
|
+
|
38
|
+
require 'kramdown/man/task'
|
39
|
+
Kramdown::Man::Task.new
|
data/bin/ronin-web
CHANGED
@@ -2,26 +2,18 @@
|
|
2
2
|
|
3
3
|
require 'rubygems'
|
4
4
|
|
5
|
-
|
6
|
-
if File.directory?(File.join(
|
7
|
-
Dir.chdir(
|
8
|
-
require 'bundler'
|
9
|
-
|
5
|
+
root = File.expand_path(File.join(File.dirname(__FILE__),'..'))
|
6
|
+
if File.directory?(File.join(root,'.git'))
|
7
|
+
Dir.chdir(root) do
|
10
8
|
begin
|
11
|
-
|
12
|
-
rescue
|
13
|
-
|
14
|
-
|
15
|
-
exit
|
9
|
+
require 'bundler/setup'
|
10
|
+
rescue LoadError => e
|
11
|
+
warn e.message
|
12
|
+
warn "Run `gem install bundler` to install Bundler"
|
13
|
+
exit -1
|
16
14
|
end
|
17
15
|
end
|
18
16
|
end
|
19
17
|
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
require 'ronin/ui/console'
|
24
|
-
require 'ronin/ui/cli/commands/console'
|
25
|
-
|
26
|
-
Ronin::UI::Console.auto_load << 'ronin/web'
|
27
|
-
Ronin::UI::CLI::Commands::Console.start
|
18
|
+
require 'ronin/web/cli'
|
19
|
+
Ronin::Web::CLI.start
|
@@ -0,0 +1,22 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'ronin/web/server'
|
4
|
+
|
5
|
+
class App < Ronin::Web::Server::Base
|
6
|
+
<%- if (@host || @port) -%>
|
7
|
+
|
8
|
+
<%- if @host -%>
|
9
|
+
set :host, <%= @host.inspect %>
|
10
|
+
<%- end -%>
|
11
|
+
<%- if @port -%>
|
12
|
+
set :port, <%= @port.inspect %>
|
13
|
+
<%- end -%>
|
14
|
+
<%- end -%>
|
15
|
+
|
16
|
+
get '/' do
|
17
|
+
"hello world"
|
18
|
+
end
|
19
|
+
|
20
|
+
end
|
21
|
+
|
22
|
+
App.run! if __FILE__ == $0
|
@@ -0,0 +1,26 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'ronin/web/spider'
|
4
|
+
|
5
|
+
Ronin::Web::Spider.<%= entry_point_method -%>(<%= entry_point_argument -%>) do |agent|
|
6
|
+
<%- unless @callbacks.empty? -%>
|
7
|
+
<%- @callbacks.each_with_index do |(method,arguments,yield_arg),index| -%>
|
8
|
+
<%- unless arguments.empty? -%>
|
9
|
+
agent.<%= method -%>(<%= arguments.map(&:inspect).join(', ') -%>) do |<%= yield_arg -%>|
|
10
|
+
# ...
|
11
|
+
end
|
12
|
+
<%- else -%>
|
13
|
+
agent.<%= method -%> do |<%= yield_arg -%>|
|
14
|
+
# ...
|
15
|
+
end
|
16
|
+
<%- end -%>
|
17
|
+
<%- unless index == (@callbacks.length-1) -%>
|
18
|
+
|
19
|
+
<%- end -%>
|
20
|
+
<%- end -%>
|
21
|
+
<%- else -%>
|
22
|
+
agent.ever_page do |page|
|
23
|
+
puts page.url
|
24
|
+
end
|
25
|
+
<%- end -%>
|
26
|
+
end
|
@@ -0,0 +1 @@
|
|
1
|
+
<%= @ruby_version -%>
|
data/gemspec.yml
CHANGED
@@ -1,31 +1,46 @@
|
|
1
1
|
name: ronin-web
|
2
2
|
summary:
|
3
|
-
A Ruby library
|
3
|
+
A Ruby library that provides support for web scraping and
|
4
4
|
spidering functionality.
|
5
|
+
|
5
6
|
description:
|
6
|
-
|
7
|
-
|
7
|
+
ronin-web is a Ruby library that provides support for web scraping and
|
8
|
+
spidering functionality.
|
8
9
|
|
9
|
-
license: GPL-3
|
10
|
+
license: GPL-3.0
|
10
11
|
authors: Postmodern
|
11
12
|
email: postmodern.mod3@gmail.com
|
12
|
-
homepage:
|
13
|
+
homepage: https://github.com/ronin-rb/ronin-web#readme
|
13
14
|
has_yard: true
|
14
15
|
|
16
|
+
required_ruby_version: ">= 3.0.0"
|
17
|
+
|
18
|
+
generated_files:
|
19
|
+
- man/ronin-web.1
|
20
|
+
- man/ronin-web-diff.1
|
21
|
+
- man/ronin-web-html.1
|
22
|
+
- man/ronin-web-irb.1
|
23
|
+
- man/ronin-web-new.1
|
24
|
+
- man/ronin-web-new-nokogiri.1
|
25
|
+
- man/ronin-web-new-server.1
|
26
|
+
- man/ronin-web-new-spider.1
|
27
|
+
- man/ronin-web-new-webapp.1
|
28
|
+
- man/ronin-web-server.1
|
29
|
+
- man/ronin-web-spider.1
|
30
|
+
- man/ronin-web-reverse-proxy.1
|
31
|
+
|
15
32
|
dependencies:
|
16
33
|
nokogiri: ~> 1.4
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
# data_paths: ~> 0.3
|
22
|
-
data_paths: ~> 0.3.0.rc1
|
34
|
+
nokogiri-ext: ~> 0.1
|
35
|
+
nokogiri-diff: ~> 0.2
|
36
|
+
mechanize: ~> 2.0
|
37
|
+
open_namespace: ~> 0.4
|
23
38
|
# Ronin dependencies:
|
24
|
-
|
25
|
-
ronin-
|
26
|
-
|
27
|
-
ronin: ~>
|
39
|
+
ronin-support: ~> 1.0.0.beta1
|
40
|
+
ronin-web-server: ~> 0.1.0.beta1
|
41
|
+
ronin-web-spider: ~> 0.1.0.beta2
|
42
|
+
ronin-web-user_agents: ~> 0.1.0.beta1
|
43
|
+
ronin-core: ~> 0.1.0.beta1
|
28
44
|
|
29
45
|
development_dependencies:
|
30
|
-
bundler: ~>
|
31
|
-
yard: ~> 0.7.0
|
46
|
+
bundler: ~> 2.0
|