ronin-web 1.0.0.beta1 → 1.0.0.beta3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ruby.yml +0 -4
- data/README.md +5 -8
- data/data/new/nokogiri.rb.erb +1 -1
- data/data/new/spider.rb.erb +1 -1
- data/gemspec.yml +12 -8
- data/lib/ronin/web/cli/command.rb +1 -1
- data/lib/ronin/web/cli/commands/diff.rb +1 -1
- data/lib/ronin/web/cli/commands/html.rb +1 -1
- data/lib/ronin/web/cli/commands/irb.rb +1 -1
- data/lib/ronin/web/cli/commands/new/nokogiri.rb +1 -1
- data/lib/ronin/web/cli/commands/new/server.rb +1 -1
- data/lib/ronin/web/cli/commands/new/spider.rb +1 -1
- data/lib/ronin/web/cli/commands/new/webapp.rb +1 -1
- data/lib/ronin/web/cli/commands/new.rb +1 -1
- data/lib/ronin/web/cli/commands/reverse_proxy.rb +10 -3
- data/lib/ronin/web/cli/commands/server.rb +8 -2
- data/lib/ronin/web/cli/commands/spider.rb +7 -3
- data/lib/ronin/web/cli/ruby_shell.rb +1 -1
- data/lib/ronin/web/cli.rb +6 -1
- data/lib/ronin/web/html.rb +1 -1
- data/lib/ronin/web/mechanize.rb +1 -1
- data/lib/ronin/web/root.rb +1 -1
- data/lib/ronin/web/version.rb +2 -2
- data/lib/ronin/web/xml.rb +1 -1
- data/lib/ronin/web.rb +8 -8
- data/man/ronin-web-spider.1 +1 -1
- data/man/ronin-web-spider.1.md +1 -1
- data/ronin-web.gemspec +2 -1
- metadata +10 -21
- data/spec/cli/ruby_shell_spec.rb +0 -14
- data/spec/html_spec.rb +0 -43
- data/spec/mechanize_spec.rb +0 -72
- data/spec/spec_helper.rb +0 -8
- data/spec/web_spec.rb +0 -97
- data/spec/xml_spec.rb +0 -42
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4e36e87f767a665ff3dafb90022b8cb0b0d8d7d9ebc23255c873dcebde04d7b1
|
4
|
+
data.tar.gz: af5f942251ddefcfd8940d2de5cc688edbd6e55ba96715d8798b0e85efff33f6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9e0bc704b1177e0f958c00a0f1f33d606258c78186dc2ac38de85d7188d29e9911aab5d337cb39a7e90798865841b947abd8c6e928325df2a394e7a61b6576a2
|
7
|
+
data.tar.gz: c7dd821942055424f990ab2f5511252308d9b12bf3bdb2ccb07281d523d40066425989d1645a27f464321147973b1ff12410c6bcc79cece614d09efbaf730e57
|
data/.github/workflows/ruby.yml
CHANGED
@@ -21,10 +21,6 @@ jobs:
|
|
21
21
|
uses: ruby/setup-ruby@v1
|
22
22
|
with:
|
23
23
|
ruby-version: ${{ matrix.ruby }}
|
24
|
-
- name: Install libsqlite3
|
25
|
-
run: |
|
26
|
-
sudo apt update -y && \
|
27
|
-
sudo apt install -y --no-install-recommends --no-install-suggests libsqlite3-dev libxml2-dev libxslt1-dev
|
28
24
|
- name: Install dependencies
|
29
25
|
run: bundle install --jobs 4 --retry 3
|
30
26
|
- name: Run tests
|
data/README.md
CHANGED
@@ -12,8 +12,8 @@
|
|
12
12
|
|
13
13
|
## Description
|
14
14
|
|
15
|
-
ronin-web is a
|
16
|
-
|
15
|
+
ronin-web is a Ruby library that provides common web security commands and
|
16
|
+
additional libraries.
|
17
17
|
|
18
18
|
ronin-web is part of the [ronin-rb] project, a [Ruby] toolkit for security
|
19
19
|
research and development.
|
@@ -26,7 +26,7 @@ research and development.
|
|
26
26
|
* Supports diffing HTML/XML documents using [nokogiri-diff].
|
27
27
|
* Automated Web Browsing using [Mechanize][mechanize].
|
28
28
|
* Supports random `User-Agent` generation using [ronin-web-user_agents].
|
29
|
-
* Provides an easy to use [Sinatra][
|
29
|
+
* Provides an easy to use [Sinatra][sinatra] based web server using
|
30
30
|
[ronin-web-server].
|
31
31
|
* Provides an easy to use web spider using [ronin-web-spider].
|
32
32
|
* Provides a CLI for common web related tasks.
|
@@ -147,8 +147,6 @@ puts doc.to_xml
|
|
147
147
|
|
148
148
|
* [Ruby] >= 3.0.0
|
149
149
|
* [nokogiri] ~> 1.4
|
150
|
-
* [libxml2]
|
151
|
-
* [libxslt1]
|
152
150
|
* [nokogiri-ext] ~> 0.1
|
153
151
|
* [nokogiri-diff] ~> 0.2
|
154
152
|
* [mechanize] ~> 2.0
|
@@ -180,7 +178,7 @@ $ gem install ronin-web
|
|
180
178
|
|
181
179
|
ronin-web - A collection of useful web helper methods and commands.
|
182
180
|
|
183
|
-
Copyright (c) 2006-
|
181
|
+
Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
|
184
182
|
|
185
183
|
ronin-web is free software: you can redistribute it and/or modify
|
186
184
|
it under the terms of the GNU General Public License as published by
|
@@ -201,8 +199,6 @@ along with ronin-web. If not, see <https://www.gnu.org/licenses/>.
|
|
201
199
|
[nokogiri]: https://nokogiri.org/
|
202
200
|
[nokogiri-ext]: https://github.com/postmodern/nokogiri-ext#readme
|
203
201
|
[nokogiri-diff]: https://github.com/postmodern/nokogiri-diff#readme
|
204
|
-
[libxml2]: http://xmlsoft.org/
|
205
|
-
[libxslt1]: http://xmlsoft.org/XSLT/
|
206
202
|
[mechanize]: https://github.com/sparklemotion/mechanize#readme
|
207
203
|
[open_namespace]: https://github.com/postmodern/open_namespace#readme
|
208
204
|
[ronin-support]: https://github.com/ronin-rb/ronin-support#readme
|
@@ -211,3 +207,4 @@ along with ronin-web. If not, see <https://www.gnu.org/licenses/>.
|
|
211
207
|
[ronin-web-spider]: https://github.com/ronin-rb/ronin-web-spider#readme
|
212
208
|
[ronin-web-user_agents]: https://github.com/ronin-rb/ronin-web-user_agents#readme
|
213
209
|
[ronin]: https://github.com/ronin-rb/ronin#readme
|
210
|
+
[sinatra]: https://sinatrarb.com/
|
data/data/new/nokogiri.rb.erb
CHANGED
data/data/new/spider.rb.erb
CHANGED
data/gemspec.yml
CHANGED
@@ -1,11 +1,8 @@
|
|
1
1
|
name: ronin-web
|
2
|
-
summary:
|
3
|
-
A Ruby library that provides support for web scraping and
|
4
|
-
spidering functionality.
|
5
|
-
|
2
|
+
summary: A collection of common web security commands and libraries.
|
6
3
|
description:
|
7
|
-
ronin-web is a Ruby library that provides
|
8
|
-
|
4
|
+
ronin-web is a Ruby library that provides common web security commands and
|
5
|
+
additional libraries.
|
9
6
|
|
10
7
|
license: GPL-3.0
|
11
8
|
authors: Postmodern
|
@@ -15,6 +12,13 @@ has_yard: true
|
|
15
12
|
|
16
13
|
required_ruby_version: ">= 3.0.0"
|
17
14
|
|
15
|
+
metadata:
|
16
|
+
documentation_uri: https://ronin-rb.dev/docs/ronin-web
|
17
|
+
source_code_uri: https://github.com/ronin-rb/ronin-web
|
18
|
+
bug_tracker_uri: https://github.com/ronin-rb/ronin-web/issues
|
19
|
+
changelog_uri: https://github.com/ronin-rb/ronin-web/blob/main/ChangeLog.md
|
20
|
+
rubygems_mfa_required: 'true'
|
21
|
+
|
18
22
|
generated_files:
|
19
23
|
- man/ronin-web.1
|
20
24
|
- man/ronin-web-diff.1
|
@@ -36,8 +40,8 @@ dependencies:
|
|
36
40
|
mechanize: ~> 2.0
|
37
41
|
open_namespace: ~> 0.4
|
38
42
|
# Ronin dependencies:
|
39
|
-
ronin-support: ~> 1.0.0.
|
40
|
-
ronin-web-server: ~> 0.1.0.
|
43
|
+
ronin-support: ~> 1.0.0.beta2
|
44
|
+
ronin-web-server: ~> 0.1.0.beta3
|
41
45
|
ronin-web-spider: ~> 0.1.0.beta2
|
42
46
|
ronin-web-user_agents: ~> 0.1.0.beta1
|
43
47
|
ronin-core: ~> 0.1.0.beta1
|
@@ -1,7 +1,7 @@
|
|
1
1
|
#
|
2
2
|
# ronin-web - A collection of useful web helper methods and commands.
|
3
3
|
#
|
4
|
-
# Copyright (c) 2006-
|
4
|
+
# Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
|
5
5
|
#
|
6
6
|
# ronin-web is free software: you can redistribute it and/or modify
|
7
7
|
# it under the terms of the GNU General Public License as published by
|
@@ -1,7 +1,7 @@
|
|
1
1
|
#
|
2
2
|
# ronin-web - A collection of useful web helper methods and commands.
|
3
3
|
#
|
4
|
-
# Copyright (c) 2006-
|
4
|
+
# Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
|
5
5
|
#
|
6
6
|
# ronin-web is free software: you can redistribute it and/or modify
|
7
7
|
# it under the terms of the GNU General Public License as published by
|
@@ -1,7 +1,7 @@
|
|
1
1
|
#
|
2
2
|
# ronin-web - A collection of useful web helper methods and commands.
|
3
3
|
#
|
4
|
-
# Copyright (c) 2006-
|
4
|
+
# Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
|
5
5
|
#
|
6
6
|
# ronin-web is free software: you can redistribute it and/or modify
|
7
7
|
# it under the terms of the GNU General Public License as published by
|
@@ -1,7 +1,7 @@
|
|
1
1
|
#
|
2
2
|
# ronin-web - A collection of useful web helper methods and commands.
|
3
3
|
#
|
4
|
-
# Copyright (c) 2006-
|
4
|
+
# Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
|
5
5
|
#
|
6
6
|
# ronin-web is free software: you can redistribute it and/or modify
|
7
7
|
# it under the terms of the GNU General Public License as published by
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
#
|
3
|
-
# Copyright (c) 2006-
|
3
|
+
# Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
|
4
4
|
#
|
5
5
|
# ronin-web is free software: you can redistribute it and/or modify
|
6
6
|
# it under the terms of the GNU General Public License as published by
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
#
|
3
|
-
# Copyright (c) 2006-
|
3
|
+
# Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
|
4
4
|
#
|
5
5
|
# ronin-web is free software: you can redistribute it and/or modify
|
6
6
|
# it under the terms of the GNU General Public License as published by
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
#
|
3
|
-
# Copyright (c) 2006-
|
3
|
+
# Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
|
4
4
|
#
|
5
5
|
# ronin-web is free software: you can redistribute it and/or modify
|
6
6
|
# it under the terms of the GNU General Public License as published by
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
#
|
3
|
-
# Copyright (c) 2006-
|
3
|
+
# Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
|
4
4
|
#
|
5
5
|
# ronin-web is free software: you can redistribute it and/or modify
|
6
6
|
# it under the terms of the GNU General Public License as published by
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
#
|
3
|
-
# Copyright (c) 2006-
|
3
|
+
# Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
|
4
4
|
#
|
5
5
|
# ronin-web is free software: you can redistribute it and/or modify
|
6
6
|
# it under the terms of the GNU General Public License as published by
|
@@ -1,7 +1,7 @@
|
|
1
1
|
#
|
2
2
|
# ronin-web - A collection of useful web helper methods and commands.
|
3
3
|
#
|
4
|
-
# Copyright (c) 2006-
|
4
|
+
# Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
|
5
5
|
#
|
6
6
|
# ronin-web is free software: you can redistribute it and/or modify
|
7
7
|
# it under the terms of the GNU General Public License as published by
|
@@ -45,7 +45,7 @@ module Ronin
|
|
45
45
|
#
|
46
46
|
# @api private
|
47
47
|
#
|
48
|
-
class
|
48
|
+
class ReverseProxy < Command
|
49
49
|
|
50
50
|
include Core::CLI::Logging
|
51
51
|
|
@@ -142,7 +142,14 @@ module Ronin
|
|
142
142
|
end
|
143
143
|
|
144
144
|
log_info "Starting proxy server on #{options[:host]}:#{options[:port]} ..."
|
145
|
-
|
145
|
+
|
146
|
+
begin
|
147
|
+
proxy.run!(host: options[:host], port: options[:port])
|
148
|
+
rescue Errno::EADDRINUSE => error
|
149
|
+
log_error(error.message)
|
150
|
+
exit(1)
|
151
|
+
end
|
152
|
+
|
146
153
|
log_info "shutting down ..."
|
147
154
|
end
|
148
155
|
|
@@ -1,7 +1,7 @@
|
|
1
1
|
#
|
2
2
|
# ronin-web - A collection of useful web helper methods and commands.
|
3
3
|
#
|
4
|
-
# Copyright (c) 2006-
|
4
|
+
# Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
|
5
5
|
#
|
6
6
|
# ronin-web is free software: you can redistribute it and/or modify
|
7
7
|
# it under the terms of the GNU General Public License as published by
|
@@ -144,7 +144,13 @@ module Ronin
|
|
144
144
|
end
|
145
145
|
|
146
146
|
log_info "Starting web server listening on #{App.host}:#{App.port} ..."
|
147
|
-
|
147
|
+
begin
|
148
|
+
App.run!
|
149
|
+
rescue Errno::EADDRINUSE => error
|
150
|
+
log_error(error.message)
|
151
|
+
exit(1)
|
152
|
+
end
|
153
|
+
|
148
154
|
log_info "Shutting down ..."
|
149
155
|
end
|
150
156
|
|
@@ -1,7 +1,7 @@
|
|
1
1
|
#
|
2
2
|
# ronin-web - A collection of useful web helper methods and commands.
|
3
3
|
#
|
4
|
-
# Copyright (c) 2006-
|
4
|
+
# Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
|
5
5
|
#
|
6
6
|
# ronin-web is free software: you can redistribute it and/or modify
|
7
7
|
# it under the terms of the GNU General Public License as published by
|
@@ -49,7 +49,7 @@ module Ronin
|
|
49
49
|
# -P, --proxy PROXY Sets the proxy to use.
|
50
50
|
# -H, --header NAME: VALUE Sets a default header
|
51
51
|
# --host-header NAME=VALUE Sets a default header
|
52
|
-
# -u
|
52
|
+
# -u chrome-linux|chrome-macos|chrome-windows|chrome-iphone|chrome-ipad|chrome-android|firefox-linux|firefox-macos|firefox-windows|firefox-iphone|firefox-ipad|firefox-android|safari-macos|safari-iphone|safari-ipad|edge,
|
53
53
|
# --user-agent The User-Agent to use
|
54
54
|
# -U, --user-agent-string STRING The User-Agent string to use
|
55
55
|
# -R, --referer URL Sets the Referer URL
|
@@ -183,7 +183,11 @@ module Ronin
|
|
183
183
|
|
184
184
|
option :user_agent, short: '-u',
|
185
185
|
value: {
|
186
|
-
type:
|
186
|
+
type: Hash[
|
187
|
+
Support::Network::HTTP::UserAgents::ALIASES.keys.map { |key|
|
188
|
+
[key.to_s.tr('_','-'), key]
|
189
|
+
}
|
190
|
+
]
|
187
191
|
},
|
188
192
|
desc: 'The User-Agent to use' do |name|
|
189
193
|
@user_agent = name
|
@@ -1,7 +1,7 @@
|
|
1
1
|
#
|
2
2
|
# ronin-web - A collection of useful web helper methods and commands.
|
3
3
|
#
|
4
|
-
# Copyright (c) 2006-
|
4
|
+
# Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
|
5
5
|
#
|
6
6
|
# ronin-web is free software: you can redistribute it and/or modify
|
7
7
|
# it under the terms of the GNU General Public License as published by
|
data/lib/ronin/web/cli.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
#
|
2
2
|
# ronin-web - A collection of useful web helper methods and commands.
|
3
3
|
#
|
4
|
-
# Copyright (c) 2006-
|
4
|
+
# Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
|
5
5
|
#
|
6
6
|
# ronin-web is free software: you can redistribute it and/or modify
|
7
7
|
# it under the terms of the GNU General Public License as published by
|
@@ -17,8 +17,11 @@
|
|
17
17
|
# along with ronin-web. If not, see <https://www.gnu.org/licenses/>.
|
18
18
|
#
|
19
19
|
|
20
|
+
require 'ronin/web/version'
|
21
|
+
|
20
22
|
require 'command_kit/commands'
|
21
23
|
require 'command_kit/commands/auto_load'
|
24
|
+
require 'command_kit/options/version'
|
22
25
|
|
23
26
|
module Ronin
|
24
27
|
module Web
|
@@ -36,8 +39,10 @@ module Ronin
|
|
36
39
|
dir: "#{__dir__}/cli/commands",
|
37
40
|
namespace: "#{self}::Commands"
|
38
41
|
)
|
42
|
+
include CommandKit::Options::Version
|
39
43
|
|
40
44
|
command_name 'ronin-web'
|
45
|
+
version Ronin::Web::VERSION
|
41
46
|
|
42
47
|
end
|
43
48
|
end
|
data/lib/ronin/web/html.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
#
|
2
2
|
# ronin-web - A collection of useful web helper methods and commands.
|
3
3
|
#
|
4
|
-
# Copyright (c) 2006-
|
4
|
+
# Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
|
5
5
|
#
|
6
6
|
# ronin-web is free software: you can redistribute it and/or modify
|
7
7
|
# it under the terms of the GNU General Public License as published by
|
data/lib/ronin/web/mechanize.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
#
|
2
2
|
# ronin-web - A collection of useful web helper methods and commands.
|
3
3
|
#
|
4
|
-
# Copyright (c) 2006-
|
4
|
+
# Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
|
5
5
|
#
|
6
6
|
# ronin-web is free software: you can redistribute it and/or modify
|
7
7
|
# it under the terms of the GNU General Public License as published by
|
data/lib/ronin/web/root.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
#
|
2
2
|
# ronin-web - A collection of useful web helper methods and commands.
|
3
3
|
#
|
4
|
-
# Copyright (c) 2006-
|
4
|
+
# Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
|
5
5
|
#
|
6
6
|
# ronin-web is free software: you can redistribute it and/or modify
|
7
7
|
# it under the terms of the GNU General Public License as published by
|
data/lib/ronin/web/version.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
#
|
2
2
|
# ronin-web - A collection of useful web helper methods and commands.
|
3
3
|
#
|
4
|
-
# Copyright (c) 2006-
|
4
|
+
# Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
|
5
5
|
#
|
6
6
|
# ronin-web is free software: you can redistribute it and/or modify
|
7
7
|
# it under the terms of the GNU General Public License as published by
|
@@ -20,6 +20,6 @@
|
|
20
20
|
module Ronin
|
21
21
|
module Web
|
22
22
|
# ronin-web Version
|
23
|
-
VERSION = '1.0.0.
|
23
|
+
VERSION = '1.0.0.beta3'
|
24
24
|
end
|
25
25
|
end
|
data/lib/ronin/web/xml.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
#
|
2
2
|
# ronin-web - A collection of useful web helper methods and commands.
|
3
3
|
#
|
4
|
-
# Copyright (c) 2006-
|
4
|
+
# Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
|
5
5
|
#
|
6
6
|
# ronin-web is free software: you can redistribute it and/or modify
|
7
7
|
# it under the terms of the GNU General Public License as published by
|
data/lib/ronin/web.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
#
|
2
2
|
# ronin-web - A collection of useful web helper methods and commands.
|
3
3
|
#
|
4
|
-
# Copyright (c) 2006-
|
4
|
+
# Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
|
5
5
|
#
|
6
6
|
# ronin-web is free software: you can redistribute it and/or modify
|
7
7
|
# it under the terms of the GNU General Public License as published by
|
@@ -195,15 +195,15 @@ module Ronin
|
|
195
195
|
#
|
196
196
|
# @api public
|
197
197
|
#
|
198
|
-
def self.open(url, proxy:
|
199
|
-
user_agent:
|
198
|
+
def self.open(url, proxy: Support::Network::HTTP.proxy,
|
199
|
+
user_agent: Support::Network::HTTP.user_agent,
|
200
200
|
referer: nil,
|
201
201
|
cookie: nil,
|
202
202
|
**kwargs)
|
203
|
-
|
203
|
+
options = {proxy: proxy, **kwargs}
|
204
204
|
|
205
205
|
if user_agent
|
206
|
-
|
206
|
+
options['User-Agent'] = case user_agent
|
207
207
|
when Symbol
|
208
208
|
Support::Network::HTTP::UserAgents[user_agent]
|
209
209
|
else
|
@@ -211,10 +211,10 @@ module Ronin
|
|
211
211
|
end
|
212
212
|
end
|
213
213
|
|
214
|
-
|
215
|
-
|
214
|
+
options['Referer'] = referer if referer
|
215
|
+
options['Cookie'] = cookie if cookie
|
216
216
|
|
217
|
-
return URI.open(url,
|
217
|
+
return URI.open(url,options)
|
218
218
|
end
|
219
219
|
|
220
220
|
#
|
data/man/ronin-web-spider.1
CHANGED
@@ -47,7 +47,7 @@ Sets a default header\.
|
|
47
47
|
Sets a default header\.
|
48
48
|
.LP
|
49
49
|
.HP
|
50
|
-
\fB-u\fR, \fB--user-agent\fR chrome
|
50
|
+
\fB-u\fR, \fB--user-agent\fR chrome\-linux\[or]chrome\-macos\[or]chrome\-windows\[or]chrome\-iphone\[or]chrome\-ipad\[or]chrome\-android\[or]firefox\-linux\[or]firefox\-macos\[or]firefox\-windows\[or]firefox\-iphone\[or]firefox\-ipad\[or]firefox\-android\[or]safari\-macos\[or]safari\-iphone\[or]safari\-ipad\[or]edge
|
51
51
|
The \fBUser-Agent\fR to use\.
|
52
52
|
.LP
|
53
53
|
.TP
|
data/man/ronin-web-spider.1.md
CHANGED
@@ -34,7 +34,7 @@ Spiders a website.
|
|
34
34
|
`--host-header` *NAME*=*VALUE*
|
35
35
|
Sets a default header.
|
36
36
|
|
37
|
-
`-u`, `--user-agent`
|
37
|
+
`-u`, `--user-agent` chrome-linux|chrome-macos|chrome-windows|chrome-iphone|chrome-ipad|chrome-android|firefox-linux|firefox-macos|firefox-windows|firefox-iphone|firefox-ipad|firefox-android|safari-macos|safari-iphone|safari-ipad|edge
|
38
38
|
The `User-Agent` to use.
|
39
39
|
|
40
40
|
`-U`, `--user-agent-string` *STRING*
|
data/ronin-web.gemspec
CHANGED
@@ -26,6 +26,8 @@ Gem::Specification.new do |gem|
|
|
26
26
|
gem.files = `git ls-files`.split($/)
|
27
27
|
gem.files = glob[gemspec['files']] if gemspec['files']
|
28
28
|
gem.files += Array(gemspec['generated_files'])
|
29
|
+
# exclude test files from the packages gem
|
30
|
+
gem.files -= glob[gemspec['test_files'] || 'spec/{**/}*']
|
29
31
|
|
30
32
|
gem.executables = gemspec.fetch('executables') do
|
31
33
|
glob['bin/*'].map { |path| File.basename(path) }
|
@@ -33,7 +35,6 @@ Gem::Specification.new do |gem|
|
|
33
35
|
gem.default_executable = gem.executables.first if Gem::VERSION < '1.7.'
|
34
36
|
|
35
37
|
gem.extensions = glob[gemspec['extensions'] || 'ext/**/extconf.rb']
|
36
|
-
gem.test_files = glob[gemspec['test_files'] || 'spec/{**/}*_spec.rb']
|
37
38
|
gem.extra_rdoc_files = glob[gemspec['extra_doc_files'] || '*.{txt,md}']
|
38
39
|
|
39
40
|
gem.require_paths = Array(gemspec.fetch('require_paths') {
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ronin-web
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.0.
|
4
|
+
version: 1.0.0.beta3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Postmodern
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-01-
|
11
|
+
date: 2023-01-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: nokogiri
|
@@ -86,28 +86,28 @@ dependencies:
|
|
86
86
|
requirements:
|
87
87
|
- - "~>"
|
88
88
|
- !ruby/object:Gem::Version
|
89
|
-
version: 1.0.0.
|
89
|
+
version: 1.0.0.beta2
|
90
90
|
type: :runtime
|
91
91
|
prerelease: false
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
94
|
- - "~>"
|
95
95
|
- !ruby/object:Gem::Version
|
96
|
-
version: 1.0.0.
|
96
|
+
version: 1.0.0.beta2
|
97
97
|
- !ruby/object:Gem::Dependency
|
98
98
|
name: ronin-web-server
|
99
99
|
requirement: !ruby/object:Gem::Requirement
|
100
100
|
requirements:
|
101
101
|
- - "~>"
|
102
102
|
- !ruby/object:Gem::Version
|
103
|
-
version: 0.1.0.
|
103
|
+
version: 0.1.0.beta3
|
104
104
|
type: :runtime
|
105
105
|
prerelease: false
|
106
106
|
version_requirements: !ruby/object:Gem::Requirement
|
107
107
|
requirements:
|
108
108
|
- - "~>"
|
109
109
|
- !ruby/object:Gem::Version
|
110
|
-
version: 0.1.0.
|
110
|
+
version: 0.1.0.beta3
|
111
111
|
- !ruby/object:Gem::Dependency
|
112
112
|
name: ronin-web-spider
|
113
113
|
requirement: !ruby/object:Gem::Requirement
|
@@ -164,8 +164,8 @@ dependencies:
|
|
164
164
|
- - "~>"
|
165
165
|
- !ruby/object:Gem::Version
|
166
166
|
version: '2.0'
|
167
|
-
description: ronin-web is a Ruby library that provides
|
168
|
-
|
167
|
+
description: ronin-web is a Ruby library that provides common web security commands
|
168
|
+
and additional libraries.
|
169
169
|
email: postmodern.mod3@gmail.com
|
170
170
|
executables:
|
171
171
|
- ronin-web
|
@@ -245,12 +245,6 @@ files:
|
|
245
245
|
- man/ronin-web.1
|
246
246
|
- man/ronin-web.1.md
|
247
247
|
- ronin-web.gemspec
|
248
|
-
- spec/cli/ruby_shell_spec.rb
|
249
|
-
- spec/html_spec.rb
|
250
|
-
- spec/mechanize_spec.rb
|
251
|
-
- spec/spec_helper.rb
|
252
|
-
- spec/web_spec.rb
|
253
|
-
- spec/xml_spec.rb
|
254
248
|
homepage: https://github.com/ronin-rb/ronin-web#readme
|
255
249
|
licenses:
|
256
250
|
- GPL-3.0
|
@@ -273,10 +267,5 @@ requirements: []
|
|
273
267
|
rubygems_version: 3.3.26
|
274
268
|
signing_key:
|
275
269
|
specification_version: 4
|
276
|
-
summary: A
|
277
|
-
test_files:
|
278
|
-
- spec/cli/ruby_shell_spec.rb
|
279
|
-
- spec/html_spec.rb
|
280
|
-
- spec/mechanize_spec.rb
|
281
|
-
- spec/web_spec.rb
|
282
|
-
- spec/xml_spec.rb
|
270
|
+
summary: A collection of common web security commands and libraries.
|
271
|
+
test_files: []
|
data/spec/cli/ruby_shell_spec.rb
DELETED
@@ -1,14 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
require 'ronin/web/cli/ruby_shell'
|
3
|
-
|
4
|
-
describe Ronin::Web::CLI::RubyShell do
|
5
|
-
describe "#initialize" do
|
6
|
-
it "must default #name to 'ronin-web'" do
|
7
|
-
expect(subject.name).to eq('ronin-web')
|
8
|
-
end
|
9
|
-
|
10
|
-
it "must default #context to Ronin::Web" do
|
11
|
-
expect(subject.context).to be(Ronin::Web)
|
12
|
-
end
|
13
|
-
end
|
14
|
-
end
|
data/spec/html_spec.rb
DELETED
@@ -1,43 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
require 'ronin/web/html'
|
3
|
-
|
4
|
-
describe Ronin::Web::HTML do
|
5
|
-
describe ".parse" do
|
6
|
-
let(:html) do
|
7
|
-
<<~HTML
|
8
|
-
<html>
|
9
|
-
<body>Hello</body>
|
10
|
-
</html>
|
11
|
-
HTML
|
12
|
-
end
|
13
|
-
|
14
|
-
it "must parse an HTML String and return a Nokogiri::HTML::Document" do
|
15
|
-
doc = subject.parse(html)
|
16
|
-
|
17
|
-
expect(doc).to be_kind_of(Nokogiri::HTML::Document)
|
18
|
-
expect(doc.at('body').inner_text).to eq("Hello")
|
19
|
-
end
|
20
|
-
|
21
|
-
context "when given a block" do
|
22
|
-
it "must yield the Nokogiri::HTML::Document object" do
|
23
|
-
expect { |b|
|
24
|
-
subject.parse(html,&b)
|
25
|
-
}.to yield_with_args(Nokogiri::HTML::Document)
|
26
|
-
end
|
27
|
-
end
|
28
|
-
end
|
29
|
-
|
30
|
-
describe ".build" do
|
31
|
-
it "must build an HTML document" do
|
32
|
-
doc = subject.build do
|
33
|
-
html {
|
34
|
-
body {
|
35
|
-
div { text("hello") }
|
36
|
-
}
|
37
|
-
}
|
38
|
-
end
|
39
|
-
|
40
|
-
expect(doc.to_html).to include("<html><body><div>hello</div></body></html>")
|
41
|
-
end
|
42
|
-
end
|
43
|
-
end
|
data/spec/mechanize_spec.rb
DELETED
@@ -1,72 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
require 'ronin/web/mechanize'
|
3
|
-
|
4
|
-
describe Ronin::Web::Mechanize do
|
5
|
-
describe "#initialize" do
|
6
|
-
context "when Ronin::Support::Network::HTTP.user_agent is set" do
|
7
|
-
let(:user_agent) { 'test' }
|
8
|
-
|
9
|
-
before { Ronin::Support::Network::HTTP.user_agent = user_agent }
|
10
|
-
|
11
|
-
it "should set #user_agent to Ronin::Support::Network::HTTP.user_agent" do
|
12
|
-
expect(subject.user_agent).to eq(user_agent)
|
13
|
-
end
|
14
|
-
|
15
|
-
after { Ronin::Support::Network::HTTP.user_agent = nil }
|
16
|
-
end
|
17
|
-
|
18
|
-
context "when the :user_agent option is given" do
|
19
|
-
context "and it's a String" do
|
20
|
-
let(:user_agent) { 'test2' }
|
21
|
-
|
22
|
-
subject { described_class.new(user_agent: user_agent) }
|
23
|
-
|
24
|
-
it "should set #user_agent to the custom User-Agent string" do
|
25
|
-
expect(subject.user_agent).to eq(user_agent)
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
|
-
context "and it's a Symbol" do
|
30
|
-
let(:user_agent) { :chrome_linux }
|
31
|
-
let(:expected_user_agent) do
|
32
|
-
Ronin::Support::Network::HTTP::UserAgents[user_agent]
|
33
|
-
end
|
34
|
-
|
35
|
-
subject { described_class.new(user_agent: user_agent) }
|
36
|
-
|
37
|
-
it "should set #user_agent to the custom User-Agent alias" do
|
38
|
-
expect(subject.user_agent).to eq(expected_user_agent)
|
39
|
-
end
|
40
|
-
end
|
41
|
-
end
|
42
|
-
|
43
|
-
let(:host) { '127.0.0.1' }
|
44
|
-
let(:port) { 8080 }
|
45
|
-
let(:proxy) { URI::HTTP.build(host: host, port: port) }
|
46
|
-
|
47
|
-
context "when Ronin::Support::Network::HTTP.proxy is set" do
|
48
|
-
before { Ronin::Support::Network::HTTP.proxy = proxy }
|
49
|
-
|
50
|
-
it "should set #proxy_addr and #proxy_port to Ronin::Support::Network::HTTP.proxy" do
|
51
|
-
expect(subject.proxy_addr).to eq(
|
52
|
-
Ronin::Support::Network::HTTP.proxy.host
|
53
|
-
)
|
54
|
-
|
55
|
-
expect(subject.proxy_port).to eq(
|
56
|
-
Ronin::Support::Network::HTTP.proxy.port
|
57
|
-
)
|
58
|
-
end
|
59
|
-
|
60
|
-
after { Ronin::Support::Network::HTTP.proxy = nil }
|
61
|
-
end
|
62
|
-
|
63
|
-
context "when the :proxy option is given" do
|
64
|
-
subject { described_class.new(proxy: proxy) }
|
65
|
-
|
66
|
-
it "should set #proxy_addr and #proxy_port to the custom proxy" do
|
67
|
-
expect(subject.proxy_addr).to eq(host)
|
68
|
-
expect(subject.proxy_port).to eq(port)
|
69
|
-
end
|
70
|
-
end
|
71
|
-
end
|
72
|
-
end
|
data/spec/spec_helper.rb
DELETED
data/spec/web_spec.rb
DELETED
@@ -1,97 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
require 'ronin/web'
|
3
|
-
|
4
|
-
describe Ronin::Web do
|
5
|
-
let(:url) { 'https://ronin-rb.dev/' }
|
6
|
-
let(:title) { 'Ronin' }
|
7
|
-
|
8
|
-
it "should have a VERSION constant" do
|
9
|
-
expect(subject.const_defined?('VERSION')).to eq(true)
|
10
|
-
end
|
11
|
-
|
12
|
-
describe ".html" do
|
13
|
-
it "should be able to parse HTML" do
|
14
|
-
doc = subject.html(%{
|
15
|
-
<html>
|
16
|
-
<body>Hello</body>
|
17
|
-
</html>
|
18
|
-
})
|
19
|
-
|
20
|
-
expect(doc.at('body').inner_text).to eq("Hello")
|
21
|
-
end
|
22
|
-
end
|
23
|
-
|
24
|
-
describe ".build_html" do
|
25
|
-
it "should be able to build HTML documents" do
|
26
|
-
doc = subject.build_html do
|
27
|
-
html {
|
28
|
-
body {
|
29
|
-
div { text("hello") }
|
30
|
-
}
|
31
|
-
}
|
32
|
-
end
|
33
|
-
|
34
|
-
expect(doc.to_html).to include("<html><body><div>hello</div></body></html>")
|
35
|
-
end
|
36
|
-
end
|
37
|
-
|
38
|
-
describe ".xml" do
|
39
|
-
it "should be able to parse XML" do
|
40
|
-
doc = subject.xml(%{
|
41
|
-
<?xml version="1.0"?>
|
42
|
-
<root>
|
43
|
-
<stuff>Hello</stuff>
|
44
|
-
</root>
|
45
|
-
})
|
46
|
-
|
47
|
-
expect(doc.at('stuff').inner_text).to eq("Hello")
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
51
|
-
describe ".build_xml" do
|
52
|
-
it "should be able to build XML documents" do
|
53
|
-
doc = subject.build_xml do
|
54
|
-
root {
|
55
|
-
stuff(name: 'bla') { text("hello") }
|
56
|
-
}
|
57
|
-
end
|
58
|
-
|
59
|
-
expect(doc.to_xml).to include("<root>\n <stuff name=\"bla\">hello</stuff>\n</root>")
|
60
|
-
end
|
61
|
-
end
|
62
|
-
|
63
|
-
describe ".open", :network do
|
64
|
-
it "should open URLs as temporary files" do
|
65
|
-
file = subject.open(url)
|
66
|
-
|
67
|
-
expect(file.read).to include(title)
|
68
|
-
end
|
69
|
-
end
|
70
|
-
|
71
|
-
describe ".agent" do
|
72
|
-
it "must return a #{described_class}::Mechanize object" do
|
73
|
-
expect(subject.agent).to be_kind_of(described_class::Mechanize)
|
74
|
-
end
|
75
|
-
|
76
|
-
it "must return the same object each time" do
|
77
|
-
expect(subject.agent).to be(subject.agent)
|
78
|
-
end
|
79
|
-
end
|
80
|
-
|
81
|
-
describe ".get", :network do
|
82
|
-
it "should be able to get Mechanize pages" do
|
83
|
-
page = subject.get(url)
|
84
|
-
|
85
|
-
expect(page.class).to eq(Mechanize::Page)
|
86
|
-
expect(page.at('title').inner_text).to include(title)
|
87
|
-
end
|
88
|
-
end
|
89
|
-
|
90
|
-
describe ".get_body", :network do
|
91
|
-
it "should be able to get the bodies of Mechanize pages" do
|
92
|
-
body = subject.get_body(url)
|
93
|
-
|
94
|
-
expect(body).to include(title)
|
95
|
-
end
|
96
|
-
end
|
97
|
-
end
|
data/spec/xml_spec.rb
DELETED
@@ -1,42 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
require 'ronin/web/xml'
|
3
|
-
|
4
|
-
describe Ronin::Web::XML do
|
5
|
-
describe ".parse" do
|
6
|
-
let(:xml) do
|
7
|
-
<<~XML
|
8
|
-
<?xml version="1.0"?>
|
9
|
-
<root>
|
10
|
-
<stuff>Hello</stuff>
|
11
|
-
</root>
|
12
|
-
XML
|
13
|
-
end
|
14
|
-
|
15
|
-
it "must parse an XML String and return a Nokogiri::XML::Document" do
|
16
|
-
doc = subject.parse(xml)
|
17
|
-
|
18
|
-
expect(doc).to be_kind_of(Nokogiri::XML::Document)
|
19
|
-
expect(doc.at('stuff').inner_text).to eq("Hello")
|
20
|
-
end
|
21
|
-
|
22
|
-
context "when given a block" do
|
23
|
-
it "must yield the Nokogiri::XML::Document object" do
|
24
|
-
expect { |b|
|
25
|
-
subject.parse(xml,&b)
|
26
|
-
}.to yield_with_args(Nokogiri::XML::Document)
|
27
|
-
end
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
31
|
-
describe ".build" do
|
32
|
-
it "must build an XML document" do
|
33
|
-
doc = subject.build do
|
34
|
-
root {
|
35
|
-
stuff(name: 'bla') { text("hello") }
|
36
|
-
}
|
37
|
-
end
|
38
|
-
|
39
|
-
expect(doc.to_xml).to include("<root>\n <stuff name=\"bla\">hello</stuff>\n</root>")
|
40
|
-
end
|
41
|
-
end
|
42
|
-
end
|