ronin-web 1.0.2 → 2.0.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d74a1711d3d63c034cb9a4e37f765c27cc1dd4b7e5cbf21154fc936a1db02e02
-  data.tar.gz: 06b58a6806d23dd2a201942978349cedfd57341f9acd9c9c687dcf1c41ec95bc
+  metadata.gz: 8bd87f5c3602ae1ece1f09ff4d689221c41e63b1302f9edf58576a3aec06108e
+  data.tar.gz: 5fa836bd013ea5b74a4b79fbda0fc4dd84ac09d9abafea689aba504c3decb32e
 SHA512:
-  metadata.gz: be61036695c8a3ec44f095008352ab9d7fbc741218ca202f3dfbea9fd235bbb7a38ddc3d79896d62d349512f17085473ac2ac73aff7ae2bad4e2d7fda013c49a
-  data.tar.gz: 2ea04d21c0cd5dbdee6ffe714ffd94f89a4b2c8f2dceefeafd9daba0908f890da0b78a78b9df302ae14b39b44ca51e674c262b5fd470671382d9786f3904b6af
+  metadata.gz: b56e783acb097f2c43f7c144f5bb5225a060b48000b458ba098280a8bd690d0831141ead1c00d3cafba1922428022d36643bba780c483400c27ac3d05af3ce65
+  data.tar.gz: c78a468778822cbe8dfc0d3efa845d08471e1440fecfa849784b54bd8719bb4c8abca4568213df8de961acfafa85beb33f13dc853a1480dfafc3f37127a308b2

data/.github/workflows/ruby.yml

@@ -12,11 +12,12 @@ jobs:
           - '3.0'
           - '3.1'
           - '3.2'
+          - '3.3'
           # - jruby
           - truffleruby
     name: Ruby ${{ matrix.ruby }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
@@ -31,7 +32,7 @@ jobs:
   rubocop:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:

data/.gitignore

@@ -1,4 +1,5 @@
 /coverage
+/data/completions/ronin-web
 /doc
 /pkg
 /man/*.[1-9]

data/.rubocop.yml

@@ -18,3 +18,8 @@ Lint/ShadowingOuterLocalVariable:
   Exclude:
     - 'lib/ronin/web/cli/commands/reverse_proxy.rb'
     - 'lib/ronin/web/cli/commands/spider.rb'
+
+Naming/MethodParameterName:
+  Exclude:
+    - 'lib/ronin/web/cli/browser_shell.rb'
+    - 'lib/ronin/web/cli/js_shell.rb'

data/ChangeLog.md

@@ -1,7 +1,47 @@
+### 2.0.0 / 2024-XX-XX
+
+* Require [wordlist] ~> 1.0, >= 1.0.1.
+* Require [ronin-support-web] ~> 0.1.
+* Require [ronin-web-browser] ~> 0.1.
+* Require [ronin-web-session_cookie] ~> 0.1.
+* Require [ronin-web-spider] ~> 0.2.
+
+#### CLI
+
+* Added the `ronin-web xml` command.
+* Added the `ronin-web session-cookie` command.
+* Added the `ronin-web user-agent` command.
+* Added the `ronin-web wordlist` command.
+* Added the `ronin-web browser` command.
+* Added the `ronin-web screenshot` command.
+* Added the `ronin-web vulns` command.
+* Added the `ronin-web completion` command to install shell completion files
+  for all `ronin-web` commands for Bash and Zsh shells.
+* Added the `--format=html|xml` option to the `ronin-web diff` command.
+* Added the `-t,--text` option to `ronin-web html`.
+* Added the `--print-js-url-strings` option to the `ronin-web spider` command.
+* Added the `--print-js-path-strings` option to the `ronin-web spider` command.
+* Added the `--print-js-relative-path-strings` option to the `ronin-web spider`
+  command.
+* Added the `--print-js-absolute-path-strings` option to the `ronin-web spider`
+  command.
+* Added ANSI colored output to the `ronin-web diff` command.
+* Renamed `ronin-web new webapp` to `ronin-web new app`.
+
+### 1.0.2 / 2023-04-04
+
+* Improved documentation.
+
+#### CLI
+
+* Fixed a bug in `ronin-web server` where `App.host` was being called instead of
+  `App.bind`.
+* Fixed a typo in the `ronin-web spider --print-status` option.
+
 ### 1.0.1 / 2023-03-01
 
 * Require `ronin-web-server` ~> 0.1, >= 0.1.1.
-* Disable SSL/TLS verification by default in {Ronin::Web::Mechanize}.
+* Disable SSL/TLS verification by default in `Ronin::Web::Mechanize`.
 
 ### 1.0.0 / 2023-02-01
 
@@ -169,6 +209,11 @@
 [rack]: https://github.com/rack/rack
 [sinatra]: https://github.com/sinatra/sinatra
 [data_paths]: https://github.com/postmodern/data_paths
+[wordlist]: https://github.com/postmodern/wordlist.rb#readme
 [ronin-support]: https://github.com/ronin-rb/ronin-support
+[ronin-support-web]: https://github.com/ronin-rb/ronin-support-web#readme
+[ronin-web-browser]: https://github.com/ronin-rb/ronin-web-browser#readme
+[ronin-web-session_cookie]: https://github.com/ronin-rb/ronin-web-session_cookie#readme
+[ronin-web-spider]: https://github.com/ronin-rb/ronin-web-spider#readme
 [ronin]: https://github.com/ronin-rb/ronin
 [ronin-scanners]: https://github.com/ronin-rb/ronin-scanners

data/Gemfile

@@ -4,11 +4,6 @@ source 'https://rubygems.org'
 
 gemspec
 
-if RUBY_VERSION >= '3.0'
-  # XXX: dep in webrick for mechanize for Ruby 3.0
-  gem 'webrick', platform: :ruby
-end
-
 platforms :jruby do
   gem 'jruby-openssl', '~> 0.7'
 end
@@ -21,18 +16,33 @@ end
 # gem 'command_kit',  '~> 0.4', github: 'postmodern/command_kit.rb',
 #                               branch: '0.4.0'
 
-# gem 'spidr', '~> 0.7', github: 'postmodern/spidr'
+# gem 'spidr',    '~> 0.7', github: 'postmodern/spidr'
+# gem 'wordlist', '~> 1.0', github: 'postmodern/wordlist.rb'
 
 # Ronin dependencies
-# gem 'ronin-support',         '~> 1.0', github: "ronin-rb/ronin-support",
-#                                        branch: 'main'
+# gem 'ronin-support',     '~> 1.1', github: "ronin-rb/ronin-support",
+#                                    branch: 'main'
+# gem 'ronin-support-web', '~> 0.1', github: "ronin-rb/ronin-support-web",
+#                                    branch: 'main'
+# gem 'ronin-core',        '~> 0.2', github: "ronin-rb/ronin-core",
+#                                    branch: 'main'
+
+# gem 'ferrum',                      github: 'rubycdp/ferrum'
+# gem 'ronin-web-browser', '~> 0.1', github: 'ronin-rb/ronin-web-browser'
 # gem 'ronin-web-server',      '~> 0.1', github: "ronin-rb/ronin-web-server",
 #                                        branch: 'main'
-# gem 'ronin-web-spider',      '~> 0.1', github: "ronin-rb/ronin-web-spider",
+# gem 'ronin-web-spider',      '~> 0.2', github: 'ronin-rb/ronin-web-spider',
 #                                        branch: 'main'
-# gem 'ronin-web-user_agents', '~> 0.1', github: "ronin-rb/ronin-web-user_agents",
+# gem 'ronin-web-user_agents', '~> 0.1', github: 'ronin-rb/ronin-web-user_agents',
 #                                        branch: 'main'
-# gem 'ronin-core',	       '~> 0.1', github: "ronin-rb/ronin-core",
+# gem 'ronin-web-session_cookie', '~> 0.1', github: 'ronin-rb/ronin-web-session_cookie',
+#                                           branch: 'main'
+
+# gem 'ronin-db',              '~> 0.2', github: 'ronin-rb/ronin-db',
+#                                        branch: 'main'
+# gem 'ronin-db-activerecord', '~> 0.2', github: 'ronin-rb/ronin-db-activerecord',
+#                                        branch: 'main'
+# gem 'ronin-vulns',           '~> 0.2', github: 'ronin-rb/ronin-vulns',
 #                                        branch: 'main'
 
 group :development do
@@ -42,17 +52,20 @@ group :development do
   gem 'rspec',           '~> 3.0'
   gem 'simplecov',       '~> 0.20'
   gem 'rack-test',       '~> 0.6'
+  gem 'webmock',         '~> 3.0'
 
   gem 'kramdown',        '~> 2.0'
   gem 'redcarpet',       platform: :mri
   gem 'yard',            '~> 0.9'
   gem 'yard-spellcheck', require: false
 
-  gem 'kramdown-man',    '~> 0.1'
+  gem 'kramdown-man',    '~> 1.0'
 
   gem 'dead_end',        require: false
   gem 'sord',            require: false, platform: :mri
   gem 'stackprof',       require: false, platform: :mri
   gem 'rubocop',         require: false, platform: :mri
   gem 'rubocop-ronin',   require: false, platform: :mri
+
+  gem 'command_kit-completion', '~> 0.2', require: false
 end

data/README.md

@@ -8,7 +8,6 @@
 * [Issues](https://github.com/ronin-rb/ronin-web/issues)
 * [Documentation](https://ronin-rb.dev/docs/ronin-web/frames)
 * [Discord](https://discord.gg/6WAb3PsVX9) |
-  [Twitter](https://twitter.com/ronin_rb) |
   [Mastodon](https://infosec.exchange/@ronin_rb)
 
 ## Description
@@ -25,7 +24,6 @@ research and development.
   * Also provides additional extensions to [Nokogiri][nokogiri] using
     [nokogiri-ext].
 * Supports diffing HTML/XML documents using [nokogiri-diff].
-* Automated Web Browsing using [Mechanize][mechanize].
 * Supports random `User-Agent` generation using [ronin-web-user_agents].
 * Provides an easy to use [Sinatra][sinatra] based web server using
   [ronin-web-server].
@@ -47,14 +45,21 @@ Arguments:
     [ARGS ...]                       Additional arguments for the command
 
 Commands:
+    completion
     diff
     help
     html
     irb
     new
     reverse-proxy
+    screenshot
     server
+    session-cookie
     spider
+    user-agent
+    vulns
+    wordlist
+    xml
 ```
 
 Open the `ronin-web` Ruby REPL:
@@ -287,56 +292,98 @@ $ ronin-web new server server.rb
 Generate a new web app:
 
 ```shell
-$ ronin-web new webapp app
-	mkdir	app
-	mkdir	app/lib
-	mkdir	app/views
-	mkdir	app/public
-	erb	.ruby-version.erb	app/.ruby-version
-	cp	Gemfile	app
-	erb	app.rb.erb	app/app.rb
-	cp	config.ru	app
+$ ronin-web new app myapp
+	mkdir	myapp
+	mkdir	myapp/lib
+	mkdir	myapp/views
+	mkdir	myapp/public
+	erb	.ruby-version.erb	myapp/.ruby-version
+	cp	Gemfile	myapp
+	erb	app.rb.erb	myapp/app.rb
+	cp	config.ru	myapp
 ```
 
-## Examples
-
-Get a web-page:
+Open the Ronin Web Ruby REPL:
 
-```ruby
-Web.get('http://www.rubyinside.com/')
+```
+$ ronin-web irb
+                                                                   , Jµ     ▓▓█▓
+                                                  J▌      ▐▓██▌ ████ ██    ▐███D
+                                      ╓▄▓▓█████▌  ██µ     ████ ▄███ÖJ██▌   ███▌
+        ,╓µ▄▄▄▄▄▄▄▄µ;,            ,▄▓██████████  ▐███    ▐███▀ ███▌ ████µ ▄███
+¬∞MÆ▓███████████████████████▓M  ▄██████▀▀╙████▌  ████▌   ████ ▄███ J█████ ███▌
+           `█████▀▀▀▀▀███████  -████▀└    ████  ▐█████n ▄███O ███▌ ██████████
+           ▓████L       ████▀  ▓████     ▓███Ö  ███████ ███▌ ▓███ ▐█████████▀
+          ▄████▀  ,╓▄▄▄█████  J████Ü    ,███▌  ▄███████████ J███▀ ████ █████
+         J█████████████████─  ████▌     ████   ████`██████▌ ████ ▐███Ü ▐███Ü
+         ███████████▀▀▀╙└    ▐████     J███▌  ▓███▌ ²█████ J███Ü ███▌   ▀█▌
+        ▓██████████▌         ████▌     ████  ;████   ▀███▀ ███▌ J▀▀▀-    █
+       ▄█████▀ ▀█████µ      ▐████  ,▄▓████▀  ████▀    ███ J███           `
+      J█████-    ╙▀███▄     ████████████▀╙  J█▀▀▀      █U  ▀█▌
+      ████▀         ▀███   ▄████████▀▀                 ╨    █
+     ▓██▀             ²▀█▄ █▀▀▀╙└
+    ▄██╜                 ╙W
+   J█▀
+   ▌└
+  ┘
+
+irb(ronin-web)>
 ```
 
-Get only the body of the web-page:
+## Examples
 
-```ruby
-Web.get_body('http://www.rubyinside.com/')
-```
+### HTML
 
-Get a [Mechanize agent][mechanize]:
+Parse an HTML string:
 
 ```ruby
-agent = Web.agent
+doc = html_parse("<html>\n  <body>\n    <p>Hello world</p>\n  </body>\n</html>\n")
+# =>
+# #(Document:0x6ab8 {
+#   name = "document",
+#   children = [
+#     #(DTD:0x6be4 { name = "html" }),
+#     #(Element:0x6cd4 {
+#       name = "html",
+#       children = [
+#         #(Text "\n  "),
+#         #(Element:0x6e64 {
+#           name = "body",
+#           children = [
+#             #(Text "\n    "),
+#             #(Element:0x6ff4 { name = "p", children = [ #(Text "Hello world")] }),
+#             #(Text "\n  ")]
+#           }),
+#         #(Text "\n")]
+#       })]
+#   })
+```
+
+Parse a HTML file:
+
+```ruby
+doc = html_open("index.html")
+# => #<Nokogiri::HTML::Document:...>
 ```
 
-Parse HTML:
+Searching an HTML document using [XPath] or CSS-path:
 
 ```ruby
-HTML.parse(open('some_file.html'))
-# => <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">
-# <html>
-#   <head>
-#     <script type="text/javascript" src="redirect.js"></script>
-#   </head>
-# </html>
+nodes = doc.search('//div/p')
+nodes = doc.search('div p.class')
+# => [#<Nokogiri::HTML::Element:...>, ...]
+
+node = doc.at('#id')
+# => #<Nokogiri::HTML::Element:...>
 ```
 
 Build a HTML document:
 
 ```ruby
-doc = HTML.build do
+doc = html_build do
   html {
     head {
-      script(:type => 'text/javascript', :src => 'redirect.js')
+      script(type: 'text/javascript', src: 'redirect.js')
     }
   }
 end
@@ -346,23 +393,56 @@ puts doc.to_html
 # <html><head><script src="redirect.js" type="text/javascript"></script></head></html>
 ```
 
-Parse XML:
+### XML
+
+Parse an XML response body:
+
+```ruby
+xml_parse("<?xml version=\"1.0\"?>\n<users>\n  <user>\n    <name>admin</name>\n    <password>0mni</password>\n  <user>\n</users>\n")
+# =>
+# #(Document:0xdebc {
+#   name = "document",
+#   children = [
+#     #(Element:0xdfe8 {
+#       name = "users",
+#       children = [
+#         #(Text "\n  "),
+#         #(Element:0xe178 {
+#           name = "user",
+#           children = [
+#             #(Text "\n    "),
+#             #(Element:0xe308 { name = "name", children = [ #(Text "admin")] }),
+#             #(Text "\n    "),
+#             #(Element:0xe538 { name = "password", children = [ #(Text "0mni")] }),
+#             #(Text "\n  "),
+#             #(Element:0xe768 { name = "user", children = [ #(Text "\n")] }),
+#             #(Text "\n")]
+#           })]
+#       })]
+#   })
+```
+
+Parse a XML file:
+
+```ruby
+doc = html_open("data.xml")
+# => #<Nokogiri:XML:::Document:...>
+```
+
+Searching an XML document using [XPath]:
 
 ```ruby
-XML.parse(some_text)
-# => <?xml version="1.0"?>
-# <users>
-#   <user>
-#     <name>admin</name>
-#     <password>0mni</password>
-#   </user>
-# </users>
+users = doc.search('//user')
+# => [#<Nokogiri::XML::Element:...>, ...]
+
+admin = doc.at('//user[@name="admin"]')
+# => #<Nokogiri::XML::Element:...>
 ```
 
 Build a XML document:
 
 ```ruby
-doc = XML.build do
+doc = xml_build do
   playlist {
     mp3 {
       file { text('02 THE WAIT.mp3') }
@@ -385,19 +465,143 @@ puts doc.to_xml
 # </playlist>
 ```
 
+### Web Requests
+
+Gets a URL and follows any redirects:
+
+```ruby
+get 'https://example.com/'
+# => #<Net::HTTPResponse:...>
+```
+
+Gets a URL and parses the HTML response:
+
+```ruby
+get_html 'https://example.com/'
+# => #<Nokogiri::HTML::Document:...>
+```
+
+Gets a URL and parses the XML response:
+
+```ruby
+get_xml 'https://example.com/sitemap.xml'
+# => #<Nokogiri::XML::Document:...>
+```
+
+Gets a URL and parses the JSON response:
+
+```ruby
+get_json 'https://example.com/api/endpoint.json'
+# => {...}
+```
+
+POSTs to a URL and follows any redirects:
+
+```ruby
+post 'https://example.com/form', form_data: {'foo' => 'bar'}
+# => #<Net::HTTPResponse:...>
+```
+
+POSTs to a URL and parses the HTML response:
+
+```ruby
+post_html 'https://example.com/form', form_data: {'foo' => 'bar'}
+# => #<Nokogiri::HTML::Document:...>
+```
+
+POSTs to a URL and parses the XML response:
+
+```ruby
+post_xml 'https://example.com/form', form_data: {'foo' => 'bar'}
+# => #<Nokogiri::XML::Document:...>
+```
+
+POSTs to a URL and parses the JSON response:
+
+```ruby
+post_json 'https://example.com/api/endpoint.json', json: {foo: 'bar'}
+# => {...}
+```
+
+### User Agents
+
+Get a random `User-Agent` string:
+
+```ruby
+user_agent = UserAgents.random
+# => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.230 Safari/537.36"
+```
+
+For more examples, see [ronin-web-user_agents][ronin-web-user_agents-examples].
+
+[ronin-web-user_agents-examples]: https://github.com/ronin-rb/ronin-web-user_agents#examples
+
+### Session Cookie
+
+Parse a Django JSON session cookie:
+
+```ruby
+SessionCookie.parse('sessionid=eyJmb28iOiJiYXIifQ:1pQcTx:UufiSnuPIjNs7zOAJS0UpqnyvRt7KET7BVes0I8LYbA')
+# =>
+# #<Ronin::Web::SessionCookie::Django:0x00007f29bb9c6b70
+#  @hmac=
+#   "R\xE7\xE2J{\x8F\"3l\xEF3\x80%-\x14\xA6\xA9\xF2\xBD\e{(D\xFB\x05W\xAC\xD0\x8F\va\xB0",
+#  @params={"foo"=>"bar"},
+#  @salt=1676070425>
+```
+
+For more examples, see [ronin-web-session_cookie][ronin-web-session_cookie-examples].
+
+[ronin-web-session_cookie-examples]: https://github.com/ronin-rb/ronin-web-session_cookie#examples
+
+### Spider
+
+Spider a website and print out visited URLs:
+
+```ruby
+Spider.site('http://www.rubyinside.com/') do |spider|
+  spider.every_url { |url| puts url }
+end
+```
+
+For more examples, see [ronin-web-spider][ronin-web-spider-examples].
+
+[ronin-web-spider-examples]: https://github.com/ronin-rb/ronin-web-spider#examples
+
+### Browser
+
+Open a visible web browser and intercept all requests:
+
+```ruby
+browser = Ronin::Web::Browser.new(visible: true)
+browser.every_request do |request|
+  puts "> #{request.method} #{request.url}"
+end
+
+browser.go_to("https://twitter.com/login")
+```
+
+For more examples, see [ronin-web-browser][ronin-web-browser-examples].
+
+[ronin-web-browser-examples]: https://github.com/ronin-rb/ronin-web-browser#examples
+
 ## Requirements
 
 * [Ruby] >= 3.0.0
 * [nokogiri] ~> 1.4
-* [nokogiri-ext] ~> 0.1
 * [nokogiri-diff] ~> 0.2
-* [mechanize] ~> 2.0
+* [robots] ~> 0.10
 * [open_namespace] ~> 0.4
-* [ronin-support] ~> 1.0
+* [wordlist] ~> 1.0, >= 1.0.1
+* [ronin-support] ~> 1.1
+* [ronin-support-web] ~> 0.1
+* [ronin-web-browser] ~> 0.1
 * [ronin-web-server] ~> 0.1
-* [ronin-web-spider] ~> 0.1
+* [ronin-web-spider] ~> 0.2
 * [ronin-web-user_agents] ~> 0.1
-* [ronin-core] ~> 0.1
+* [ronin-web-session_cookie] ~> 0.1
+* [ronin-core] ~> 0.2
+* [ronin-vulns] ~> 0.2
 
 ## Install
 
@@ -410,7 +614,7 @@ $ gem install ronin-web
 1. [Fork It!](https://github.com/ronin-rb/ronin-web/fork)
 2. Clone It!
 3. `cd ronin-web`
-4. `bundle install`
+4. `./scripts/setup`
 5. `git checkout -b my_feature`
 6. Code It!
 7. `bundle exec rake spec`
@@ -420,7 +624,7 @@ $ gem install ronin-web
 
 ronin-web - A collection of useful web helper methods and commands.
 
-Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
+Copyright (c) 2006-2024 Hal Brodigan (postmodern.mod3 at gmail.com)
 
 ronin-web is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -439,14 +643,16 @@ along with ronin-web.  If not, see <https://www.gnu.org/licenses/>.
 [Ruby]: https://www.ruby-lang.org
 
 [nokogiri]: https://nokogiri.org/
-[nokogiri-ext]: https://github.com/postmodern/nokogiri-ext#readme
 [nokogiri-diff]: https://github.com/postmodern/nokogiri-diff#readme
-[mechanize]: https://github.com/sparklemotion/mechanize#readme
 [open_namespace]: https://github.com/postmodern/open_namespace#readme
 [ronin-support]: https://github.com/ronin-rb/ronin-support#readme
+[ronin-support-web]: https://github.com/ronin-rb/ronin-support-web#readme
 [ronin-core]: https://github.com/ronin-rb/ronin-core#readme
+[ronin-web-browser]: https://github.com/ronin-rb/ronin-web-browser#readme
 [ronin-web-server]: https://github.com/ronin-rb/ronin-web-server#readme
 [ronin-web-spider]: https://github.com/ronin-rb/ronin-web-spider#readme
 [ronin-web-user_agents]: https://github.com/ronin-rb/ronin-web-user_agents#readme
 [ronin]: https://github.com/ronin-rb/ronin#readme
 [sinatra]: https://sinatrarb.com/
+
+[XPath]: https://developer.mozilla.org/en-US/docs/Web/XPath

data/Rakefile

@@ -37,3 +37,12 @@ YARD::Rake::YardocTask.new
 
 require 'kramdown/man/task'
 Kramdown::Man::Task.new
+
+require 'command_kit/completion/task'
+CommandKit::Completion::Task.new(
+  class_file:  'ronin/web/cli',
+  class_name:  'Ronin::Web::CLI',
+  output_file: 'data/completions/ronin-web'
+)
+
+task :setup => %w[man command_kit:completion]