ronin-web 1.0.0 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b096e4cbd0c20e6ff15103dd87681a23a1253f1da9af3bf6b74aac474082fcf2
-  data.tar.gz: bb0b34acff8be4bf2b7a89522c7acc593df7adea843f258b616596dd07632423
+  metadata.gz: d74a1711d3d63c034cb9a4e37f765c27cc1dd4b7e5cbf21154fc936a1db02e02
+  data.tar.gz: 06b58a6806d23dd2a201942978349cedfd57341f9acd9c9c687dcf1c41ec95bc
 SHA512:
-  metadata.gz: 798cfd3b63bcc624b9f79819794923bad26c4127a0b0fbc8f28d55fe02afaf2f90cf070f265afc6360e29a35d1afcc8a62f0721ad0256ff1accb86a3cfb8f6ee
-  data.tar.gz: f87af34ce8994118a453ac1ec672e5af5f140ee916271291fa23a194d7d87f80f906a87fefe8bf82e2453aad3a2fed60b267729e8de6330419dca916bcc13f8f
+  metadata.gz: be61036695c8a3ec44f095008352ab9d7fbc741218ca202f3dfbea9fd235bbb7a38ddc3d79896d62d349512f17085473ac2ac73aff7ae2bad4e2d7fda013c49a
+  data.tar.gz: 2ea04d21c0cd5dbdee6ffe714ffd94f89a4b2c8f2dceefeafd9daba0908f890da0b78a78b9df302ae14b39b44ca51e674c262b5fd470671382d9786f3904b6af

data/.github/workflows/ruby.yml

@@ -26,3 +26,17 @@ jobs:
         run: bundle install --jobs 4 --retry 3
       - name: Run tests
         run: bundle exec rake test
+
+  # rubocop linting
+  rubocop:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 3.0
+      - name: Install dependencies
+        run: bundle install --jobs 4 --retry 3
+      - name: Run rubocop
+        run: bundle exec rubocop --parallel

data/.rubocop.yml

@@ -0,0 +1,20 @@
+AllCops:
+  NewCops: enable
+  SuggestExtensions: false
+  TargetRubyVersion: 3.1
+
+inherit_gem:
+  rubocop-ronin: rubocop.yml
+
+#
+# ronin-web specific exceptions
+#
+
+Style/FrozenStringLiteralComment:
+  Exclude:
+    - 'data/new/**/*'
+
+Lint/ShadowingOuterLocalVariable:
+  Exclude:
+    - 'lib/ronin/web/cli/commands/reverse_proxy.rb'
+    - 'lib/ronin/web/cli/commands/spider.rb'

data/ChangeLog.md

@@ -1,4 +1,9 @@
-### 1.0.0 / 2023-XX-XX
+### 1.0.1 / 2023-03-01
+
+* Require `ronin-web-server` ~> 0.1, >= 0.1.1.
+* Disable SSL/TLS verification by default in {Ronin::Web::Mechanize}.
+
+### 1.0.0 / 2023-02-01
 
 * Added [nokogiri-ext] ~> 0.1 as a dependency.
 * Added [nokogiri-diff] ~> 0.1 as a dependency.

data/Gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 gemspec
@@ -16,21 +18,21 @@ end
 # gem 'rack',         '~> 1.2', github: 'rack/rack'
 # gem 'sinatra',      '~> 1.2', github: 'sinatra/sinatra'
 
-# gem 'command_kit', '~> 0.4', github: 'postmodern/command_kit.rb',
-#                              branch: '0.4.0'
+# gem 'command_kit',  '~> 0.4', github: 'postmodern/command_kit.rb',
+#                               branch: '0.4.0'
 
 # gem 'spidr', '~> 0.7', github: 'postmodern/spidr'
 
 # Ronin dependencies
-# gem 'ronin-support',	       '~> 1.0', github: "ronin-rb/ronin-support",
+# gem 'ronin-support',         '~> 1.0', github: "ronin-rb/ronin-support",
 #                                        branch: 'main'
-# gem 'ronin-web-server',	     '~> 0.1', github: "ronin-rb/ronin-web-server",
+# gem 'ronin-web-server',      '~> 0.1', github: "ronin-rb/ronin-web-server",
 #                                        branch: 'main'
-# gem 'ronin-web-spider',	     '~> 0.1', github: "ronin-rb/ronin-web-spider",
+# gem 'ronin-web-spider',      '~> 0.1', github: "ronin-rb/ronin-web-spider",
 #                                        branch: 'main'
 # gem 'ronin-web-user_agents', '~> 0.1', github: "ronin-rb/ronin-web-user_agents",
 #                                        branch: 'main'
-# gem 'ronin-core',	           '~> 0.1', github: "ronin-rb/ronin-core",
+# gem 'ronin-core',	       '~> 0.1', github: "ronin-rb/ronin-core",
 #                                        branch: 'main'
 
 group :development do
@@ -49,6 +51,8 @@ group :development do
   gem 'kramdown-man',    '~> 0.1'
 
   gem 'dead_end',        require: false
-  gem 'sord',            require: false
-  gem 'stackprof',       require: false
+  gem 'sord',            require: false, platform: :mri
+  gem 'stackprof',       require: false, platform: :mri
+  gem 'rubocop',         require: false, platform: :mri
+  gem 'rubocop-ronin',   require: false, platform: :mri
 end

data/README.md

@@ -57,6 +57,247 @@ Commands:
     spider
 ```
 
+Open the `ronin-web` Ruby REPL:
+
+```shell
+$ ronin-web irb
+```
+
+Diff two HTML files:
+
+```shell
+$ ronin-web diff index1.html index2.html
++
+
++ <div>hax</div>
+```
+
+Diff two URLs:
+
+```shell
+$ ronin-web diff http://example.com/index.html http://example.com/index2.html
+```
+
+Perform an XPath query on an HTML file:
+
+```shell
+$ ronin-web html --xpath //meta index.html
+<meta charset="utf-8">
+<meta http-equiv="Content-type" content="text/html; charset=utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+```
+
+Perform an XPath query on a URL:
+
+```shell
+$ ronin-web html --xpath //meta https://example.com/
+<meta charset="utf-8">
+<meta http-equiv="Content-type" content="text/html; charset=utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+```
+
+Dump all links from a web page:
+
+```shell
+$ ronin-web html --links https://www.google.com/
+https://www.google.com/imghp?hl=en&tab=wi
+https://maps.google.com/maps?hl=en&tab=wl
+https://play.google.com/?hl=en&tab=w8
+https://www.youtube.com/?tab=w1
+https://news.google.com/?tab=wn
+https://mail.google.com/mail/?tab=wm
+https://drive.google.com/?tab=wo
+https://www.google.com/intl/en/about/products?tab=wh
+http://www.google.com/history/optout?hl=en
+/preferences?hl=en
+https://accounts.google.com/ServiceLogin?hl=en&passive=true&continue=https://www.google.com/&ec=GAZAAQ
+/advanced_search?hl=en&authuser=0
+https://www.google.com/url?q=https://blog.google/products/search/google-search-new-fact-checking-misinformation/&source=hpp&id=19034203&ct=3&usg=AOvVaw3UxG35a-5UX1Rl8M_VwPbd&sa=X&ved=0ahUKEwjM4Iq--JD-AhVtGTQIHXMBBaYQ8IcBCAU
+/intl/en/ads/
+/services/
+/intl/en/about.html
+/intl/en/policies/privacy/
+/intl/en/policies/terms/
+```
+
+Spiders a host and print all visited URLs:
+
+```shell
+$ ronin-web spider --host www.ruby-lang.org
+http://www.ruby-lang.org/
+http://www.ruby-lang.org/en/
+http://www.ruby-lang.org/en/downloads/
+http://www.ruby-lang.org/en/documentation/
+http://www.ruby-lang.org/en/libraries/
+http://www.ruby-lang.org/en/community/
+https://www.ruby-lang.org/en/news/
+https://www.ruby-lang.org/en/security/
+https://www.ruby-lang.org/en/about/
+...
+```
+
+Spiders the domain and sub-domains and print every visited URL:
+
+```shell
+$ ronin-web spider --domain ruby-lang.org
+http://ruby-lang.org/
+https://www.ruby-lang.org/
+https://www.ruby-lang.org/en/
+https://www.ruby-lang.org/en/downloads/
+https://www.ruby-lang.org/en/documentation/
+https://www.ruby-lang.org/en/libraries/
+https://www.ruby-lang.org/en/community/
+https://www.ruby-lang.org/en/news/
+https://www.ruby-lang.org/en/security/
+https://www.ruby-lang.org/en/about/
+...
+```
+
+Spiders a specific web-site and print every visited URL:
+
+```shell
+$ ronin-web spider --site https://www.ruby-lang.org/
+https://www.ruby-lang.org/
+https://www.ruby-lang.org/en/
+https://www.ruby-lang.org/en/downloads/
+https://www.ruby-lang.org/en/documentation/
+https://www.ruby-lang.org/en/libraries/
+https://www.ruby-lang.org/en/community/
+https://www.ruby-lang.org/en/news/
+https://www.ruby-lang.org/en/security/
+https://www.ruby-lang.org/en/about/
+...
+```
+
+Spider a host and print the response statuses:
+
+```shell
+200 http://www.ruby-lang.org/
+200 http://www.ruby-lang.org/en/
+200 http://www.ruby-lang.org/en/downloads/
+200 http://www.ruby-lang.org/en/documentation/
+200 http://www.ruby-lang.org/en/libraries/
+200 http://www.ruby-lang.org/en/community/
+200 http://www.ruby-lang.org/en/news/
+200 http://www.ruby-lang.org/en/security/
+200 http://www.ruby-lang.org/en/about/
+...
+```
+
+Spider a host and print the response status and headers:
+
+```shell
+ronin-web spider --print-headers --host www.ruby-lang.org
+200 http://www.ruby-lang.org/
+    Connection: close
+    Content-Length: 887
+    Server: Cowboy
+    Strict-Transport-Security: max-age=31536000
+    Content-Type: text/html
+    Etag: W/"496ac7fab29a6094e490da28025c5857"
+    X-Frame-Options: SAMEORIGIN
+    Via: 1.1 vegur, 1.1 varnish
+    Accept-Ranges: bytes
+    Date: Tue, 04 Apr 2023 19:42:51 GMT
+    Age: 155
+    X-Served-By: cache-pdx12330-PDX
+    X-Cache: HIT
+    X-Cache-Hits: 1
+    X-Timer: S1680637372.808609,VS0,VE1
+    Vary: Accept-Encoding
+...
+```
+
+Start a debug web server on http://localhost:8000/:
+
+```shell
+$ ronin-web server
+[2023-04-04 12:26:59] INFO  WEBrick 1.7.0
+[2023-04-04 12:26:59] INFO  ruby 3.1.3 (2022-11-24) [x86_64-linux]
+== Sinatra (v3.0.4) has taken the stage on 8000 for development with backup from WEBrick
+[2023-04-04 12:26:59] INFO  WEBrick::HTTPServer#start: pid=8966 port=8000
+```
+
+Start a debug web server on a different address and port:
+
+```shell
+$ ronin-web server --host 0.0.0.0 --port 1337
+```
+
+Host the files in a directory on http://localhost:8000/:
+
+```shell
+$ ronin-web server --root .
+```
+
+Mount a specific file at a specific HTTP path:
+
+```shell
+$ ronin-web server --dir /dir/index.html:./index.html
+```
+
+Mount a specific directory at a specific HTTP path:
+
+```shell
+$ ronin-web server --dir /dir:.
+```
+
+Add a redirect to the web server:
+
+```shell
+$ ronin-web server --redirect /redirect:https://example.com/
+```
+
+Start a HTTP reverse proxy that rewrites HTTP responses on http://localhost:8080:
+
+```shell
+$ ronin-web reverse-proxy --rewrite-response Example:Hax
+```
+
+Test the reverse proxy:
+
+```shell
+$ curl -H "Host: example.com" http://localhost:8080/
+```
+
+Generate a new Ruby script for parsing HTML/XML:
+
+```shell
+$ ronin-web new nokogiri parse.rb
+	erb	nokogiri.rb.erb	parse.rb
+	chmod	parse.rb
+```
+
+Generate a new web spider script:
+
+```shell
+$ ronin-web new spider --host=www.example.com spider.rb
+	erb	spider.rb.erb	spider.rb
+	chmod	spider.rb
+```
+
+Generate a new web server script:
+
+```shell
+$ ronin-web new server server.rb
+	erb	server.rb.erb	server.rb
+	chmod	server.rb
+```
+
+Generate a new web app:
+
+```shell
+$ ronin-web new webapp app
+	mkdir	app
+	mkdir	app/lib
+	mkdir	app/views
+	mkdir	app/public
+	erb	.ruby-version.erb	app/.ruby-version
+	cp	Gemfile	app
+	erb	app.rb.erb	app/app.rb
+	cp	config.ru	app
+```
+
 ## Examples
 
 Get a web-page:

data/Rakefile

@@ -1,4 +1,4 @@
-require 'rubygems'
+# frozen_string_literal: true
 
 begin
   require 'bundler'

data/bin/ronin-web

@@ -1,17 +1,16 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 require 'rubygems'
 
 root = File.expand_path(File.join(File.dirname(__FILE__),'..'))
 if File.directory?(File.join(root,'.git'))
   Dir.chdir(root) do
-    begin
-      require 'bundler/setup'
-    rescue LoadError => e
-      warn e.message
-      warn "Run `gem install bundler` to install Bundler"
-      exit -1
-    end
+    require 'bundler/setup'
+  rescue LoadError => e
+    warn e.message
+    warn "Run `gem install bundler` to install Bundler"
+    exit(-1)
   end
 end
 

data/data/new/webapp/Gemfile

@@ -2,5 +2,5 @@ source 'https://rubygems.org'
 
 gem 'puma', '~> 6.0'
 
-# gem 'ronin-support', '~> 1.0' #, github: 'ronin-rb/ronin-support'
-gem 'ronin-web-server', '~> 0.1' #, github: 'ronin-rb/ronin-web-server'
+# gem 'ronin-support', '~> 1.0' # , github: 'ronin-rb/ronin-support'
+gem 'ronin-web-server', '~> 0.1' # , github: 'ronin-rb/ronin-web-server'

data/data/new/webapp/config.ru

@@ -1,4 +1,2 @@
-#!/usr/bin/env rackup
-
 require './app'
 run App

data/gemspec.yml

@@ -41,7 +41,7 @@ dependencies:
   open_namespace: ~> 0.4
   # Ronin dependencies:
   ronin-support: ~> 1.0
-  ronin-web-server: ~> 0.1
+  ronin-web-server: ~> 0.1, >= 0.1.1
   ronin-web-spider: ~> 0.1
   ronin-web-user_agents: ~> 0.1
   ronin-core: ~> 0.1

data/lib/ronin/web/cli/command.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 #
 # ronin-web - A collection of useful web helper methods and commands.
 #
@@ -24,6 +25,9 @@ require 'ronin/core/cli/command'
 module Ronin
   module Web
     class CLI
+      #
+      # Base class for all `ronin-web` commands.
+      #
       class Command < Core::CLI::Command
 
         man_dir File.join(ROOT,'man')

data/lib/ronin/web/cli/commands/diff.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 #
 # ronin-web - A collection of useful web helper methods and commands.
 #
@@ -42,7 +43,7 @@ module Ronin
         # ## Options
         #
         #     -h, --help                       Print help information
-        # 
+        #
         class Diff < Command
 
           include CommandKit::Terminal

data/lib/ronin/web/cli/commands/html.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 #
 # ronin-web - A collection of useful web helper methods and commands.
 #
@@ -56,7 +57,7 @@ module Ronin
         class Html < Command
 
           usage '[options] {URL | FILE} [XPATH | CSS-path]'
-          
+
           option :xpath, short: '-X',
                          value: {type: String, usage: 'XPATH'},
                          desc:  'XPath query' do |xpath|
@@ -136,7 +137,7 @@ module Ronin
           def run(source,query=@query)
             unless query
               print_error "must specify --xpath, --css-path, or an XPath/CSS-path argument"
-              exit -1
+              exit(-1)
             end
 
             doc   = Nokogiri::HTML(read(source))

data/lib/ronin/web/cli/commands/irb.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 #
 # ronin-web - A collection of useful web helper methods and commands.
 #

data/lib/ronin/web/cli/commands/new/spider.rb

@@ -104,10 +104,9 @@ module Ronin
                               required: false
                             },
                             desc: 'Spiders a domain' do |value|
-                            @entry_point_method   = :domain
-                            @entry_point_argument = value
-                          end
-
+                              @entry_point_method   = :domain
+                              @entry_point_argument = value
+                            end
 
             option :site, equals: true,
                           value: {

data/lib/ronin/web/cli/commands/reverse_proxy.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 #
 # ronin-web - A collection of useful web helper methods and commands.
 #
@@ -181,19 +182,19 @@ module Ronin
           # @return [(Regexp, String), (String, String)]
           #
           def parse_rewrite_rule(value)
-            if (index = value.rindex('/:'))
+            if    (index = value.rindex('/:'))
               regexp  = Regexp.new(value[1...index])
-              replace = value[index+2..]
+              replace = value[(index + 2)..]
 
               return [regexp, replace]
             elsif (index = value.rindex(':'))
               string  = value[0...index]
-              replace = value[(index+1)..]
+              replace = value[(index + 1)..]
 
               return [string, replace]
             end
           end
-          
+
           #
           # Rewrites a request or response body.
           #

data/lib/ronin/web/cli/commands/server.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 #
 # ronin-web - A collection of useful web helper methods and commands.
 #
@@ -42,7 +43,7 @@ module Ronin
         #    -r, --root DIR                   Root directory to serve
         #    -R, --redirect /PATH:URL         Registers a 302 Found redirect at the given PATH
         #    -h, --help                       Print help information
-        # 
+        #
         class Server < Command
 
           include Core::CLI::Logging
@@ -57,7 +58,7 @@ module Ronin
                           default: 'localhost'
                         },
                         desc: 'Host name or IP to bind to' do |host|
-                          App.host = host
+                          App.bind = host
                         end
 
           option :port, short: '-p',
@@ -143,7 +144,7 @@ module Ronin
               end
             end
 
-            log_info "Starting web server listening on #{App.host}:#{App.port} ..."
+            log_info "Starting web server listening on #{App.bind}:#{App.port} ..."
             begin
               App.run!
             rescue Errno::EADDRINUSE => error

data/lib/ronin/web/cli/commands/spider.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 #
 # ronin-web - A collection of useful web helper methods and commands.
 #
@@ -119,18 +120,18 @@ module Ronin
                                 desc: 'Sets the read timeout'
 
           option :ssl_timeout, value: {
-                                  type: Integer,
-                                  usage: 'SECS',
-                                  default: Spidr.ssl_timeout
-                                },
-                                desc: 'Sets the SSL connection timeout'
+                                 type: Integer,
+                                 usage: 'SECS',
+                                 default: Spidr.ssl_timeout
+                               },
+                               desc: 'Sets the SSL connection timeout'
 
           option :continue_timeout, value: {
-                                  type: Integer,
-                                  usage: 'SECS',
-                                  default: Spidr.continue_timeout
-                                },
-                                desc: 'Sets the continue timeout'
+                                      type:    Integer,
+                                      usage:   'SECS',
+                                      default: Spidr.continue_timeout
+                                    },
+                                    desc: 'Sets the continue timeout'
 
           option :keep_alive_timeout, value: {
                                         type:    Integer,
@@ -168,14 +169,14 @@ module Ronin
                                end
 
           option :user_agent, value: {
-                                type: String,
+                                type:  String,
                                 usage: 'USER-AGENT'
                               },
                               desc: 'Sets the User-Agent string'
 
           option :user_agent_string, short: '-U',
                                      value: {
-                                       type: String,
+                                       type:  String,
                                        usage: 'STRING'
                                      },
                                      desc: 'The User-Agent string to use' do |ua|
@@ -184,11 +185,9 @@ module Ronin
 
           option :user_agent, short: '-u',
                               value: {
-                                type: Hash[
-                                  Support::Network::HTTP::UserAgents::ALIASES.keys.map { |key|
-                                    [key.to_s.tr('_','-'), key]
-                                  }
-                                ]
+                                type: Support::Network::HTTP::UserAgents::ALIASES.transform_keys { |key|
+                                  key.to_s.tr('_','-')
+                                }
                               },
                               desc: 'The User-Agent to use' do |name|
                                 @user_agent = name
@@ -196,34 +195,34 @@ module Ronin
 
           option :referer, short: '-R',
                            value: {
-                             type: String,
+                             type:  String,
                              usage: 'URL'
                            },
                            desc: 'Sets the Referer URL'
 
           option :delay, short: '-d',
                          value: {
-                           type: Numeric,
+                           type:  Numeric,
                            usage: 'SECS'
                          },
                          desc: 'Sets the delay in seconds between each request'
 
           option :limit, short: '-l',
                          value: {
-                           type: Integer,
+                           type:  Integer,
                            usage: 'COUNT'
                          },
                          desc: 'Only spiders up to COUNT pages'
 
           option :max_depth, short: '-d',
                              value: {
-                               type: Integer,
-                               usage: 'DEPTH',
+                               type:  Integer,
+                               usage: 'DEPTH'
                              },
                              desc: 'Only spiders up to max depth'
 
           option :enqueue, value: {
-                             type: String,
+                             type:  String,
                              usage: 'URL'
                            },
                            desc: 'Adds the URL to the queue' do |url|
@@ -231,7 +230,7 @@ module Ronin
                            end
 
           option :visited, value: {
-                             type: String,
+                             type:  String,
                              usage: 'URL'
                            },
                            desc: 'Marks the URL as previously visited' do |url|
@@ -243,7 +242,7 @@ module Ronin
           option :strip_query, desc: 'Enables/disables stripping the query component of every URL'
 
           option :visit_host, value: {
-                                type: String,
+                                type:  String,
                                 usage: 'HOST'
                               },
                               desc: 'Visit URLs with the matching host name' do |host|
@@ -267,15 +266,15 @@ module Ronin
                                end
 
           option :ignore_hosts_like, value: {
-                                      type:  Regexp,
-                                      usage: '/REGEX/'
-                                    },
-                                    desc: 'Ignore the host names matching the REGEX' do |regex|
-                                      @ignore_hosts << regex
-                                    end
+                                       type:  Regexp,
+                                       usage: '/REGEX/'
+                                     },
+                                     desc: 'Ignore the host names matching the REGEX' do |regex|
+                                       @ignore_hosts << regex
+                                     end
 
           option :visit_port, value: {
-                                type: Integer,
+                                type:  Integer,
                                 usage: 'PORT'
                               },
                               desc: 'Visit URLs with the matching port number' do |port|
@@ -299,15 +298,15 @@ module Ronin
                                end
 
           option :ignore_ports_like, value: {
-                                      type:  Regexp,
-                                      usage: '/REGEX/'
-                                    },
-                                    desc: 'Ignore the port numbers matching the REGEXP' do |regex|
-                                      @ignore_ports << regex
-                                    end
+                                       type:  Regexp,
+                                       usage: '/REGEX/'
+                                     },
+                                     desc: 'Ignore the port numbers matching the REGEXP' do |regex|
+                                       @ignore_ports << regex
+                                     end
 
           option :visit_link, value: {
-                                type: String,
+                                type:  String,
                                 usage: 'URL'
                               },
                               desc: 'Visit the URL' do |link|
@@ -331,36 +330,36 @@ module Ronin
                                end
 
           option :ignore_links_like, value: {
-                                      type:  Regexp,
-                                      usage: '/REGEX/'
-                                    },
-                                    desc: 'Ignore URLs matching the REGEX' do |regex|
-                                      @ignore_links << regex
-                                    end
+                                       type:  Regexp,
+                                       usage: '/REGEX/'
+                                     },
+                                     desc: 'Ignore URLs matching the REGEX' do |regex|
+                                       @ignore_links << regex
+                                     end
 
           option :visit_ext, value: {
-                                type: String,
-                                usage: 'FILE_EXT'
-                              },
-                              desc: 'Visit URLs with the matching file ext' do |ext|
-                                @visit_exts << ext
-                              end
+                               type:  String,
+                               usage: 'FILE_EXT'
+                             },
+                             desc: 'Visit URLs with the matching file ext' do |ext|
+                               @visit_exts << ext
+                             end
 
           option :visit_exts_like, value: {
-                                      type:  Regexp,
-                                      usage: '/REGEX/'
-                                    },
-                                    desc: 'Visit URLs with file exts that match the REGEX' do |regex|
-                                      @visit_exts << regex
-                                    end
+                                     type:  Regexp,
+                                     usage: '/REGEX/'
+                                   },
+                                   desc: 'Visit URLs with file exts that match the REGEX' do |regex|
+                                     @visit_exts << regex
+                                   end
 
           option :ignore_ext, value: {
-                                 type:  String,
-                                 usage: 'FILE_EXT'
-                               },
-                               desc: 'Ignore the URLs with the file ext' do |ext|
-                                 @ignore_exts << ext
-                               end
+                                type:  String,
+                                usage: 'FILE_EXT'
+                              },
+                              desc: 'Ignore the URLs with the file ext' do |ext|
+                                @ignore_exts << ext
+                              end
 
           option :ignore_exts_like, value: {
                                       type:  Regexp,
@@ -374,47 +373,47 @@ module Ronin
                           desc:  'Specifies whether to honor robots.txt'
 
           option :host, value: {
-                          type: String,
+                          type:  String,
                           usage: 'HOST'
                         },
                         desc: 'Spiders the specific HOST'
 
           option :domain, value: {
-                            type: String,
-                            usage: 'DOMAIN',
+                            type:  String,
+                            usage: 'DOMAIN'
                           },
                           desc: 'Spiders the whole domain'
 
           option :site, value: {
-                          type: String,
+                          type:  String,
                           usage: 'URL'
                         },
                         desc: 'Spiders the website, starting at the URL'
 
-          option :print_stauts, desc: 'Print the status codes for each URL'
+          option :print_status, desc: 'Print the status codes for each URL'
 
           option :print_headers, desc: 'Print response headers for each URL'
 
           option :print_header, value: {
-                                  type: String,
+                                  type:  String,
                                   usage: 'NAME'
                                 },
                                 desc: 'Prints a specific header'
 
           option :history, value: {
-                             type: String,
+                             type:  String,
                              usage: 'FILE'
                            },
                            desc: 'The history file'
 
           option :archive, value: {
-                             type: String,
+                             type:  String,
                              usage: 'DIR'
                            },
                            desc: 'Archive every visited page to the DIR'
 
           option :git_archive, value: {
-                                 type: String,
+                                 type:  String,
                                  usage: 'DIR'
                                },
                                desc: 'Archive every visited page to the git repository'
@@ -588,6 +587,8 @@ module Ronin
               end
             end
 
+            # post-spidering tasks
+
             if options[:git_archive]
               archive.commit "Updated #{Time.now}"
             end
@@ -699,6 +700,7 @@ module Ronin
           #
           def agent_kwargs
             kwargs = {}
+
             kwargs[:proxy] = options[:proxy] if options[:proxy]
 
             unless @default_headers.empty?

data/lib/ronin/web/cli/ruby_shell.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 #
 # ronin-web - A collection of useful web helper methods and commands.
 #

data/lib/ronin/web/cli.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 #
 # ronin-web - A collection of useful web helper methods and commands.
 #

data/lib/ronin/web/html.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 #
 # ronin-web - A collection of useful web helper methods and commands.
 #

data/lib/ronin/web/mechanize.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 #
 # ronin-web - A collection of useful web helper methods and commands.
 #
@@ -31,7 +32,9 @@ module Ronin
     class Mechanize < ::Mechanize
 
       #
-      # Creates a new [Mechanize](http://mechanize.rubyforge.org/) Agent.
+      # Creates a new [Mechanize] Agent.
+      #
+      # [Mechanize]: https://github.com/sparklemotion/mechanize#readme
       #
       # @param [Network::HTTP::Proxy, Hash, String] proxy
       #   Proxy information.
@@ -56,6 +59,8 @@ module Ronin
                      user_agent: Support::Network::HTTP.user_agent)
         super()
 
+        self.verify_mode = OpenSSL::SSL::VERIFY_NONE
+
         if proxy
           proxy = URI(proxy)
 

data/lib/ronin/web/root.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 #
 # ronin-web - A collection of useful web helper methods and commands.
 #

data/lib/ronin/web/version.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 #
 # ronin-web - A collection of useful web helper methods and commands.
 #
@@ -20,6 +21,6 @@
 module Ronin
   module Web
     # ronin-web Version
-    VERSION = '1.0.0'
+    VERSION = '1.0.2'
   end
 end

data/lib/ronin/web/xml.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 #
 # ronin-web - A collection of useful web helper methods and commands.
 #

data/lib/ronin/web.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 #
 # ronin-web - A collection of useful web helper methods and commands.
 #
@@ -34,6 +35,9 @@ require 'nokogiri/diff'
 require 'open_namespace'
 
 module Ronin
+  #
+  # Top-level web helper methods.
+  #
   module Web
     include OpenNamespace
 
@@ -200,6 +204,7 @@ module Ronin
                        referer:    nil,
                        cookie:     nil,
                        **kwargs)
+      uri     = URI(url)
       options = {proxy: proxy, **kwargs}
 
       if user_agent
@@ -214,7 +219,7 @@ module Ronin
       options['Referer'] = referer if referer
       options['Cookie']  = cookie  if cookie
 
-      return URI.open(url,options)
+      return uri.open(options)
     end
 
     #
@@ -344,7 +349,7 @@ module Ronin
     # @api public
     #
     def self.post(url,query={},headers={},&block)
-      agent.post(url,query,headers={},&block)
+      agent.post(url,query,headers,&block)
     end
 
     #

data/ronin-web.gemspec

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# frozen_string_literal: true
 
 require 'yaml'
 
@@ -21,10 +21,10 @@ Gem::Specification.new do |gem|
   gem.email       = gemspec['email']
   gem.homepage    = gemspec['homepage']
 
-  glob = lambda { |patterns| gem.files & Dir[*patterns] }
+  glob = ->(patterns) { gem.files & Dir[*patterns] }
 
-  gem.files = `git ls-files`.split($/)
-  gem.files = glob[gemspec['files']] if gemspec['files']
+  gem.files  = `git ls-files`.split($/)
+  gem.files  = glob[gemspec['files']] if gemspec['files']
   gem.files += Array(gemspec['generated_files'])
   # exclude test files from the packages gem
   gem.files -= glob[gemspec['test_files'] || 'spec/{**/}*']
@@ -32,6 +32,7 @@ Gem::Specification.new do |gem|
   gem.executables = gemspec.fetch('executables') do
     glob['bin/*'].map { |path| File.basename(path) }
   end
+
   gem.default_executable = gem.executables.first if Gem::VERSION < '1.7.'
 
   gem.extensions       = glob[gemspec['extensions'] || 'ext/**/extconf.rb']
@@ -46,7 +47,7 @@ Gem::Specification.new do |gem|
   gem.required_rubygems_version = gemspec['required_rubygems_version']
   gem.post_install_message      = gemspec['post_install_message']
 
-  split = lambda { |string| string.split(/,\s*/) }
+  split = ->(string) { string.split(/,\s*/) }
 
   if gemspec['dependencies']
     gemspec['dependencies'].each do |name,versions|

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ronin-web
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.2
 platform: ruby
 authors:
 - Postmodern
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-02-01 00:00:00.000000000 Z
+date: 2023-04-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -101,6 +101,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -108,6 +111,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.1
 - !ruby/object:Gem::Dependency
   name: ronin-web-spider
   requirement: !ruby/object:Gem::Requirement
@@ -181,6 +187,7 @@ files:
 - ".gitignore"
 - ".mailmap"
 - ".rspec"
+- ".rubocop.yml"
 - ".ruby-version"
 - ".yardopts"
 - COPYING.txt