ronin-web 1.0.2 → 2.0.0

data/lib/ronin/web/cli/spider_options.rb

@@ -0,0 +1,919 @@
+# frozen_string_literal: true
+#
+# ronin-web - A collection of useful web helper methods and commands.
+#
+# Copyright (c) 2006-2024 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# ronin-web is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# ronin-web is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with ronin-web.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+require 'ronin/web/spider'
+require 'ronin/support/network/http/user_agents'
+
+module Ronin
+  module Web
+    class CLI
+      #
+      # Adds options for spidering a website.
+      #
+      # @since 2.0.0
+      #
+      module SpiderOptions
+        #
+        # Adds options for configuring a web spider and spidering a website.
+        #
+        # @param [Class<Command>] command
+        #   The command class including {SpiderOptions}.
+        #
+        def self.included(command)
+          command.usage '[options] {--host HOST | --domain DOMAIN | --site URL}'
+
+          command.option :host, value: {
+                                  type:  String,
+                                  usage: 'HOST'
+                                },
+                                desc: 'Spiders the specific HOST'
+
+          command.option :domain, value: {
+                                    type:  String,
+                                    usage: 'DOMAIN'
+                                  },
+                                  desc: 'Spiders the whole domain'
+
+          command.option :site, value: {
+                                  type:  String,
+                                  usage: 'URL'
+                                },
+                                desc: 'Spiders the website, starting at the URL'
+
+          command.option :open_timeout, value: {
+                                          type: Integer,
+                                          usage: 'SECS',
+                                          default: Spidr.open_timeout
+                                        },
+                                        desc: 'Sets the connection open timeout' do |timeout|
+                                          self.open_timeout = timeout
+                                        end
+
+          command.option :read_timeout, value: {
+                                          type: Integer,
+                                          usage: 'SECS',
+                                          default: Spidr.read_timeout
+                                        },
+                                        desc: 'Sets the read timeout' do |timeout|
+                                          self.read_timeout = timeout
+                                        end
+
+          command.option :ssl_timeout, value: {
+                                         type: Integer,
+                                         usage: 'SECS',
+                                         default: Spidr.ssl_timeout
+                                       },
+                                       desc: 'Sets the SSL connection timeout' do |timeout|
+                                         self.ssl_timeout = timeout
+                                       end
+
+          command.option :continue_timeout, value: {
+                                              type:    Integer,
+                                              usage:   'SECS',
+                                              default: Spidr.continue_timeout
+                                            },
+                                            desc: 'Sets the continue timeout' do |timeout|
+                                              self.continue_timeout = timeout
+                                            end
+
+          command.option :keep_alive_timeout, value: {
+                                                type:    Integer,
+                                                usage:   'SECS',
+                                                default: Spidr.keep_alive_timeout
+                                              },
+                                              desc: 'Sets the connection keep alive timeout' do |timeout|
+                                                self.keep_alive_timeout = timeout
+                                              end
+
+          command.option :proxy, short: '-P',
+                                 value: {
+                                   type:  String,
+                                   usage: 'PROXY'
+                                 },
+                                 desc: 'Sets the proxy to use' do |proxy|
+                                   self.proxy = proxy
+                                 end
+
+          command.option :header, short: '-H',
+                                  value: {
+                                    type:  /\A[^\s:]+:.*\z/,
+                                    usage: 'NAME: VALUE'
+                                  },
+                                  desc: 'Sets a default header' do |header|
+                                    name, value = header.split(/:\s*/,2)
+
+                                    self.default_headers[name] = value
+                                  end
+
+          command.option :host_header, value: {
+                                         type: /\A[^\s=]+=[^\s=]+\z/,
+                                         usage: 'NAME=VALUE'
+                                       },
+                                       desc: 'Sets a default header' do |name_value|
+                                         name, value = name_value.split('=',2)
+
+                                         self.host_headers[name] = value
+                                       end
+
+          command.option :user_agent_string, short: '-U',
+                                             value: {
+                                               type:  String,
+                                               usage: 'STRING'
+                                             },
+                                             desc: 'The User-Agent string to use' do |ua|
+                                               self.user_agent = ua
+                                             end
+
+          command.option :user_agent, short: '-u',
+                                      value: {
+                                        type: Support::Network::HTTP::UserAgents::ALIASES.transform_keys { |key|
+                                          key.to_s.tr('_','-')
+                                        }
+                                      },
+                                      desc: 'The User-Agent to use' do |name|
+                                        self.user_agent = name
+                                      end
+
+          command.option :referer, short: '-R',
+                                   value: {
+                                     type:  String,
+                                     usage: 'URL'
+                                   },
+                                   desc: 'Sets the Referer URL' do |referer|
+                                     self.referer = referer
+                                   end
+
+          command.option :delay, short: '-d',
+                                 value: {
+                                   type:  Numeric,
+                                   usage: 'SECS'
+                                 },
+                                 desc: 'Sets the delay in seconds between each request' do |delay|
+                                   self.delay = delay
+                                 end
+
+          command.option :limit, short: '-l',
+                                 value: {
+                                   type:  Integer,
+                                   usage: 'COUNT'
+                                 },
+                                 desc: 'Only spiders up to COUNT pages' do |limit|
+                                   self.limit = limit
+                                 end
+
+          command.option :max_depth, short: '-d',
+                                     value: {
+                                       type:  Integer,
+                                       usage: 'DEPTH'
+                                     },
+                                     desc: 'Only spiders up to max depth' do |depth|
+                                       self.max_depth = depth
+                                     end
+
+          command.option :enqueue, value: {
+                                     type:  String,
+                                     usage: 'URL'
+                                   },
+                                   desc: 'Adds the URL to the queue' do |url|
+                                     self.queue << url
+                                   end
+
+          command.option :visited, value: {
+                                     type:  String,
+                                     usage: 'URL'
+                                   },
+                                   desc: 'Marks the URL as previously visited' do |url|
+                                     self.history << url
+                                   end
+
+          command.option :strip_fragments, desc: 'Enables/disables stripping the fragment component of every URL' do
+            self.strip_fragments = true
+          end
+
+          command.option :strip_query, desc: 'Enables/disables stripping the query component of every URL' do
+            self.strip_query = true
+          end
+
+          command.option :visit_scheme, value: {
+                                          type:  String,
+                                          usage: 'SCHEME'
+                                        },
+                                        desc: 'Visit URLs with the URI scheme' do |scheme|
+                                          self.visit_schemes << scheme
+                                        end
+
+          command.option :visit_schemes_like, value: {
+                                                type:  Regexp,
+                                                usage: '/REGEX/'
+                                              },
+                                              desc: 'Visit URLs with URI schemes that match the REGEX' do |regex|
+                                                self.visit_schemes << regex
+                                              end
+
+          command.option :ignore_scheme, value: {
+                                           type:  String,
+                                           usage: 'SCHEME'
+                                         },
+                                         desc: 'Ignore the URLs with the URI scheme' do |scheme|
+                                           self.ignore_schemes << scheme
+                                         end
+
+          command.option :ignore_schemes_like, value: {
+                                                 type:  Regexp,
+                                                 usage: '/REGEX/'
+                                               },
+                                               desc: 'Ignore the URLs with URI schemes matching the REGEX' do |regex|
+                                                 self.ignore_schemes << regex
+                                               end
+
+          command.option :visit_host, value: {
+                                        type:  String,
+                                        usage: 'HOST'
+                                      },
+                                      desc: 'Visit URLs with the matching host name' do |host|
+                                        self.visit_hosts << host
+                                      end
+
+          command.option :visit_hosts_like, value: {
+                                              type:  Regexp,
+                                              usage: '/REGEX/'
+                                            },
+                                            desc: 'Visit URLs with hostnames that match the REGEX' do |regex|
+                                              self.visit_hosts << regex
+                                            end
+
+          command.option :ignore_host, value: {
+                                         type:  String,
+                                         usage: 'HOST'
+                                       },
+                                       desc: 'Ignore the host name' do |host|
+                                         self.ignore_hosts << host
+                                       end
+
+          command.option :ignore_hosts_like, value: {
+                                               type:  Regexp,
+                                               usage: '/REGEX/'
+                                             },
+                                             desc: 'Ignore the host names matching the REGEX' do |regex|
+                                               self.ignore_hosts << regex
+                                             end
+
+          command.option :visit_port, value: {
+                                        type:  Integer,
+                                        usage: 'PORT'
+                                      },
+                                      desc: 'Visit URLs with the matching port number' do |port|
+                                        self.visit_ports << port
+                                      end
+
+          command.option :visit_ports_like, value: {
+                                              type:  Regexp,
+                                              usage: '/REGEX/'
+                                            },
+                                            desc: 'Visit URLs with port numbers that match the REGEX' do |regex|
+                                              self.visit_ports << regex
+                                            end
+
+          command.option :ignore_port, value: {
+                                         type:  Integer,
+                                         usage: 'PORT'
+                                       },
+                                       desc: 'Ignore the port number' do |port|
+                                         self.ignore_ports << port
+                                       end
+
+          command.option :ignore_ports_like, value: {
+                                               type:  Regexp,
+                                               usage: '/REGEX/'
+                                             },
+                                             desc: 'Ignore the port numbers matching the REGEXP' do |regex|
+                                               self.ignore_ports << regex
+                                             end
+
+          command.option :visit_link, value: {
+                                        type:  String,
+                                        usage: 'URL'
+                                      },
+                                      desc: 'Visit the URL' do |link|
+                                        self.visit_links << link
+                                      end
+
+          command.option :visit_links_like, value: {
+                                              type:  Regexp,
+                                              usage: '/REGEX/'
+                                            },
+                                            desc: 'Visit URLs that match the REGEX' do |regex|
+                                              self.visit_links << regex
+                                            end
+
+          command.option :ignore_link, value: {
+                                         type:  String,
+                                         usage: 'URL'
+                                       },
+                                       desc: 'Ignore the URL' do |link|
+                                         self.ignore_links << link
+                                       end
+
+          command.option :ignore_links_like, value: {
+                                               type:  Regexp,
+                                               usage: '/REGEX/'
+                                             },
+                                             desc: 'Ignore URLs matching the REGEX' do |regex|
+                                               self.ignore_links << regex
+                                             end
+
+          command.option :visit_ext, value: {
+                                       type:  String,
+                                       usage: 'FILE_EXT'
+                                     },
+                                     desc: 'Visit URLs with the matching file ext' do |ext|
+                                       self.visit_exts << ext
+                                     end
+
+          command.option :visit_exts_like, value: {
+                                             type:  Regexp,
+                                             usage: '/REGEX/'
+                                           },
+                                           desc: 'Visit URLs with file exts that match the REGEX' do |regex|
+                                             self.visit_exts << regex
+                                           end
+
+          command.option :ignore_ext, value: {
+                                        type:  String,
+                                        usage: 'FILE_EXT'
+                                      },
+                                      desc: 'Ignore the URLs with the file ext' do |ext|
+                                        self.ignore_exts << ext
+                                      end
+
+          command.option :ignore_exts_like, value: {
+                                              type:  Regexp,
+                                              usage: '/REGEX/'
+                                            },
+                                            desc: 'Ignore URLs with file exts matching the REGEX' do |regex|
+                                              self.ignore_exts << regex
+                                            end
+
+          command.option :robots, short: '-r',
+                                  desc:  'Specifies whether to honor robots.txt' do
+                                    self.robots = true
+                                  end
+        end
+
+        # Keyword arguments to initialize a new `Spidr::Agent`.
+        #
+        # @return [Hash{Symbol => Object}]
+        #
+        # @since 2.0.0
+        attr_reader :agent_kwargs
+
+        #
+        # Initializes the command.
+        #
+        # @param [Hash{Symbol => Object}] kwargs
+        #   Additional keyword arguments.
+        #
+        def initialize(**kwargs)
+          super(**kwargs)
+
+          @agent_kwargs = {}
+        end
+
+        #
+        # Creates a new web spider agent.
+        #
+        # @yield [agent]
+        #   The given block will be given the newly created and configured
+        #   web spider agent.
+        #
+        # @yieldparam [Ronin::Web::Spider::Agent] agent
+        #   The newly created web spider agent.
+        #
+        # @return [Ronin::Web::Spider::Agent]
+        #   The newly created web spider agent, after the agent has completed
+        #   it's spidering.
+        #
+        def new_agent(&block)
+          if options[:host]
+            Web::Spider.host(options[:host],**agent_kwargs,&block)
+          elsif options[:domain]
+            Web::Spider.domain(options[:domain],**agent_kwargs,&block)
+          elsif options[:site]
+            Web::Spider.site(options[:site],**agent_kwargs,&block)
+          else
+            print_error "must specify --host, --domain, or --site"
+            exit(-1)
+          end
+        end
+
+        #
+        # The open connection timeout.
+        #
+        # @return [Integer, nil]
+        #
+        # @since 2.0.0
+        #
+        def open_timeout
+          @agent_kwargs[:open_timeout]
+        end
+
+        #
+        # Sets the open connection timeout.
+        #
+        # @param [Integer] new_timeout
+        #
+        # @return [Integer]
+        #
+        # @since 2.0.0
+        #
+        def open_timeout=(new_timeout)
+          @agent_kwargs[:open_timeout] = new_timeout
+        end
+
+        #
+        # The read timeout.
+        #
+        # @return [Integer, nil]
+        #
+        # @since 2.0.0
+        #
+        def read_timeout
+          @agent_kwargs[:read_timeout]
+        end
+
+        #
+        # Sets the read timeout.
+        #
+        # @param [Integer] new_timeout
+        #
+        # @return [Integer]
+        #
+        # @since 2.0.0
+        #
+        def read_timeout=(new_timeout)
+          @agent_kwargs[:read_timeout] = new_timeout
+        end
+
+        #
+        # The SSL timeout.
+        #
+        # @return [Integer, nil]
+        #
+        # @since 2.0.0
+        #
+        def ssl_timeout
+          @agent_kwargs[:ssl_timeout]
+        end
+
+        #
+        # Sets the SSL timeout.
+        #
+        # @param [Integer] new_timeout
+        #
+        # @return [Integer]
+        #
+        # @since 2.0.0
+        #
+        def ssl_timeout=(new_timeout)
+          @agent_kwargs[:ssl_timeout] = new_timeout
+        end
+
+        #
+        # The continue timeout.
+        #
+        # @return [Integer, nil]
+        #
+        # @since 2.0.0
+        #
+        def continue_timeout
+          @agent_kwargs[:continue_timeout]
+        end
+
+        #
+        # Sets the continue timeout.
+        #
+        # @param [Integer] new_timeout
+        #
+        # @return [Integer]
+        #
+        # @since 2.0.0
+        #
+        def continue_timeout=(new_timeout)
+          @agent_kwargs[:continue_timeout] = new_timeout
+        end
+
+        #
+        # The `Keep-Alive` timeout.
+        #
+        # @return [Integer, nil]
+        #
+        # @since 2.0.0
+        #
+        def keep_alive_timeout
+          @agent_kwargs[:keep_alive_timeout]
+        end
+
+        #
+        # Sets the `Keep-Alive` timeout.
+        #
+        # @param [Integer] new_timeout
+        #
+        # @return [Integer]
+        #
+        # @since 2.0.0
+        #
+        def keep_alive_timeout=(new_timeout)
+          @agent_kwargs[:keep_alive_timeout] = new_timeout
+        end
+
+        #
+        # The proxy to use for spidering.
+        #
+        # @return [String, nil]
+        #
+        # @since 0.2.0
+        #
+        def proxy
+          @agent_kwargs[:proxy]
+        end
+
+        #
+        # Sets the proxy to use for spidering.
+        #
+        # @param [String] new_proxy
+        #   The new proxy URI.
+        #
+        # @return [String]
+        #
+        # @since 2.0.0
+        #
+        def proxy=(new_proxy)
+          @agent_kwargs[:proxy] = new_proxy
+        end
+
+        #
+        # The default headers to send with every request.
+        #
+        # @return [Hash{String => String}]
+        #
+        # @since 2.0.0
+        #
+        def default_headers
+          @agent_kwargs[:default_headers] ||= {}
+        end
+
+        #
+        # The default `Host` headers to send with every request.
+        #
+        # @return [Hash{String => String}]
+        #
+        # @since 2.0.0
+        #
+        def host_headers
+          @agent_kwargs[:host_headers] ||= {}
+        end
+
+        #
+        # Sets the new `User-Agent` header to use for spidering.
+        #
+        # @return [String, nil]
+        #
+        # @since 2.0.0
+        #
+        def user_agent
+          @agent_kwargs[:user_agent]
+        end
+
+        #
+        # Sets the new `User-Agent` header to use for spidering.
+        #
+        # @param [String] new_user_agent
+        #
+        # @return [String]
+        #
+        # @since 2.0.0
+        #
+        def user_agent=(new_user_agent)
+          @agent_kwargs[:user_agent] = new_user_agent
+        end
+
+        #
+        # The `Referer` header to use for spidering.
+        #
+        # @return [String, nil]
+        #
+        # @since 2.0.0
+        #
+        def referer
+          @agent_kwargs[:referer]
+        end
+
+        #
+        # Sets the `Referer` header to use for spidering.
+        #
+        # @param [String] new_referer
+        #
+        # @return [String, nil]
+        #
+        # @since 2.0.0
+        #
+        def referer=(new_referer)
+          @agent_kwargs[:referer] = new_referer
+        end
+
+        #
+        # The amount of seconds to pause between each request.
+        #
+        # @return [Integer, Float, nil]
+        #
+        # @since 2.0.0
+        #
+        def delay
+          @agent_kwargs[:delay]
+        end
+
+        #
+        # Sets the amount of seconds to pause between each request.
+        #
+        # @param [Integer, Float] new_delay
+        #
+        # @return [Integer, Float]
+        #
+        # @since 2.0.0
+        #
+        def delay=(new_delay)
+          @agent_kwargs[:delay] = new_delay
+        end
+
+        #
+        # The limit to how many URLs to visit.
+        #
+        # @return [Integer, nil]
+        #
+        # @since 2.0.0
+        #
+        def limit
+          @agent_kwargs[:limit]
+        end
+
+        #
+        # Sets the limit of how many URLs to visit.
+        #
+        # @param [Integer] new_limit
+        #
+        # @return [Integer]
+        #
+        # @since 2.0.0
+        #
+        def limit=(new_limit)
+          @agent_kwargs[:limit] = new_limit
+        end
+
+        #
+        # The maximum depth to spider.
+        #
+        # @return [Integer, nil]
+        #
+        # @since 2.0.0
+        #
+        def max_depth
+          @agent_kwargs[:max_depth]
+        end
+
+        #
+        # Sets the maximum depth to spider.
+        #
+        # @param [Integer] new_max_depth
+        #
+        # @return [Integer]
+        #
+        # @since 2.0.0
+        #
+        def max_depth=(new_max_depth)
+          @agent_kwargs[:max_depth] = new_max_depth
+        end
+
+        #
+        # The pre-existing queue of URLs to start spidering.
+        #
+        # @return [Array<String>]
+        #
+        # @since 2.0.0
+        #
+        def queue
+          @agent_kwargs[:queue] ||= []
+        end
+
+        #
+        # The pre-existing history of URLs that have already been spidered.
+        #
+        # @return [Array<String>]
+        #
+        # @since 2.0.0
+        #
+        def history
+          @agent_kwargs[:history] ||= []
+        end
+
+        #
+        # Whether to strip the `#fragment` components of links.
+        #
+        # @return [Boolean]
+        #
+        # @since 2.0.0
+        #
+        def strip_fragments
+          @agent_kwargs[:strip_fragments]
+        end
+
+        #
+        # Sets whether to strip the `#fragment` components of links.
+        #
+        # @param [Boolean] new_value
+        #
+        # @return [Boolean]
+        #
+        # @since 2.0.0
+        #
+        def strip_fragments=(new_value)
+          @agent_kwargs[:strip_fragments] = new_value
+        end
+
+        #
+        # Whether to strip the `?query` components of links.
+        #
+        # @return [Boolean]
+        #
+        # @since 2.0.0
+        #
+        def strip_query
+          @agent_kwargs[:strip_query]
+        end
+
+        #
+        # Sets whether to strip the `?query` components of links.
+        #
+        # @param [Boolean] new_value
+        #
+        # @return [Boolean]
+        #
+        # @since 2.0.0
+        #
+        def strip_query=(new_value)
+          @agent_kwargs[:strip_query] = new_value
+        end
+
+        #
+        # The list of URI schemes to allow spidering.
+        #
+        # @return [Array<String>]
+        #
+        # @since 2.0.0
+        #
+        def visit_schemes
+          @agent_kwargs[:schemes] ||= []
+        end
+
+        #
+        # The list of URI hosts to allow spidering.
+        #
+        # @return [Array<String>]
+        #
+        # @since 2.0.0
+        #
+        def visit_hosts
+          @agent_kwargs[:hosts] ||= []
+        end
+
+        #
+        # The list of URI ports to allow spidering.
+        #
+        # @return [Array<Integer>]
+        #
+        # @since 2.0.0
+        #
+        def visit_ports
+          @agent_kwargs[:ports] ||= []
+        end
+
+        #
+        # The list of URI links to allow spidering.
+        #
+        # @return [Array<String>]
+        #
+        # @since 2.0.0
+        #
+        def visit_links
+          @agent_kwargs[:links] ||= []
+        end
+
+        #
+        # The list of URI file extensions to allow spidering.
+        #
+        # @return [Array<String>]
+        #
+        # @since 2.0.0
+        #
+        def visit_exts
+          @agent_kwargs[:exts] ||= []
+        end
+
+        #
+        # The list of URI schemes to ignore while spidering.
+        #
+        # @return [Array<String>]
+        #
+        # @since 2.0.0
+        #
+        def ignore_schemes
+          @agent_kwargs[:ignore_schemes] ||= []
+        end
+
+        #
+        # The list of URI hosts to ignore while spidering.
+        #
+        # @return [Array<String>]
+        #
+        # @since 2.0.0
+        #
+        def ignore_hosts
+          @agent_kwargs[:ignore_hosts] ||= []
+        end
+
+        #
+        # The list of URI ports to ignore while spidering.
+        #
+        # @return [Array<Integer>]
+        #
+        # @since 2.0.0
+        #
+        def ignore_ports
+          @agent_kwargs[:ignore_ports] ||= []
+        end
+
+        #
+        # The list of URI links to ignore while spidering.
+        #
+        # @return [Array<String>]
+        #
+        # @since 2.0.0
+        #
+        def ignore_links
+          @agent_kwargs[:ignore_links] ||= []
+        end
+
+        #
+        # The list of URI file extensions to ignore while spidering.
+        #
+        # @return [Array<String>]
+        #
+        # @since 2.0.0
+        #
+        def ignore_exts
+          @agent_kwargs[:ignore_exts] ||= []
+        end
+
+        #
+        # Whether to honor the `robots.txt` file while spidering.
+        #
+        # @return [Boolean]
+        #
+        # @since 2.0.0
+        #
+        def robots
+          @agent_kwargs[:robots]
+        end
+
+        #
+        # Sets whether to honor the `robots.txt` file while spidering.
+        #
+        # @param [Boolean] new_value
+        #
+        # @return [Boolean]
+        #
+        # @since 2.0.0
+        #
+        def robots=(new_value)
+          @agent_kwargs[:robots] = new_value
+        end
+      end
+    end
+  end
+end