ronin-web-spider 0.1.0.beta1

data/lib/ronin/web/spider.rb

@@ -0,0 +1,115 @@
+#
+# ronin-web-spider - A collection of common web spidering routines.
+#
+# Copyright (c) 2006-2022 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# ronin-web-spider is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# ronin-web-spider is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with ronin-web-spider.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+require 'ronin/web/spider/agent'
+require 'ronin/web/spider/version'
+
+module Ronin
+  module Web
+    module Spider
+      #
+      # Creates a new agent and begin spidering at the given URL.
+      #
+      # @param [URI::HTTP, String] url
+      #   The URL to start spidering at.
+      #
+      # @param [Hash{Symbol => Object}] kwargs
+      #   Additional keyword arguments. See {Agent#initialize}.
+      #
+      # @yield [agent]
+      #   If a block is given, it will be passed the newly created agent
+      #   before it begins spidering.
+      #
+      # @yieldparam [Agent] agent
+      #   The newly created agent.
+      #
+      # @see https://rubydoc.info/gems/spidr/Spidr/Agent#start_at-class_method
+      #
+      def self.start_at(url,**kwargs,&block)
+        Agent.start_at(url,**kwargs,&block)
+      end
+
+      #
+      # Creates a new agent and spiders the given host.
+      #
+      # @param [String] name
+      #   The host-name to spider.
+      #
+      # @param [Hash{Symbol => Object}] kwargs
+      #   Additional keyword arguments. See {Agent#initialize}.
+      #
+      # @yield [agent]
+      #   If a block is given, it will be passed the newly created agent
+      #   before it begins spidering.
+      #
+      # @yieldparam [Agent] agent
+      #   The newly created agent.
+      #
+      # @see https://rubydoc.info/gems/spidr/Spidr/Agent#host-class_method
+      #
+      def self.host(name,**kwargs,&block)
+        Agent.host(name,**kwargs,&block)
+      end
+
+      #
+      # Creates a new agent and spiders the web-site located at the given URL.
+      #
+      # @param [URI::HTTP, String] url
+      #   The web-site to spider.
+      #
+      # @param [Hash{Symbol => Object}] kwargs
+      #   Additional keyword arguments. See {Agent#initialize}.
+      #
+      # @yield [agent]
+      #   If a block is given, it will be passed the newly created agent
+      #   before it begins spidering.
+      #
+      # @yieldparam [Agent] agent
+      #   The newly created agent.
+      #
+      # @see https://rubydoc.info/gems/spidr/Spidr/Agent#site-class_method
+      #
+      def self.site(url,**kwargs,&block)
+        Agent.site(url,**kwargs,&block)
+      end
+
+      #
+      # Creates a new agent and spiders the entire domain.
+      #
+      # @param [String] name
+      #   The top-level domain to spider.
+      #
+      # @param [Hash{Symbol => Object}] kwargs
+      #   Additional keyword arguments. See {Agent#initialize}.
+      #
+      # @yield [agent]
+      #   If a block is given, it will be passed the newly created agent
+      #   before it begins spidering.
+      #
+      # @yieldparam [Agent] agent
+      #   The newly created agent.
+      #
+      # @see https://rubydoc.info/gems/spidr/Spidr/Agent#domain-class_method
+      #
+      def self.domain(name,**kwargs,&block)
+        Agent.domain(name,**kwargs,&block)
+      end
+    end
+  end
+end

data/ronin-web-spider.gemspec

@@ -0,0 +1,61 @@
+# encoding: utf-8
+
+require 'yaml'
+
+Gem::Specification.new do |gem|
+  gemspec = YAML.load_file('gemspec.yml')
+
+  gem.name    = gemspec.fetch('name')
+  gem.version = gemspec.fetch('version') do
+                  lib_dir = File.join(File.dirname(__FILE__),'lib')
+                  $LOAD_PATH << lib_dir unless $LOAD_PATH.include?(lib_dir)
+
+                  require 'ronin/web/spider/version'
+                  Ronin::Web::Spider::VERSION
+                end
+
+  gem.summary     = gemspec['summary']
+  gem.description = gemspec['description']
+  gem.licenses    = Array(gemspec['license'])
+  gem.authors     = Array(gemspec['authors'])
+  gem.email       = gemspec['email']
+  gem.homepage    = gemspec['homepage']
+  gem.metadata    = gemspec['metadata'] if gemspec['metadata']
+
+  glob = lambda { |patterns| gem.files & Dir[*patterns] }
+
+  gem.files  = `git ls-files`.split($/)
+  gem.files  = glob[gemspec['files']] if gemspec['files']
+  gem.files += Array(gemspec['generated_files'])
+
+  gem.executables = gemspec.fetch('executables') do
+    glob['bin/*'].map { |path| File.basename(path) }
+  end
+
+  gem.extensions       = glob[gemspec['extensions'] || 'ext/**/extconf.rb']
+  gem.test_files       = glob[gemspec['test_files'] || 'spec/{**/}*_spec.rb']
+  gem.extra_rdoc_files = glob[gemspec['extra_doc_files'] || '*.{txt,md}']
+
+  gem.require_paths = Array(gemspec.fetch('require_paths') {
+    %w[ext lib].select { |dir| File.directory?(dir) }
+  })
+
+  gem.requirements              = gemspec['requirements']
+  gem.required_ruby_version     = gemspec['required_ruby_version']
+  gem.required_rubygems_version = gemspec['required_rubygems_version']
+  gem.post_install_message      = gemspec['post_install_message']
+
+  split = lambda { |string| string.split(/,\s*/) }
+
+  if gemspec['dependencies']
+    gemspec['dependencies'].each do |name,versions|
+      gem.add_dependency(name,split[versions])
+    end
+  end
+
+  if gemspec['development_dependencies']
+    gemspec['development_dependencies'].each do |name,versions|
+      gem.add_development_dependency(name,split[versions])
+    end
+  end
+end