ronin-web 0.3.0.pre2 → 0.3.0.rc1

data/lib/ronin/web/spider.rb

@@ -85,6 +85,9 @@ module Ronin
       # @option options [Array<String, Regexp, Proc>] :ignore_exts
       #   The patterns which match the URI path extensions to not visit.
       #
+      # @option options [Boolean] :verbose (true)
+      #   Specifies whether every URL will be printed.
+      #
       # @yield [spider]
       #   If a block is given, it will be passed the newly created spider.
       #
@@ -103,8 +106,10 @@ module Ronin
 
         super(options)
 
-        every_url do |url|
-          print_info("Spidering #{url}")
+        if options.fetch(:verbose,true)
+          every_url do |url|
+            print_info("Spidering #{url}")
+          end
         end
 
         yield self if block_given?

data/lib/ronin/web/user_agents.rb

@@ -0,0 +1,196 @@
+#
+# Ronin Web - A Ruby library for Ronin that provides support for web
+# scraping and spidering functionality.
+#
+# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# This file is part of Ronin Web.
+#
+# Ronin is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ronin is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ronin.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+require 'ronin/web/config'
+
+require 'set'
+
+module Ronin
+  module Web
+    #
+    # Represents the set of `User-Agent` strings loaded from all
+    # `data/ronin/web/user_agents.yml` files.
+    #
+    # ## ronin/web/user_agents.yml
+    #
+    # The `user_agent.yml` files are essentially YAML files listing
+    # `User-Agent` strings grouped by category:
+    #
+    #     ---
+    #     :googlebot:
+    #       - "Googlebot/2.1 ( http://www.googlebot.com/bot.html)"
+    #       - "Googlebot-Image/1.0 ( http://www.googlebot.com/bot.html)"
+    #       - "Mediapartners-Google/2.1"
+    #       - "Google-Sitemaps/1.0"
+    #
+    # These files can be added to Ronin Repositories or to Ronin libraries,
+    # and will be loaded by the {UserAgents} objects. 
+    #
+    # @since 0.3.0
+    #
+    class UserAgents
+
+      include Enumerable
+      
+      # Relative path to the User-Agents file.
+      FILE = File.join('ronin','web','user_agents.yml')
+
+      #
+      # Creates a new User-Agent set.
+      #
+      # @api semipublic
+      #
+      def initialize
+        @files = Set[]
+        @user_agents = Hash.new { |hash,key| hash[key] = Set[] }
+      end
+
+      # 
+      # The categories of `User-Agent` strings.
+      #
+      # @return [Array<Symbol>]
+      #   The names of the categories.
+      #
+      # @api public
+      #
+      def categories
+        reload!
+
+        @user_agents.keys
+      end
+
+      #
+      # Iterates over each User-Agent in the set.
+      #
+      # @yield [ua]
+      #   The given block will be passed each User-Agent.
+      #
+      # @yieldparam [String] ua
+      #   A User-Agent string within the set.
+      #
+      # @return [Enumerator]
+      #   If no block is given, an Enmerator will be returned.
+      #
+      # @api public
+      #
+      def each(&block)
+        return enum_for(:each) unless block_given?
+
+        @user_agents.each do |name,strings|
+          strings.each(&block)
+        end
+      end
+
+      #
+      # Selects a `User-Agent` string from the set.
+      #
+      # @param [Symbol, String, Regexp] key
+      #   The User-Agents group name, sub-string or Regexp to search for.
+      #
+      # @return [String, nil]
+      #   The matching `User-Agent` string.
+      #
+      # @api public
+      #
+      def [](key)
+        reload!
+
+        case key
+        when Symbol
+          if @user_agents.has_key?(key)
+            strings = @user_agents[key]
+            return strings.entries[rand(strings.length)]
+          end
+        when String
+          @user_agents.each do |name,strings|
+            strings.each do |string|
+              return string if string.include?(key)
+            end
+          end
+
+          return nil
+        when Regexp
+          @user_agents.each do |name,strings|
+            strings.each do |string|
+              return string if string =~ key
+            end
+          end
+
+          return nil
+        else
+          raise(TypeError,"key must be a Symbol, String or Regexp")
+        end
+      end
+
+      #
+      # Fetches a `User-Agent` string from the set.
+      #
+      # @param [Symbol, String, Regexp] key
+      #   The User-Agents group name, sub-string or Regexp to search for.
+      #
+      # @param [String] default
+      #   The `User-Agent` string to default to if no match is found.
+      #
+      # @return [String]
+      #   The matching `User-Agent` string.
+      #
+      # @raise [ArgumentError]
+      #   No matching `User-Agent` string was found, and no default value
+      #   was given.
+      #
+      # @api public
+      #
+      def fetch(key,default=nil)
+        unless (string = (self[key] || default))
+          raise(ArgumentError,"no User-Agent strings match #{key.inspect}")
+        end
+
+        return string
+      end
+
+      protected
+
+      #
+      # Reloads the set of User-Agents.
+      #
+      # @api private
+      #
+      def reload!
+        Config.each_data_file(FILE) do |path|
+          next if @files.include?(path)
+
+          data = YAML.load_file(path)
+
+          unless data.kind_of?(Hash)
+            warn "#{path.dump} did not contain a Hash"
+            next
+          end
+
+          data.each do |name,strings|
+            @user_agents[name.to_sym].merge(strings)
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/ronin/web/version.rb

@@ -23,6 +23,6 @@
 module Ronin
   module Web
     # Ronin Web Version
-    VERSION = '0.3.0.pre2'
+    VERSION = '0.3.0.rc1'
   end
 end

data/lib/ronin/web/web.rb

@@ -20,12 +20,13 @@
 # along with Ronin.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+require 'ronin/web/user_agents'
+require 'ronin/web/mechanize'
 require 'ronin/network/http/proxy'
 require 'ronin/network/http/http'
 
 require 'uri/http'
 require 'nokogiri'
-require 'mechanize'
 require 'open-uri'
 
 module Ronin
@@ -168,12 +169,29 @@ module Ronin
       @proxy = Network::HTTP::Proxy.create(new_proxy)
     end
 
+    #
+    # A set of common `User-Agent` strings.
+    #
+    # @return [UserAgents]
+    #   The set of `User-Agent` strings.
+    #
+    # @since 0.3.0
+    #
+    # @api public
+    #
+    def Web.user_agents
+      @user_agents ||= UserAgents.new
+    end
+
     #
     # @return [Array]
     #   The supported Web User-Agent Aliases.
     #
     # @see http://rubydoc.info/gems/mechanize/Mechanize#AGENT_ALIASES-constant
     #
+    # @deprecated
+    #   Will be replaced by {user_agents} in 1.0.0.
+    #
     # @api public
     #
     def Web.user_agent_aliases
@@ -195,18 +213,30 @@ module Ronin
     end
 
     #
-    # Sets the User-Agent string used by {Web}.
+    # Sets the `User-Agent` string used by {Web}.
     #
-    # @param [String] new_agent
+    # @param [String, Symbol, Regexp, nil] value
     #   The User-Agent string to use.
+    #   Setting {user_agent} to `nil` will disable the `User-Agent` string.
     #
     # @return [String]
     #   The new User-Agent string.
     #
+    # @raise [RuntimeError]
+    #   Either no User-Agent group exists with the given `Symbol`,
+    #   or no User-Agent string matched the given `Regexp`.
+    #
     # @api public
     #
-    def Web.user_agent=(new_agent)
-      @user_agent = new_agent
+    def Web.user_agent=(value)
+      @user_agent = case value
+                    when String
+                      user_agents.fetch(value,value)
+                    when nil
+                      nil
+                    else
+                      user_agents.fetch(value)
+                    end
     end
 
     #
@@ -220,6 +250,10 @@ module Ronin
     #
     # @see user_agent_aliases
     #
+    # @deprecated
+    #   Will be replaced by calling {user_agent=} with a `Symbol`
+    #   and will be removed in 1.0.0.
+    #
     # @api public
     #
     def Web.user_agent_alias=(name)
@@ -232,12 +266,12 @@ module Ronin
     # @param [Hash] options
     #   Additional options.
     #
-    # @option options [String] :user_agent_alias
-    #   The User-Agent Alias to use.
-    #
     # @option options [String] :user_agent
     #   The User-Agent string to use.
     #
+    # @option options [String] :user_agent_alias
+    #   The User-Agent Alias to use.
+    #
     # @option options [Network::HTTP::Proxy, Hash, String] :proxy
     #   (Web.proxy)
     #   Proxy information.
@@ -309,68 +343,21 @@ module Ronin
     end
 
     #
-    # Creates a new Mechanize Agent.
-    #
-    # @param [Hash] options
-    #   Additional options.
-    #
-    # @option options [String] :user_agent_alias
-    #   The User-Agent Alias to use.
-    #
-    # @option options [String] :user_agent
-    #   The User-Agent string to use.
-    #
-    # @option options [Network::HTTP::Proxy, Hash, String] :proxy
-    #   (Web.proxy)
-    #   Proxy information.
-    #
-    # @yield [agent]
-    #   If a block is given, it will be passed the newly created Mechanize
-    #   agent.
-    #
-    # @yieldparam [Mechanize] agent
-    #   The new Mechanize agent.
+    # A persistant Mechanize Agent.
     #
     # @return [Mechanize]
-    #   The new Mechanize agent.
+    #   The persistant Mechanize Agent.
     #
-    # @example Create a new agent.
-    #   Web.agent
-    #
-    # @example Create a new agent, with a custom User-Agent alias.
-    #   Web.agent(:user_agent_alias => 'Linux Mozilla')
-    #
-    # @example Create a new agent, with a custom User-Agent string.
-    #   Web.agent(:user_agent => 'wooden pants')
-    #
-    # @see http://rubydoc.info/gems/mechanize/Mechanize
+    # @see Mechanize
     #
     # @api public
     #
     def Web.agent(options={})
-      agent = Mechanize.new
-
-      if options[:user_agent_alias]
-        agent.user_agent_alias = options[:user_agent_alias]
-      elsif options[:user_agent]
-        agent.user_agent = options[:user_agent]
-      elsif Web.user_agent
-        agent.user_agent = Web.user_agent
-      end
-
-      proxy = Network::HTTP::Proxy.new(options[:proxy] || Web.proxy)
-
-      if proxy[:host]
-        agent.set_proxy(
-          proxy[:host],
-          proxy[:port],
-          proxy[:user],
-          proxy[:password]
-        )
+      if options.empty?
+        @agent ||= Mechanize.new(options)
+      else
+        @agent = Mechanize.new(options)
       end
-
-      yield agent if block_given?
-      return agent
     end
 
     #
@@ -382,12 +369,12 @@ module Ronin
     # @param [Hash] options
     #   Additional options.
     #
-    # @option options [String] :user_agent_alias
-    #   The User-Agent Alias to use.
-    #
     # @option options [String] :user_agent
     #   The User-Agent string to use.
     #
+    # @option options [String] :user_agent_alias
+    #   The User-Agent Alias to use.
+    #
     # @option options [Network::HTTP::Proxy, Hash] :proxy (Web.proxy)
     #   Proxy information.
     #
@@ -433,12 +420,12 @@ module Ronin
     # @param [Hash] options
     #   Additional options.
     #
-    # @option options [String] :user_agent_alias
-    #   The User-Agent Alias to use.
-    #
     # @option options [String] :user_agent
     #   The User-Agent string to use.
     #
+    # @option options [String] :user_agent_alias
+    #   The User-Agent Alias to use.
+    #
     # @option options [Network::HTTP::Proxy, Hash] :proxy (Web.proxy)
     #   Proxy information.
     #
@@ -482,12 +469,12 @@ module Ronin
     # @option options [Hash] :query
     #   Additional query parameters to post with.
     #
-    # @option options [String] :user_agent_alias
-    #   The User-Agent Alia to use.
-    #
     # @option options [String] :user_agent
     #   The User-Agent string to use.
     #
+    # @option options [String] :user_agent_alias
+    #   The User-Agent Alia to use.
+    #
     # @option options [Network::HTTP::Proxy, Hash] :proxy (Web.proxy)
     #   Proxy information.
     #
@@ -532,12 +519,12 @@ module Ronin
     # @option options [Hash] :query
     #   Additional query parameters to post with.
     #
-    # @option options [String] :user_agent_alias
-    #   The User-Agent Alias to use.
-    #
     # @option options [String] :user_agent
     #   The User-Agent string to use.
     #
+    # @option options [String] :user_agent_alias
+    #   The User-Agent Alias to use.
+    #
     # @option options [Network::HTTP::Proxy, Hash] :proxy (Web.proxy)
     #   Proxy information.
     #