ronin-web 0.2.1 → 0.3.0.pre1

data/lib/ronin/web/server/web.rb

@@ -2,21 +2,22 @@
 # Ronin Web - A Ruby library for Ronin that provides support for web
 # scraping and spidering functionality.
 #
-# Copyright (c) 2006-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
-# This program is free software; you can redistribute it and/or modify
+# This file is part of Ronin Web.
+#
+# Ronin is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
+# the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
-# This program is distributed in the hope that it will be useful,
+# Ronin is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+# along with Ronin.  If not, see <http://www.gnu.org/licenses/>.
 #
 
 require 'ronin/web/server/app'
@@ -46,8 +47,11 @@ module Ronin
     #   end
     #
     # @see Server::Base.run!
+    #
     # @since 0.2.0
     #
+    # @api public
+    #
     def Web.server(options={},&block)
       unless class_variable_defined?('@@ronin_web_server')
         @@ronin_web_server = Server::App

data/lib/ronin/web/spider.rb

@@ -2,21 +2,22 @@
 # Ronin Web - A Ruby library for Ronin that provides support for web
 # scraping and spidering functionality.
 #
-# Copyright (c) 2006-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
-# This program is free software; you can redistribute it and/or modify
+# This file is part of Ronin Web.
+#
+# Ronin is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
+# the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
-# This program is distributed in the hope that it will be useful,
+# Ronin is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+# along with Ronin.  If not, see <http://www.gnu.org/licenses/>.
 #
 
 require 'ronin/web/web'
@@ -26,6 +27,10 @@ require 'spidr/agent'
 
 module Ronin
   module Web
+    #
+    # Extends [Spidr::Agent](http://rubydoc.info/gems/spidr/Agent) with
+    # [Ronin::UI::Output::Helpers](http://rubydoc.info/gems/ronin/Ronin/UI/Output/Helpers).
+    #
     class Spider < Spidr::Agent
 
       include UI::Output::Helpers
@@ -39,7 +44,7 @@ module Ronin
       # @option options [Hash] :proxy (Web.proxy)
       #   The proxy to use while spidering.
       #
-      # @option options [String] :user_agent
+      # @option options [String] :user_agent (Web.user_agent)
       #   The User-Agent string to send.
       #
       # @option options [String] :referer
@@ -48,32 +53,37 @@ module Ronin
       # @option options [Integer] :delay (0)
       #   Duration in seconds to pause between spidering each link.
       #
-      # @option options [String, Regexp] :host
+      # @option options [Array] :schemes (['http', 'https'])
+      #   The list of acceptable URI schemes to visit.
+      #   The `https` scheme will be ignored if `net/https` cannot be
+      #   loaded.
+      #
+      # @option options [String] :host
       #   The host-name to visit.
       #
-      # @option options [Array] :hosts
-      #   Patterns of host-names to visit.
+      # @option options [Array<String, Regexp, Proc>] :hosts
+      #   The patterns which match the host-names to visit.
       #
-      # @option options [Array] :ignore_hosts
-      #   Patterns of host-names not to visit.
+      # @option options [Array<String, Regexp, Proc>] :ignore_hosts
+      #   The patterns which match the host-names to not visit.
       #
-      # @option options [Array] :ports
-      #   Ports to visit.
+      # @option options [Array<Integer, Regexp, Proc>] :ports
+      #   The patterns which match the ports to visit.
       #
-      # @option options [Array] :ignore_ports
-      #   Ports not to visit.
+      # @option options [Array<Integer, Regexp, Proc>] :ignore_ports
+      #   The patterns which match the ports to not visit.
       #
-      # @option options [Array] :links
-      #   Patterns of links to visit.
+      # @option options [Array<String, Regexp, Proc>] :links
+      #   The patterns which match the links to visit.
       #
-      # @option options [Array] :ignore_links
-      #   Patterns of links not to visit.
+      # @option options [Array<String, Regexp, Proc>] :ignore_links
+      #   The patterns which match the links to not visit.
       #
-      # @option options [Array] :ext
-      #   Patterns of file extensions to accept.
+      # @option options [Array<String, Regexp, Proc>] :exts
+      #   The patterns which match the URI path extensions to visit.
       #
-      # @option options [Array] :ignore_exts
-      #   Patterns of file extensions to ignore.
+      # @option options [Array<String, Regexp, Proc>] :ignore_exts
+      #   The patterns which match the URI path extensions to not visit.
       #
       # @yield [spider]
       #   If a block is given, it will be passed the newly created spider.
@@ -83,7 +93,9 @@ module Ronin
       #
       # @see http://spidr.rubyforge.org/docs/classes/Spidr/Agent.html
       #
-      def initialize(options={},&block)
+      # @api public
+      #
+      def initialize(options={})
         options = {
           :proxy => Web.proxy,
           :user_agent => Web.user_agent
@@ -95,7 +107,7 @@ module Ronin
           print_info("Spidering #{url}")
         end
 
-        block.call(self) if block
+        yield self if block_given?
       end
 
     end

data/lib/ronin/web/version.rb

@@ -2,26 +2,27 @@
 # Ronin Web - A Ruby library for Ronin that provides support for web
 # scraping and spidering functionality.
 #
-# Copyright (c) 2006-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
-# This program is free software; you can redistribute it and/or modify
+# This file is part of Ronin Web.
+#
+# Ronin is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
+# the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
-# This program is distributed in the hope that it will be useful,
+# Ronin is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+# along with Ronin.  If not, see <http://www.gnu.org/licenses/>.
 #
 
 module Ronin
   module Web
     # Ronin Web Version
-    VERSION = '0.2.1'
+    VERSION = '0.3.0.pre1'
   end
 end

data/lib/ronin/web/web.rb

@@ -2,24 +2,26 @@
 # Ronin Web - A Ruby library for Ronin that provides support for web
 # scraping and spidering functionality.
 #
-# Copyright (c) 2006-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
-# This program is free software; you can redistribute it and/or modify
+# This file is part of Ronin Web.
+#
+# Ronin is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
+# the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
-# This program is distributed in the hope that it will be useful,
+# Ronin is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+# along with Ronin.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-require 'ronin/network/http'
+require 'ronin/network/http/proxy'
+require 'ronin/network/http/http'
 
 require 'uri/http'
 require 'nokogiri'
@@ -41,12 +43,17 @@ module Ronin
     # @yieldparam [Nokogiri::HTML::Document] doc
     #   The new HTML document object.
     #
-    # @return [Nokogiri::HTML::Document] The new HTML document object.
+    # @return [Nokogiri::HTML::Document]
+    #   The new HTML document object.
+    #
+    # @see http://rubydoc.info/gems/nokogiri/Nokogiri/HTML/Document
     #
-    def Web.html(body,&block)
+    # @api public
+    #
+    def Web.html(body)
       doc = Nokogiri::HTML(body)
 
-      block.call(doc) if block
+      yield doc if block_given?
       return doc
     end
 
@@ -70,6 +77,10 @@ module Ronin
     #     }
     #   end
     #
+    # @see http://rubydoc.info/gems/nokogiri/Nokogiri/HTML/Builder
+    #
+    # @api public
+    #
     def Web.build_html(&block)
       Nokogiri::HTML::Builder.new(&block)
     end
@@ -87,12 +98,17 @@ module Ronin
     # @yieldparam [Nokogiri::XML::Document] doc
     #   The new XML document object.
     #
-    # @return [Nokogiri::XML::Document] The new XML document object.
+    # @return [Nokogiri::XML::Document]
+    #   The new XML document object.
+    #
+    # @see http://rubydoc.info/gems/nokogiri/Nokogiri/XML/Document
     #
-    def Web.xml(body,&block)
+    # @api public
+    #
+    def Web.xml(body)
       doc = Nokogiri::XML(body)
 
-      block.call(doc) if block
+      yield doc if block_given?
       return doc
     end
 
@@ -113,72 +129,73 @@ module Ronin
     #    }
     #  end
     #
+    # @see http://rubydoc.info/gems/nokogiri/Nokogiri/XML/Builder
+    #
+    # @api public
+    #
     def Web.build_xml(&block)
       Nokogiri::XML::Builder.new(&block)
     end
 
     #
-    # Proxy information for Ronin::Web to use.
+    # Proxy information for {Web} to use.
     #
     # @return [Network::HTTP::Proxy]
     #   The Ronin Web proxy information.
     #
-    # @see Ronin::Network::HTTP.proxy
+    # @see http://rubydoc.info/gems/ronin-support/Ronin/Network/HTTP#proxy-class_method
+    #
+    # @api public
     #
     def Web.proxy
-      Network::HTTP.proxy
+      (@proxy ||= nil) || Network::HTTP.proxy
     end
 
     #
-    # Creates a HTTP URI based on a Hash of proxy information.
+    # Sets the proxy used by {Web}.
     #
-    # @param [Network::HTTP::Proxy, Hash, String] proxy_info
-    #   The proxy information.
+    # @param [Network::HTTP::Proxy, URI::HTTP, Hash, String] new_proxy
+    #   The new proxy information to use.
     #
-    # @return [URI::HTTP, nil]
-    #   The HTTP URI that represents the proxy. If the proxy is diabled,
-    #   +nil+ will be returned.
+    # @return [Network::HTTP::Proxy]
+    #   The new proxy.
     #
-    # @example
-    #   Web.proxy_url
-    #   # => "http://www.example.com:8080"
+    # @since 0.3.0
     #
-    # @example
-    #   Web.proxy_url({:host => 'www.example.com', :port => 8081})
-    #   # => "http://www.example.com:8081"
+    # @api public
     #
-    # @example
-    #   Web.proxy_url('www.example.com:9000')
-    #   # => "http://www.example.com:9000"
-    #
-    def Web.proxy_url(proxy_info=Web.proxy)
-      proxy = if proxy_info.kind_of?(Hash)
-                Network::HTTP::Proxy.new(proxy_info)
-              else
-                Network::HTTP::Proxy.parse(proxy_info)
-              end
-
-      return proxy.url
+    def Web.proxy=(new_proxy)
+      @proxy = Network::HTTP::Proxy.create(new_proxy)
     end
 
     #
     # @return [Array]
     #   The supported Web User-Agent Aliases.
     #
+    # @see http://rubydoc.info/gems/mechanize/Mechanize#AGENT_ALIASES-constant
+    #
+    # @api public
+    #
     def Web.user_agent_aliases
-      WWW::Mechanize::AGENT_ALIASES
+      Mechanize::AGENT_ALIASES
     end
 
+    #
+    # The User-Agent string used by {Web}.
     #
     # @return [String, nil]
     #   The Ronin Web User-Agent
     #
+    # @see http://rubydoc.info/gems/ronin-support/Ronin/Network/HTTP#user_agent-class_method
+    #
+    # @api public
+    #
     def Web.user_agent
-      Network::HTTP.user_agent
+      (@user_agent ||= nil) || Network::HTTP.user_agent
     end
 
     #
-    # Sets the Ronin Web User-Agent.
+    # Sets the User-Agent string used by {Web}.
     #
     # @param [String] new_agent
     #   The User-Agent string to use.
@@ -186,8 +203,10 @@ module Ronin
     # @return [String]
     #   The new User-Agent string.
     #
+    # @api public
+    #
     def Web.user_agent=(new_agent)
-      Network::HTTP.user_agent = new_agent
+      @user_agent = new_agent
     end
 
     #
@@ -199,8 +218,12 @@ module Ronin
     # @return [String]
     #   The new User-Agent string.
     #
+    # @see user_agent_aliases
+    #
+    # @api public
+    #
     def Web.user_agent_alias=(name)
-      Network::HTTP.user_agent = Web.user_agent_aliases[name.to_s]
+      @user_agent = Web.user_agent_aliases[name.to_s]
     end
 
     #
@@ -246,9 +269,15 @@ module Ronin
     # @example Open a given URL, using a custom User-Agent string.
     #   Web.open('http://www.wired.com/', :user_agent => 'the future')
     #
+    # @see http://rubydoc.info/stdlib/open-uri/frames
+    #
+    # @api public
+    #
     def Web.open(url,options={})
       user_agent_alias = options.delete(:user_agent_alias)
-      proxy = (options.delete(:proxy) || Web.proxy)
+      proxy = Network::HTTP::Proxy.create(
+        options.delete(:proxy) || Web.proxy
+      )
       user = options.delete(:user)
       password = options.delete(:password)
       content_length_proc = options.delete(:content_length_proc)
@@ -261,7 +290,7 @@ module Ronin
       end
 
       if proxy[:host]
-        headers[:proxy] = Web.proxy_url(proxy)
+        headers[:proxy] = proxy.url
       end
 
       if user
@@ -299,10 +328,10 @@ module Ronin
     #   If a block is given, it will be passed the newly created Mechanize
     #   agent.
     #
-    # @yieldparam [WWW::Mechanize] agent
+    # @yieldparam [Mechanize] agent
     #   The new Mechanize agent.
     #
-    # @return [WWW::Mechanize]
+    # @return [Mechanize]
     #   The new Mechanize agent.
     #
     # @example Create a new agent.
@@ -314,10 +343,12 @@ module Ronin
     # @example Create a new agent, with a custom User-Agent string.
     #   Web.agent(:user_agent => 'wooden pants')
     #
-    # @see http://mechanize.rubyforge.org/mechanize/WWW/Mechanize.html
+    # @see http://rubydoc.info/gems/mechanize/Mechanize
+    #
+    # @api public
     #
-    def Web.agent(options={},&block)
-      agent = WWW::Mechanize.new
+    def Web.agent(options={})
+      agent = Mechanize.new
 
       if options[:user_agent_alias]
         agent.user_agent_alias = options[:user_agent_alias]
@@ -327,21 +358,18 @@ module Ronin
         agent.user_agent = Web.user_agent
       end
 
-      proxy = if options[:proxy].kind_of?(Hash)
-                options[:proxy]
-              elsif options[:proxy].kind_of?(String)
-                Network::HTTP::Proxy.parse(options[:proxy])
-              elsif options[:proxy].nil?
-                Web.proxy
-              else
-                raise(RuntimeError,"the given :proxy option is neither a Proxy, Hash or String",caller)
-              end
+      proxy = Network::HTTP::Proxy.new(options[:proxy] || Web.proxy)
 
       if proxy[:host]
-        agent.set_proxy(proxy[:host],proxy[:port],proxy[:user],proxy[:password])
+        agent.set_proxy(
+          proxy[:host],
+          proxy[:port],
+          proxy[:user],
+          proxy[:password]
+        )
       end
 
-      block.call(agent) if block
+      yield agent if block_given?
       return agent
     end
 
@@ -367,15 +395,15 @@ module Ronin
     #   If a block is given, it will be passed the page for the requested
     #   URL.
     #
-    # @yieldparam [WWW::Mechanize::Page] page
+    # @yieldparam [Mechanize::Page] page
     #   The requested page.
     #
-    # @return [WWW::Mechanize::Page]
+    # @return [Mechanize::Page]
     #   The requested page.
     #
     # @example
     #   Web.get('http://www.rubyinside.com')
-    #   # => WWW::Mechanize::Page
+    #   # => Mechanize::Page
     #
     # @example
     #   Web.get('http://www.rubyinside.com') do |page|
@@ -384,12 +412,14 @@ module Ronin
     #     end
     #   end
     #
-    # @see http://mechanize.rubyforge.org/mechanize/WWW/Mechanize/Page.html
+    # @see http://rubydoc.info/gems/mechanize/Mechanize/Page
+    #
+    # @api public
     #
-    def Web.get(url,options={},&block)
+    def Web.get(url,options={})
       page = Web.agent(options).get(url)
 
-      block.call(page) if block
+      yield page if block_given?
       return page
     end
 
@@ -429,10 +459,14 @@ module Ronin
     #     puts body
     #   end
     #
-    def Web.get_body(url,options={},&block)
+    # @see get
+    #
+    # @api public
+    #
+    def Web.get_body(url,options={})
       body = Web.get(url,options).body
 
-      block.call(body) if block
+      yield body if block_given?
       return body
     end
 
@@ -461,23 +495,27 @@ module Ronin
     #   If a block is given, it will be passed the page for the requested
     #   URL.
     #
-    # @yieldparam [WWW::Mechanize::Page] page
+    # @yieldparam [Mechanize::Page] page
     #   The requested page.
     #
-    # @return [WWW::Mechanize::Page]
+    # @return [Mechanize::Page]
     #   The requested page.
     #
     # @example
     #   Web.post('http://www.rubyinside.com')
-    #   # => WWW::Mechanize::Page
+    #   # => Mechanize::Page
     #
-    def Web.post(url,options={},&block)
+    # @see http://rubydoc.info/gems/mechanize/Mechanize/Page
+    #
+    # @api public
+    #
+    def Web.post(url,options={})
       query = {}
       query.merge!(options[:query]) if options[:query]
 
       page = Web.agent(options).post(url,query)
 
-      block.call(page) if block
+      yield page if block_given?
       return page
     end
 
@@ -506,10 +544,10 @@ module Ronin
     # @yield [body]
     #   If a block is given, it will be passed the body of the page.
     #
-    # @yieldparam [WWW::Mechanize::Page] page
+    # @yieldparam [Mechanize::Page] page
     #   The body of the requested page.
     #
-    # @return [WWW::Mechanize::Page]
+    # @return [Mechanize::Page]
     #   The body of the requested page.
     #
     # @example
@@ -521,10 +559,14 @@ module Ronin
     #     puts body
     #   end
     #
-    def Web.post_body(url,options={},&block)
+    # @see post
+    #
+    # @api public
+    #
+    def Web.post_body(url,options={})
       body = Web.post(url,options).body
 
-      block.call(body) if block
+      yield body if block_given?
       return body
     end
   end