ronin-web 0.3.0.pre2 → 1.0.0.beta1

data/lib/ronin/network/mixins/web.rb

@@ -1,256 +0,0 @@
-#
-# Ronin Web - A Ruby library for Ronin that provides support for web
-# scraping and spidering functionality.
-#
-# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
-#
-# This file is part of Ronin Web.
-#
-# Ronin is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ronin is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ronin.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-require 'ronin/ui/output/helpers'
-require 'ronin/network/http/proxy'
-require 'ronin/web/web'
-require 'ronin/mixin'
-
-require 'parameters'
-
-module Ronin
-  module Network
-    module Mixins
-      module Web
- 
-        mixin UI::Output::Helpers, Parameters
-
-        mixin do
-          # The Web Proxy host to connect to
-          parameter :web_proxy_host, :type => String,
-                                     :description => 'Web Proxy host'
-
-          # The Web Proxy port to connect to
-          parameter :web_proxy_port, :type => Integer,
-                                     :description => 'Web Proxy port'
-
-          # The Web Proxy user to authenticate with
-          parameter :web_proxy_user, :type => String,
-                                     :description => 'Web Proxy authentication user'
-
-          # The Web Proxy password to authenticate with
-          parameter :web_proxy_password, :type => String,
-                                         :description => 'Web Proxy authentication password'
-        end
-
-        protected
-
-        #
-        # Combines the proxy information set by the `web_proxy_host`,
-        # `web_proxy_port`, `web_proxy_user` and `web_proxy_password`
-        # parameters.
-        #
-        # @return [Network::HTTP::Proxy]
-        #   The current proxy information.
-        #
-        # @api semipublic
-        #
-        def web_proxy
-          HTTP::Proxy.new(
-            :host => @web_proxy_host,
-            :port => @web_proxy_port,
-            :user => @web_proxy_user,
-            :password => @web_proxy_password
-          )
-        end
-
-        #
-        # Provides a persistant Mechanize agent.
-        #
-        # @param [Hash] options
-        #   Additional options for initializing the agent.
-        #
-        # @option options [Hash] :proxy (web_proxy)
-        #   Proxy information.
-        #
-        # @option options [String] :user_agent (web_user_agent)
-        #   User-Agent string to use.
-        #
-        # @return [Mechanize]
-        #   The agent.
-        #
-        # @api semipublic
-        #
-        def web_agent(options={},&block)
-          unless @web_agent
-            options[:proxy] ||= web_proxy
-            options[:user_agent] ||= @web_user_agent
-
-            @web_agent = Ronin::Web.agent(options,&block)
-          end
-
-          return @web_agent
-        end
-
-        #
-        # Creates a Mechanize Page for the contents at a given URL.
-        #
-        # @param [URI::Generic, String] url
-        #   The URL to request.
-        #
-        # @param [Hash] options
-        #   Additional options to initialize the agent with.
-        #
-        # @option options [Hash] :proxy (web_proxy)
-        #   Proxy information.
-        #
-        # @option options [String] :user_agent (web_user_agent)
-        #   User-Agent string to use.
-        #
-        # @yield [page]
-        #   If a block is given, it will be passed the page for the
-        #   requested URL.
-        #
-        # @yieldparam [Mechanize::Page] page
-        #   The requested page.
-        #
-        # @return [Mechanize::Page]
-        #   The requested page.
-        #
-        # @api semipublic
-        #
-        def web_get(url,options={})
-          print_info "Requesting #{url}"
-          page = web_agent(options).get(url)
-
-          yield page if block_given?
-          return page
-        end
-
-        #
-        # Requests the body of the Mechanize Page created from the response
-        # of the given URL.
-        #
-        # @param [URI::Generic, String] url
-        #   The URL to request.
-        #
-        # @param [Hash] options
-        #   Additional options to initialize the agent with.
-        #
-        # @option options [Hash] :proxy (web_proxy)
-        #   Proxy information.
-        #
-        # @option options [String] :user_agent (web_user_agent)
-        #   User-Agent string to use.
-        #
-        # @yield [body]
-        #   If a block is given, it will be passed the body of the page.
-        #
-        # @yieldparam [String] body
-        #   The requested body of the page.
-        #
-        # @return [String]
-        #   The requested body of the page.
-        #
-        # @api semipublic
-        #
-        def web_get_body(url,options={})
-          page = web_get(url,options)
-          body = page.body
-
-          yield body if block_given?
-          return body
-        end
-
-        #
-        # Posts to a given URL and creates a Mechanize Page from the
-        # response.
-        #
-        # @param [URI::Generic, String] url
-        #   The URL to post to.
-        #
-        # @param [Hash] options
-        #   Additional options to initialize the agent with.
-        #
-        # @option options [Hash] :query
-        #   Additional query parameters to post with.
-        #
-        # @option options [Hash] :proxy (web_proxy)
-        #   Proxy information.
-        #
-        # @option options [String] :user_agent (web_user_agent)
-        #   User-Agent string to use.
-        #
-        # @yield [page]
-        #   If a block is given, it will be passed the page for the
-        #   requested URL.
-        #
-        # @yieldparam [Mechanize::Page] page
-        #   The requested page.
-        #
-        # @return [Mechanize::Page]
-        #   The requested page.
-        #
-        # @api semipublic
-        #
-        def web_post(url,options={})
-          query = {}
-          query.merge!(options[:query]) if options[:query]
-
-          print_info "Posting #{url}"
-          page = web_agent(options).post(url)
-
-          yield page if block_given?
-          return page
-        end
-
-        #
-        # Posts to a given URL and returns the body of the Mechanize Page
-        # created from the response.
-        #
-        # @param [URI::Generic, String] url
-        #   The URL to post to.
-        #
-        # @param [Hash] options
-        #   Additional options to initialize the agent with.
-        #
-        # @option options [Hash] :query
-        #   Additional query parameters to post with.
-        #
-        # @option options [Hash] :proxy (web_proxy)
-        #   Proxy information.
-        #
-        # @option options [String] :user_agent (web_user_agent)
-        #   User-Agent string to use.
-        #
-        # @yield [body]
-        #   If a block is given, it will be passed the body of the page.
-        #
-        # @yieldparam [Mechanize::Page] page
-        #   The body of the requested page.
-        #
-        # @return [String]
-        #   The requested body of the page.
-        #
-        # @api semipublic
-        #
-        def web_post_body(url,options={})
-          page = web_post(url,options)
-          body = page.body
-
-          yield body if block_given?
-          return body
-        end
-      end
-    end
-  end
-end

data/lib/ronin/web/extensions/nokogiri/xml/attr.rb

@@ -1,47 +0,0 @@
-#
-# Ronin Web - A Ruby library for Ronin that provides support for web
-# scraping and spidering functionality.
-#
-# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
-#
-# This file is part of Ronin Web.
-#
-# Ronin is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ronin is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ronin.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-require 'nokogiri'
-
-module Nokogiri
-  module XML
-    class Attr < Node
-
-      #
-      # Determines if the attribute is similar to another attribute.
-      #
-      # @param [Nokogiri::XML::Attr] other
-      #   The other attribute.
-      #
-      # @return [Boolean]
-      #   Specifies if the attribute is similar, in indentity or value,
-      #   to the other attribute.
-      #
-      # @api public
-      #
-      def ==(other)
-        super(other) && (self.value == other.value)
-      end
-
-    end
-  end
-end

data/lib/ronin/web/extensions/nokogiri/xml/document.rb

@@ -1,48 +0,0 @@
-#
-# Ronin Web - A Ruby library for Ronin that provides support for web
-# scraping and spidering functionality.
-#
-# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
-#
-# This file is part of Ronin Web.
-#
-# Ronin is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ronin is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ronin.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-require 'nokogiri'
-
-module Nokogiri
-  module XML
-    class Document < Node
-
-      #
-      # Calculates the sum of all sub-children of the document.
-      #
-      # @return [Integer]
-      #   The total number of children and sub-children of the document.
-      #   Returns `0` if the document has no root element.
-      #
-      # @api public
-      #
-      def total_children
-        if root
-          1 + root.total_children
-        else
-          0
-        end
-      end
-
-    end
-  end
-end

data/lib/ronin/web/extensions/nokogiri/xml/element.rb

@@ -1,57 +0,0 @@
-#
-# Ronin Web - A Ruby library for Ronin that provides support for web
-# scraping and spidering functionality.
-#
-# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
-#
-# This file is part of Ronin Web.
-#
-# Ronin is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ronin is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ronin.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-require 'nokogiri'
-
-module Nokogiri
-  module XML
-    class Element < Node
-
-      #
-      # Determines if the element is similar to another element.
-      #
-      # @param [Nokogiri::XML::Element] other
-      #   The other element.
-      #
-      # @return [Boolean]
-      #   Specifies whether the element is equal, in identity or value, to
-      #   another element.
-      #
-      # @api public
-      #
-      def ==(other)
-        return false unless super(other)
-        return false unless attribute_nodes.length == other.attribute_nodes.length
-
-        (0...attribute_nodes.length).each do |index|
-          attr1 = attribute_nodes[index]
-          attr2 = other.attribute_nodes[index]
-
-          return false unless attr1.similar?(attr2)
-        end
-
-        return true
-      end
-
-    end
-  end
-end

data/lib/ronin/web/extensions/nokogiri/xml/node.rb

@@ -1,86 +0,0 @@
-#
-# Ronin Web - A Ruby library for Ronin that provides support for web
-# scraping and spidering functionality.
-#
-# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
-#
-# This file is part of Ronin Web.
-#
-# Ronin is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ronin is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ronin.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-require 'nokogiri'
-
-module Nokogiri
-  module XML
-    class Node
-
-      #
-      # Calculates the sum of all children of the node.
-      #
-      # @return [Integer]
-      #   The total number of children of the node.
-      #
-      # @api public
-      #
-      def total_children
-        count = 0
-
-        traverse { |node| count += 1 }
-
-        return count - 1
-      end
-
-      #
-      # Traverses all text nodes which are children of the node.
-      #
-      # @yield [node]
-      #   A block will be passed each text node.
-      #
-      # @yieldparam [Nokogiri::XML::Text] node
-      #   A text node.
-      #
-      # @return [Enumerator]
-      #   If no block is given, an Enumerator object will be returned.
-      #
-      # @api public
-      #
-      def traverse_text
-        return enum_for(:traverse_text) unless block_given?
-
-        yield self if text?
-
-        traverse do |node|
-          yield node if node.text?
-        end
-      end
-
-      #
-      # Determines if the node is similar to another node.
-      #
-      # @return [Boolean]
-      #   Specifies whether the node is equal, in identity or value, to
-      #   another node.
-      #
-      # @api public
-      #
-      def ==(other)
-        return false unless other
-
-        (self.type == other.type) && (self.name == other.name)
-      end
-
-    end
-  end
-end

data/lib/ronin/web/extensions/nokogiri/xml/text.rb

@@ -1,47 +0,0 @@
-#
-# Ronin Web - A Ruby library for Ronin that provides support for web
-# scraping and spidering functionality.
-#
-# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
-#
-# This file is part of Ronin Web.
-#
-# Ronin is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ronin is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ronin.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-require 'nokogiri'
-
-module Nokogiri
-  module XML
-    class Text < CharacterData
-
-      #
-      # Determines if the text node is similar to another text node.
-      #
-      # @param [Nokogiri::XML::Text] other
-      #   The other text node.
-      #
-      # @return [Boolean]
-      #   Specifies if the text node is similar, in indentity or value,
-      #   to the other text node.
-      #
-      # @api public
-      #
-      def ==(other)
-        super(other) && (self.content == other.content)
-      end
-
-    end
-  end
-end

data/lib/ronin/web/extensions/nokogiri/xml.rb

@@ -1,27 +0,0 @@
-#
-# Ronin Web - A Ruby library for Ronin that provides support for web
-# scraping and spidering functionality.
-#
-# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
-#
-# This file is part of Ronin Web.
-#
-# Ronin is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ronin is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ronin.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-require 'ronin/web/extensions/nokogiri/xml/node'
-require 'ronin/web/extensions/nokogiri/xml/text'
-require 'ronin/web/extensions/nokogiri/xml/attr'
-require 'ronin/web/extensions/nokogiri/xml/element'
-require 'ronin/web/extensions/nokogiri/xml/document'

data/lib/ronin/web/extensions/nokogiri.rb

@@ -1,23 +0,0 @@
-#
-# Ronin Web - A Ruby library for Ronin that provides support for web
-# scraping and spidering functionality.
-#
-# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
-#
-# This file is part of Ronin Web.
-#
-# Ronin is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ronin is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ronin.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-require 'ronin/web/extensions/nokogiri/xml'

data/lib/ronin/web/extensions.rb

@@ -1,23 +0,0 @@
-#
-# Ronin Web - A Ruby library for Ronin that provides support for web
-# scraping and spidering functionality.
-#
-# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
-#
-# This file is part of Ronin Web.
-#
-# Ronin is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ronin is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ronin.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-require 'ronin/web/extensions/nokogiri'