ronin-web 0.2.1 → 0.3.0.pre1

data/lib/ronin/web/extensions/nokogiri/xml/node.rb

@@ -1,3 +1,25 @@
+#
+# Ronin Web - A Ruby library for Ronin that provides support for web
+# scraping and spidering functionality.
+#
+# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# This file is part of Ronin Web.
+#
+# Ronin is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ronin is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ronin.  If not, see <http://www.gnu.org/licenses/>.
+#
+
 require 'nokogiri'
 
 module Nokogiri
@@ -5,38 +27,55 @@ module Nokogiri
     class Node
 
       #
-      # Returns the total count of all sub-children of the node.
+      # Calculates the sum of all children of the node.
+      #
+      # @return [Integer]
+      #   The total number of children of the node.
+      #
+      # @api public
       #
       def total_children
         count = 0
-        first = self.child
 
-        return count unless first
+        traverse { |node| count += 1 }
 
-        while first
-          count += (1 + first.total_children)
-
-          first = first.next
-        end
-
-        count
+        return count - 1
       end
 
-      def traverse_text(&block)
-        block.call(self) if text?
-
-        first = self.child
+      #
+      # Traverses all text nodes which are children of the node.
+      #
+      # @yield [node]
+      #   A block will be passed each text node.
+      #
+      # @yieldparam [Nokogiri::XML::Text] node
+      #   A text node.
+      #
+      # @return [Enumerator]
+      #   If no block is given, an Enumerator object will be returned.
+      #
+      # @api public
+      #
+      def traverse_text
+        return enum_for(:traverse_text) unless block_given?
 
-        while first
-          first.traverse_text(&block)
+        yield self if text?
 
-          first = first.next
+        traverse do |node|
+          yield node if node.text?
         end
-
-        self
       end
 
-      def similar?(other)
+      #
+      # Determines if the node is similar to another node.
+      #
+      # @return [Boolean]
+      #   Specifies whether the node is equal, in identity or value, to
+      #   another node.
+      #
+      # @api public
+      #
+      def ==(other)
         return false unless other
 
         (self.type == other.type) && (self.name == other.name)

data/lib/ronin/web/extensions/nokogiri/xml/text.rb

@@ -1,10 +1,44 @@
+#
+# Ronin Web - A Ruby library for Ronin that provides support for web
+# scraping and spidering functionality.
+#
+# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# This file is part of Ronin Web.
+#
+# Ronin is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ronin is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ronin.  If not, see <http://www.gnu.org/licenses/>.
+#
+
 require 'nokogiri'
 
 module Nokogiri
   module XML
-    class Text < Node
+    class Text < CharacterData
 
-      def similar?(other)
+      #
+      # Determines if the text node is similar to another text node.
+      #
+      # @param [Nokogiri::XML::Text] other
+      #   The other text node.
+      #
+      # @return [Boolean]
+      #   Specifies if the text node is similar, in indentity or value,
+      #   to the other text node.
+      #
+      # @api public
+      #
+      def ==(other)
         super(other) && (self.content == other.content)
       end
 

data/lib/ronin/web/middleware.rb

@@ -0,0 +1,27 @@
+#
+# Ronin Web - A Ruby library for Ronin that provides support for web
+# scraping and spidering functionality.
+#
+# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# This file is part of Ronin Web.
+#
+# Ronin is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ronin is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ronin.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+require 'ronin/web/middleware/base'
+require 'ronin/web/middleware/files'
+require 'ronin/web/middleware/directories'
+require 'ronin/web/middleware/router'
+require 'ronin/web/middleware/proxy'

data/lib/ronin/web/middleware/base.rb

@@ -0,0 +1,144 @@
+#
+# Ronin Web - A Ruby library for Ronin that provides support for web
+# scraping and spidering functionality.
+#
+# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# This file is part of Ronin Web.
+#
+# Ronin is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ronin is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ronin.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+require 'ronin/web/middleware/helpers'
+require 'ronin/ui/output/helpers'
+
+module Ronin
+  module Web
+    module Middleware
+      #
+      # Baase-class for all Ronin Web Middleware.
+      #
+      class Base
+
+        include Helpers
+        include UI::Output::Helpers
+
+        # The default status code to return
+        DEFAULT_STATUS = 200
+
+        # The status code to return
+        attr_accessor :default_status
+
+        # The default headers to return
+        attr_reader :default_headers
+
+        #
+        # Creates a new middleware object.
+        #
+        # @param [#call] app
+        #   The application the middleware will sit in front of.
+        #
+        # @param [Hash] options
+        #   Additional options.
+        #
+        # @option options [Integer] :default_status (DEFAULT_STATUS)
+        #   The status code to return.
+        #
+        # @option options [Hash] :default_headers
+        #   The headers to return.
+        #
+        # @yield [middleware]
+        #   If a block is given, it will be passed the new middleware.
+        #
+        # @yieldparam [Base] middleware
+        #   The new middleware object.
+        #
+        # @since 0.3.0
+        #
+        # @api semipublic
+        #
+        def initialize(app,options={})
+          @app = app
+
+          @default_status = (options[:default_status] || DEFAULT_STATUS)
+          @default_headers = {}
+
+          if options.has_key?(:default_headers)
+            @default_headers.merge!(options[:default_headers])
+          end
+
+          yield self if block_given?
+        end
+
+        #
+        # Passes the request to the application.
+        #
+        # @param [Hash, Rack::Request] env
+        #   The request.
+        #
+        # @return [Rack::Response]
+        #   The response.
+        #
+        # @since 0.3.0
+        #
+        # @api semipublic
+        #
+        def call(env)
+          @app.call(env)
+        end
+
+        protected
+
+        #
+        # Creates a new response.
+        #
+        # @param [String, Array, IO] body
+        #   The body for the response.
+        #
+        # @param [Hash] headers
+        #   Additional headers for the response.
+        #
+        # @param [Integer] status
+        #   The HTTP Status Code for the response.
+        #
+        # @yield [response]
+        #   If a block is given, it will be passed the new response.
+        #
+        # @yieldparam [Response] response
+        #   The new response.
+        #
+        # @return [Array]
+        #   The new response.
+        #
+        # @example Create a response.
+        #   response ['Hello'], {'Content-Type' => 'text/txt'}, 200
+        #
+        # @example Create a response with just a String.
+        #   response 'Hello'
+        #
+        # @since 0.3.0
+        #
+        # @api semipublic
+        #
+        def response(body=[],headers={},status=nil)
+          status ||= @default_status
+          headers = @default_headers.merge(headers)
+
+          return super(body,headers,status)
+        end
+
+      end
+    end
+  end
+end

data/lib/ronin/web/middleware/directories.rb

@@ -0,0 +1,177 @@
+#
+# Ronin Web - A Ruby library for Ronin that provides support for web
+# scraping and spidering functionality.
+#
+# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# This file is part of Ronin Web.
+#
+# Ronin is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ronin is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ronin.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+require 'ronin/web/middleware/base'
+
+require 'set'
+
+module Ronin
+  module Web
+    module Middleware
+      #
+      # A Rack middleware to host local directories at specific remote
+      # paths.
+      #
+      #     use Ronin::Web::Middleware::Directories do |dirs|
+      #       dirs.map '/downloads', '/tmp/ronin_downloads'
+      #     end
+      #
+      class Directories < Base
+
+        # The predefined index file names
+        INDEX_NAMES = %w[index.html index.xhtml index.htm]
+
+        # The mapping of remote paths to local directories 
+        attr_reader :paths
+
+        #
+        # Creates a new {Directories} middleware.
+        #
+        # @param [#call] app
+        #   The application the middleware sits in front of.
+        #
+        # @param [Hash] options
+        #   Additional options.
+        #
+        # @option options [Hash{String => String}] :paths
+        #   The mapping of remote paths to local directories.
+        #
+        # @yield [directories]
+        #   If a block is given, it will be passed the new directories
+        #   middleware.
+        #
+        # @yieldparam [Directories] directories
+        #   The new directories middleware object.
+        #
+        # @since 0.3.0
+        #
+        # @api public
+        #
+        def initialize(app,options={},&block)
+          @paths = {}
+          @paths_order = []
+
+          if options.has_key?(:paths)
+            options[:paths].each do |remote_path,local_dir|
+              map(remote_path,local_dir)
+            end
+          end
+
+          super(app,&block)
+        end
+
+        #
+        # The names of index files.
+        #
+        # @return [Set]
+        #   The set of index file names.
+        #
+        # @since 0.3.0
+        #
+        # @api public
+        #
+        def Directories.index_names
+          @@directories_index_names ||= Set.new(INDEX_NAMES)
+        end
+
+        #
+        # Maps a remote path to a local directory.
+        #
+        # @param [String] remote_path
+        #   The remote path to map.
+        #
+        # @param [String] local_dir
+        #   The local directory that the remote path will map to.
+        #
+        # @return [true]
+        #
+        # @since 0.3.0
+        #
+        # @api public
+        #
+        def map(remote_path,local_dir)
+          @paths[remote_path] = local_dir
+          
+          # sort paths by number of sub-directories
+          @paths_order = @paths.keys.sort_by do |path|
+            -(path.split('/').length)
+          end
+
+          return true
+        end
+
+        #
+        # Returns a file from a local directory, if the directory
+        # was mapped to a remote path.
+        #
+        # @param [Hash, Rack::Request] env
+        #   The request.
+        #
+        # @return [Array, Response]
+        #   The response.
+        #
+        # @since 0.3.0
+        #
+        # @api public
+        #
+        def call(env)
+          path = sanitize_path(env['PATH_INFO'])
+
+          # finds the remote directory that the paths starts with or is
+          # equal to.
+          remote_path = @paths_order.find do |remote_path|
+                          if remote_path == '/'
+                            true
+                          elsif path[0,remote_path.length] == remote_path
+                            (path[remote_path.length] == '/') ||
+                            (path.length == remote_path.length)
+                          end
+                        end
+
+          if remote_path
+            local_dir = @paths[remote_path]
+            sub_path = path[remote_path.length..-1]
+
+            return_file = proc { |local_path|
+              if File.file?(local_path)
+                print_info "Returning file #{local_path.dump}"
+                return response_for(local_path)
+              end
+            }
+
+            if sub_path.empty?
+              # attempt to find an index file in the directory
+              Directories.index_names.each do |index|
+                return_file.call(File.join(local_dir,index))
+              end
+            else
+              return_file.call(File.join(local_dir,sub_path))
+            end
+          end
+
+          super(env)
+        end
+
+      end
+    end
+  end
+end