ronin-web 0.2.1 → 0.3.0.pre1

data/lib/ronin/scanners/web.rb

@@ -1,52 +0,0 @@
-#
-# Ronin Web - A Ruby library for Ronin that provides support for web
-# scraping and spidering functionality.
-#
-# Copyright (c) 2006-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
-#
-
-require 'ronin/web/spider'
-require 'ronin/scanners/scanner'
-
-module Ronin
-  module Scanners
-    class Web < Web::Spider
-
-      include Scanner
-
-      protected
-
-      def each_target(&block)
-        print_info "Started web spidering ..."
-
-        history.clear
-
-        unless visit_hosts.empty?
-          enqueue("http://#{visit_hosts.first}/")
-        end
-
-        run do |page|
-          print_info "Scanning page: #{page.url}"
-          block.call(page)
-        end
-
-        print_info "Finished web spidering."
-      end
-
-    end
-  end
-end

data/lib/ronin/web/server/files.rb

@@ -1,92 +0,0 @@
-#
-# Ronin Web - A Ruby library for Ronin that provides support for web
-# scraping and spidering functionality.
-#
-# Copyright (c) 2006-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
-#
-
-require 'ronin/web/server/helpers/files'
-
-module Ronin
-  module Web
-    module Server
-      module Files
-        def self.included(base)
-          base.module_eval do
-            #
-            # Hosts the contents of a file.
-            #
-            # @param [String] http_path
-            #   The path the web server will host the file at.
-            #
-            # @param [String] path
-            #   The path to the local file.
-            #
-            # @param [Symbol] custom_content_type
-            #   Optional content-type to host the file as.
-            #
-            # @example
-            #   MyApp.file '/robots.txt', '/path/to/my_robots.txt'
-            #
-            # @since 0.2.0
-            #
-            def self.file(http_path,path,custom_content_type=nil)
-              path = File.expand_path(path)
-
-              any(http_path) do
-                return_file(path,custom_content_type)
-              end
-            end
-
-            #
-            # Hosts the contents of the directory.
-            #
-            # @param [String] http_path
-            #   The path the web server will host the directory at.
-            #
-            # @param [String] directory
-            #   The path to the local directory.
-            #
-            # @param [Symbol] custom_content_type
-            #   Optional content-type to host the contents of the directory
-            #   with.
-            #
-            # @example
-            #   MyApp.directory '/download/', '/tmp/files/'
-            #
-            # @since 0.2.0
-            #
-            def self.directory(http_path,directory,custom_content_type=nil)
-              directory = File.expand_path(directory)
-
-              any(File.join(http_path,'*')) do
-                sub_path = File.expand_path(File.join('',params[:splat].first))
-                full_path = File.join(directory,sub_path)
-
-                return_file(full_path,custom_content_type)
-              end
-            end
-
-            protected
-
-            helpers Ronin::Web::Server::Helpers::Files
-          end
-        end
-      end
-    end
-  end
-end

data/lib/ronin/web/server/helpers.rb

@@ -1,25 +0,0 @@
-#
-# Ronin Web - A Ruby library for Ronin that provides support for web
-# scraping and spidering functionality.
-#
-# Copyright (c) 2006-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
-#
-
-require 'ronin/web/server/helpers/rendering'
-require 'ronin/web/server/helpers/files'
-require 'ronin/web/server/helpers/hosts'
-require 'ronin/web/server/helpers/proxy'

data/lib/ronin/web/server/helpers/files.rb

@@ -1,126 +0,0 @@
-#
-# Ronin Web - A Ruby library for Ronin that provides support for web
-# scraping and spidering functionality.
-#
-# Copyright (c) 2006-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
-#
-
-module Ronin
-  module Web
-    module Server
-      module Helpers
-        module Files
-          # Directory index files
-          INDICES = ['index.htm', 'index.html']
-
-          #
-          # Sets the content_type based on the extension of a given file.
-          #
-          # @param [String] path
-          #   The path to guess the content-type for.
-          #
-          # @return [String]
-          #   The MIME content-type of the file.
-          #
-          # @example
-          #   content_type_for 'file.html'
-          #
-          # @since 0.2.0
-          #
-          def content_type_for(path)
-            ext = File.extname(path).downcase
-
-            return content_type(ext[1..-1])
-          end
-
-          #
-          # Finds the index file for a given directory.
-          #
-          # @param [String] path
-          #   The path of the directory.
-          #
-          # @yield [index_path]
-          #   If a block is given, it will be passed the path of the
-          #   index file for the given directory.
-          # 
-          # @yieldparam [String] index_path
-          #   The path to the index file.
-          #
-          # @return [String]
-          #   The path to the index file.
-          #
-          # @since 0.2.0
-          #
-          def index_of(path,&block)
-            path = File.expand_path(path)
-
-            Base.indices.each do |name|
-              index = File.join(path,name)
-
-              if File.file?(index)
-                block.call(index) if block
-                return index
-              end
-            end
-
-            pass
-          end
-
-          #
-          # Returns a file to the client with the appropriate content-type.
-          #
-          # @param [String] path
-          #   The path of the file to return.
-          #
-          # @param [Symbol] custom_content_type
-          #   Optional content-type to return the file with.
-          #
-          # @example
-          #   return_file 'lol.jpg'
-          #
-          # @example
-          #   return_file '/tmp/file', :html
-          #
-          # @since 0.2.0
-          #
-          def return_file(path,custom_content_type=nil)
-            path = File.expand_path(path)
-
-            pass unless File.exists?(path)
-
-            if File.directory?(path)
-              index_of(path) { |index| path = index }
-            end
-
-            if custom_content_type
-              content_type custom_content_type
-            else
-              content_type_for path
-            end
-
-            case request.request_method
-            when 'GET', 'POST'
-              halt 200, File.new(path)
-            else
-              halt 302
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/ronin/web/server/helpers/hosts.rb

@@ -1,72 +0,0 @@
-#
-# Ronin Web - A Ruby library for Ronin that provides support for web
-# scraping and spidering functionality.
-#
-# Copyright (c) 2006-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
-#
-
-module Ronin
-  module Web
-    module Server
-      module Helpers
-        module Hosts
-          #
-          # Calls a given block for requests with matching Host headers.
-          #
-          # @param [Regexp, String] name_or_pattern
-          #   The exact name or pattern to match Host headers.
-          #
-          # @yield []
-          #   The given block will be called when a request is received
-          #   with the matching Host header.
-          #
-          # @example
-          #   downloads = 0
-          #
-          #   get '/file' do
-          #     for_host /^ftp/ do
-          #       downloads += 1
-          #   
-          #       content_type :txt
-          #       'some file'
-          #     end
-          #
-          #     for_host /^www/ do
-          #       downloads += 1
-          #   
-          #       'some file'
-          #     end
-          #
-          #     for_host 'localhost' do
-          #       "Total Downloads: #{downloads}"
-          #     end
-          #   end
-          #
-          # @since 0.2.0
-          #
-          def for_host(name_or_pattern,&block)
-            if name_or_pattern.kind_of?(Regexp)
-              halt(*block.call()) if request.host =~ name_or_pattern
-            else
-              halt(*block.call()) if request.host == name_or_pattern.to_s
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/ronin/web/server/helpers/proxy.rb

@@ -1,153 +0,0 @@
-#
-# Ronin Web - A Ruby library for Ronin that provides support for web
-# scraping and spidering functionality.
-#
-# Copyright (c) 2006-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
-#
-
-require 'ronin/network/http'
-
-module Ronin
-  module Web
-    module Server
-      module Helpers
-        module Proxy
-          #
-          # Proxies the current request.
-          #
-          # @param [Hash] options
-          #   Additional options to use when proxying the request.
-          #
-          # @yield [(response), body]
-          #   If a block is given, it will be passed the optional response
-          #   of the proxied request and the body received from the
-          #   proxied request.
-          #
-          # @yieldparam [Net::HTTP::Response] response
-          #   The response of the proxied request.
-          #
-          # @yieldparam [String] body
-          #   The body from the proxied request.
-          #
-          # @example
-          #   get '/login.php' do
-          #     proxy do |body|
-          #       body.gsub(/https/,'http')
-          #     end
-          #   end
-          #
-          # @since 0.2.0
-          #
-          def proxy(options={},&block)
-            default_options = {
-              :host => request.host,
-              :port => request.port,
-              :method => request.request_method,
-              :path => request.path_info,
-              :query => request.query_string,
-              :content_type => request.content_type
-            }
-
-            request.env.each do |name,value|
-              if name =~ /^HTTP/
-                default_options[name.gsub(/^HTTP_/,'').downcase.to_sym] = value
-              end
-            end
-            
-            options = default_options.merge(options)
-            http_response = Net.http_request(options)
-
-            response = Rack::Response.new(
-              [http_response.body],
-              http_response.code,
-              http_response.to_hash
-            )
-
-            if block
-              old_body = response.body.first
-
-              new_body = if block.arity == 2
-                block.call(response,old_body)
-              else
-                block.call(old_body)
-              end
-
-              response.body[0] = (new_body || old_body)
-            end
-
-            halt(response)
-          end
-
-          #
-          # Proxies the current request.
-          #
-          # @param [Hash] options
-          #   Additional options to use when proxying the request.
-          #
-          # @yield [(response), doc]
-          #   If a block is given, it will be passed the optional response
-          #   of the proxied request and the document representing the
-          #   proxied request.
-          #
-          # @yieldparam [Net::HTTP::Response] response
-          #   The response of the proxied request.
-          #
-          # @yieldparam [Nokogiri::HTML, Nokogiri::XML] doc
-          #   The document representing the proxied request.
-          #
-          # @example
-          #   get '/login.php' do
-          #     proxy do |doc|
-          #       doc.search('form/@action').each do |action|
-          #         action.inner_text = action.inner_text.gsub(
-          #           /^https/, 'http'
-          #         )
-          #       end
-          #     end
-          #   end
-          #
-          # @since 0.2.0
-          #
-          def proxy_doc(options={},&block)
-            proxy(options) do |response,body|
-              case response.content_type
-              when 'text/html'
-                doc = Nokogiri::HTML(body)
-              when 'text/xml'
-                doc = Nokogiri::XML(body)
-              else
-                doc = nil
-              end
-
-              if doc
-                if block
-                  if block.arity == 2
-                    block.call(response,doc)
-                  else
-                    block.call(doc)
-                  end
-                end
-
-                doc.to_s
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end