ronin-web 0.2.1 → 0.3.0.pre1

data/lib/ronin/web/middleware/rule.rb

@@ -0,0 +1,103 @@
+#
+# Ronin Web - A Ruby library for Ronin that provides support for web
+# scraping and spidering functionality.
+#
+# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# This file is part of Ronin Web.
+#
+# Ronin is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ronin is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ronin.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+require 'ronin/web/middleware/filters'
+
+module Ronin
+  module Web
+    module Middleware
+      #
+      # Matches requests against one or more filters.
+      #
+      class Rule
+
+        # Registered filters
+        FILTERS = {
+          :ip => Filters::IPFilter,
+          :campaign => Filters::CampaignFilter,
+          :path => Filters::PathFilter,
+          :vhost => Filters::VHostFilter,
+          :referer => Filters::RefererFilter,
+          :user_agent => Filters::UserAgentFilter
+        }
+
+        #
+        # Creates a new rule.
+        #
+        # @param [Hash] options
+        #   Additional filtering options for the rule.
+        #
+        # @option options [String] :campaign
+        #   The name of the campaign who's targetted hosts will be
+        #   filtered by.
+        #
+        # @option options [String, Regexp] :vhost
+        #   The Virtual-Host to filter.
+        #
+        # @option options [String, IPAddr] :ip
+        #   The IP address or IP range to filter.
+        #
+        # @option options [String, Regexp] :referer
+        #   The Referer URL or pattern to filter.
+        #
+        # @option options [String, Regexp] :user_agent
+        #   The User-Agent string to filter.
+        #
+        # @option options [Proc] :when
+        #   Custom logic to filter requests by.
+        #
+        # @since 0.3.0
+        #
+        # @api public
+        #
+        def initialize(options={},&block)
+          @filters = []
+          @when_block = options.delete(:when)
+
+          options.each do |name,value|
+            if FILTERS.has_key?(name)
+              @filters << FILTERS[name].new(value)
+            end
+          end
+        end
+
+        #
+        # Matches a request against the rule.
+        #
+        # @param [Rack::Request] request
+        #   The request to match.
+        #
+        # @return [Boolean]
+        #   Specifies if the request matches all of the filters.
+        #
+        # @api public
+        #
+        def match?(request)
+          @filters.all? { |filter| 
+            filter.match?(request)
+          } && (@when_block.nil? || (@when_block.call(request) ? true : false))
+        end
+
+      end
+    end
+  end
+end

data/lib/ronin/web/proxy.rb

@@ -2,21 +2,22 @@
 # Ronin Web - A Ruby library for Ronin that provides support for web
 # scraping and spidering functionality.
 #
-# Copyright (c) 2006-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
-# This program is free software; you can redistribute it and/or modify
+# This file is part of Ronin Web.
+#
+# Ronin is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
+# the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
-# This program is distributed in the hope that it will be useful,
+# Ronin is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+# along with Ronin.  If not, see <http://www.gnu.org/licenses/>.
 #
 
 require 'ronin/web/proxy/base'

data/lib/ronin/web/proxy/app.rb

@@ -2,21 +2,22 @@
 # Ronin Web - A Ruby library for Ronin that provides support for web
 # scraping and spidering functionality.
 #
-# Copyright (c) 2006-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
-# This program is free software; you can redistribute it and/or modify
+# This file is part of Ronin Web.
+#
+# Ronin is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
+# the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
-# This program is distributed in the hope that it will be useful,
+# Ronin is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+# along with Ronin.  If not, see <http://www.gnu.org/licenses/>.
 #
 
 require 'ronin/web/proxy/base'

data/lib/ronin/web/proxy/base.rb

@@ -2,28 +2,35 @@
 # Ronin Web - A Ruby library for Ronin that provides support for web
 # scraping and spidering functionality.
 #
-# Copyright (c) 2006-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
-# This program is free software; you can redistribute it and/or modify
+# This file is part of Ronin Web.
+#
+# Ronin is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
+# the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
-# This program is distributed in the hope that it will be useful,
+# Ronin is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+# along with Ronin.  If not, see <http://www.gnu.org/licenses/>.
 #
 
 require 'ronin/web/server/base'
+require 'ronin/web/middleware/proxy'
 
 module Ronin
   module Web
     module Proxy
+      #
+      # The base-class for all Ronin Web Proxies. Extends
+      # [Sinatra::Base](http://rubydoc.info/gems/sinatra/Sinatra/Base)
+      # with {Middleware::Proxy}.
+      #
       class Base < Server::Base
 
         # The default port to run the Web Proxy on
@@ -31,9 +38,7 @@ module Ronin
 
         set :port, DEFAULT_PORT
 
-        default do
-          proxy
-        end
+        use Middleware::Proxy
 
       end
     end

data/lib/ronin/web/proxy/web.rb

@@ -2,21 +2,22 @@
 # Ronin Web - A Ruby library for Ronin that provides support for web
 # scraping and spidering functionality.
 #
-# Copyright (c) 2006-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
-# This program is free software; you can redistribute it and/or modify
+# This file is part of Ronin Web.
+#
+# Ronin is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
+# the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
-# This program is distributed in the hope that it will be useful,
+# Ronin is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+# along with Ronin.  If not, see <http://www.gnu.org/licenses/>.
 #
 
 require 'ronin/web/proxy/app'
@@ -29,6 +30,8 @@ module Ronin
     #
     # @see Server::Base.run!
     #
+    # @api public
+    #
     def Web.proxy_server(options={},&block)
       unless class_variable_defined?('@@ronin_web_proxy')
         @@ronin_web_proxy = Proxy::App

data/lib/ronin/web/server.rb

@@ -2,21 +2,22 @@
 # Ronin Web - A Ruby library for Ronin that provides support for web
 # scraping and spidering functionality.
 #
-# Copyright (c) 2006-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
-# This program is free software; you can redistribute it and/or modify
+# This file is part of Ronin Web.
+#
+# Ronin is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
+# the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
-# This program is distributed in the hope that it will be useful,
+# Ronin is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+# along with Ronin.  If not, see <http://www.gnu.org/licenses/>.
 #
 
 require 'ronin/web/server/base'

data/lib/ronin/web/server/app.rb

@@ -2,21 +2,22 @@
 # Ronin Web - A Ruby library for Ronin that provides support for web
 # scraping and spidering functionality.
 #
-# Copyright (c) 2006-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
-# This program is free software; you can redistribute it and/or modify
+# This file is part of Ronin Web.
+#
+# Ronin is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
+# the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
-# This program is distributed in the hope that it will be useful,
+# Ronin is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+# along with Ronin.  If not, see <http://www.gnu.org/licenses/>.
 #
 
 require 'ronin/web/server/base'

data/lib/ronin/web/server/base.rb

@@ -2,33 +2,33 @@
 # Ronin Web - A Ruby library for Ronin that provides support for web
 # scraping and spidering functionality.
 #
-# Copyright (c) 2006-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
-# This program is free software; you can redistribute it and/or modify
+# This file is part of Ronin Web.
+#
+# Ronin is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
+# the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
-# This program is distributed in the hope that it will be useful,
+# Ronin is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+# along with Ronin.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-require 'ronin/web/server/helpers/rendering'
-require 'ronin/web/server/helpers/proxy'
-require 'ronin/web/server/files'
-require 'ronin/web/server/hosts'
-require 'ronin/static/finders'
+require 'ronin/web/middleware/helpers'
+require 'ronin/web/middleware/files'
+require 'ronin/web/middleware/directories'
+require 'ronin/web/middleware/router'
+require 'ronin/web/middleware/proxy'
 require 'ronin/templates/erb'
 require 'ronin/ui/output'
 require 'ronin/extensions/meta'
 
-require 'set'
 require 'thread'
 require 'rack'
 require 'sinatra'
@@ -36,15 +36,16 @@ require 'sinatra'
 module Ronin
   module Web
     module Server
+      #
+      # The base-class for all Ronin Web Servers. Extends
+      # [Sinatra::Base](http://rubydoc.info/gems/sinatra/Sinatra/Base)
+      # with additional helper methods and Rack {Middleware}.
+      #
       class Base < Sinatra::Base
 
-        include Static::Finders
-        include Rack::Utils
         include Templates::Erb
-        extend UI::Output
-
-        include Files
-        include Hosts
+        include UI::Output::Helpers
+        extend UI::Output::Helpers
 
         # Default interface to run the Web Server on
         DEFAULT_HOST = '0.0.0.0'
@@ -52,12 +53,6 @@ module Ronin
         # Default port to run the Web Server on
         DEFAULT_PORT = 8000
 
-        # Default list of index file-names to search for in directories
-        DEFAULT_INDICES = ['index.html', 'index.htm']
-
-        # Directory to search for views within
-        VIEWS_DIR = File.join('ronin','web','server','views')
-
         set :host, DEFAULT_HOST
         set :port, DEFAULT_PORT
 
@@ -69,6 +64,8 @@ module Ronin
         #
         # @since 0.2.0
         #
+        # @api public
+        #
         def Base.handler
           @@ronin_web_server_handler ||= nil
         end
@@ -84,35 +81,12 @@ module Ronin
         #
         # @since 0.2.0
         #
+        # @api public
+        #
         def Base.handler=(name)
           @@ronin_web_server_handler = name
         end
 
-        #
-        # The list of index files to search for when requesting the
-        # contents of a directory.
-        #
-        # @return [Set]
-        #   The names of index files.
-        #
-        # @since 0.2.0
-        #
-        def Base.indices
-          @@ronin_web_server_indices ||= Set[*DEFAULT_INDICES]
-        end
-
-        #
-        # Adds a new index to the +Base.indices+ list.
-        #
-        # @param [String, Symbol] name
-        #   The index name to add.
-        #
-        # @since 0.2.0
-        #
-        def Base.index(name)
-          Base.indices << name.to_s
-        end
-
         #
         # The list of Rack Handlers to attempt to use with the web server.
         #
@@ -121,6 +95,8 @@ module Ronin
         #
         # @since 0.2.0
         #
+        # @api public
+        #
         def self.handlers
           handlers = self.server
 
@@ -143,6 +119,8 @@ module Ronin
         #
         # @since 0.2.0
         #
+        # @api semipublic
+        #
         def self.handler_class
           self.handlers.find do |name|
             begin
@@ -154,12 +132,12 @@ module Ronin
             end
           end
 
-          raise(StandardError,"unable to find any Rack handlers",caller)
+          raise(StandardError,"unable to find any Rack handlers")
         end
 
         #
         # Run the web server using the Rack Handler returned by
-        # +handler_class+.
+        # {handler_class}.
         #
         # @param [Hash] options Additional options.
         #
@@ -175,6 +153,8 @@ module Ronin
         #
         # @since 0.2.0
         #
+        # @api public
+        #
         def self.run!(options={})
           rack_options = {
             :Host => (options[:host] || self.host),
@@ -223,6 +203,8 @@ module Ronin
         #
         # @since 0.2.0
         #
+        # @api public
+        #
         def self.any(path,options={},&block)
           get(path,options,&block)
           put(path,options,&block)
@@ -252,60 +234,208 @@ module Ronin
         #
         # @since 0.2.0
         #
+        # @api public
+        #
         def self.default(&block)
           class_def(:default_response,&block)
           return self
         end
 
+        #
+        # Hosts the contents of a file.
+        #
+        # @param [String] remote_path
+        #   The path the web server will host the file at.
+        #
+        # @param [String] local_path
+        #   The path to the local file.
+        #
+        # @example
+        #   file '/robots.txt', '/path/to/my_robots.txt'
+        #
+        # @see Middleware::Files
+        #
+        # @since 0.3.0
+        #
+        # @api public
+        #
+        def self.file(remote_path,local_path)
+          use Middleware::Files, :paths => {remote_path => local_path}
+        end
+
+        #
+        # Hosts the contents of files.
+        #
+        # @yield [files]
+        #   The given block will be passed the files middleware to
+        #   configure.
+        #
+        # @yieldparam [Middleware::Files]
+        #   The files middleware object.
+        #
+        # @example
+        #   files do |files|
+        #     files.map '/foo.txt', 'foo.txt'
+        #     files.map /\.exe$/, 'trojan.exe'
+        #   end
+        #
+        # @see Middleware::Files
+        #
+        # @since 0.3.0
+        #
+        # @api public
+        #
+        def self.files(&block)
+          use(Middleware::Files,&block)
+        end
+
+        #
+        # Hosts the contents of the directory.
+        #
+        # @param [String] remote_path
+        #   The path the web server will host the directory at.
+        #
+        # @param [String] local_path
+        #   The path to the local directory.
+        #
+        # @example
+        #   directory '/download/', '/tmp/files/'
+        #
+        # @see Middleware::Directories
+        #
+        # @since 0.2.0
+        #
+        # @api public
+        #
+        def self.directory(remote_path,local_path)
+          use Middleware::Directories, :paths => {remote_path => local_path}
+        end
+
+        #
+        # Hosts the contents of directories.
+        #
+        # @yield [dirs]
+        #   The given block will be passed the directories middleware to
+        #   configure.
+        #
+        # @yieldparam [Middleware::Directories]
+        #   The directories middleware object.
+        #
+        # @example
+        #   directories do |dirs|
+        #     dirs.map '/downloads', '/tmp/ronin_downloads'
+        #     dirs.map '/images', '/tmp/ronin_images'
+        #     dirs.map '/pdfs', '/tmp/ronin_pdfs'
+        #   end
+        #
+        # @see Middleware::Directories
+        #
+        # @since 0.3.0
+        #
+        # @api public
+        #
+        def self.directories(&block)
+          use(Middleware::Directories,&block)
+        end
+
+        #
+        # Hosts the static contents within a given directory.
+        #
+        # @param [String] path
+        #   The path to a directory to serve static content from.
+        #
+        # @example
+        #   public_dir 'path/to/another/public'
+        #
+        # @since 0.2.0
+        #
+        # @api public
+        #
+        def self.public_dir(path)
+          self.directory('/',path)
+        end
+
         #
         # Routes all requests within a given directory into another
         # web server.
         #
-        # @param [String] dir
+        # @param [String, Regexp] dir
         #   The directory that requests for will be routed from.
         #
-        # @param [Base, #call] server
+        # @param [#call] server
         #   The web server to route requests to.
         #
         # @example
-        #   MyApp.map '/subapp/', SubApp
+        #   map '/subapp/', SubApp
+        #
+        # @see Middleware::Router
         #
         # @since 0.2.0
         #
+        # @api public
+        #
         def self.map(dir,server)
-          dir = File.join(dir,'')
-
-          before do
-            if dir == request.path_info[0,dir.length]
-              # remove the dir from the beginning of the path
-              # before passing it to the server
-              request.env['PATH_INFO'] = request.path_info[dir.length-1..-1]
+          use Middleware::Router do |router|
+            router.draw :path => dir, :to => server
+          end
+        end
 
-              halt(*server.call(request.env))
-            end
+        #
+        # Routes requests with a specific Host header to another
+        # web server.
+        #
+        # @param [String, Regexp] name
+        #   The host-name to route requests for.
+        #
+        # @param [#call] server
+        #   The web server to route the requests to.
+        #
+        # @example
+        #   vhost 'cdn.evil.com', EvilServer
+        #
+        # @since 0.3.0
+        #
+        # @api public
+        #
+        def self.vhost(name,server)
+          use Middleware::Router do |router|
+            router.draw :vhost => name, :to => server
           end
         end
 
         #
-        # Hosts the static contents within a given directory.
+        # Proxies requests to a given path.
         #
-        # @param [String] directory
-        #   The path to a directory to serve static content from.
+        # @param [String] path
+        #   The path to proxy requests for.
+        #
+        # @param [Hash] options
+        #   Additional options.
+        # 
+        # @yield [(response), body]
+        #   If a block is given, it will be passed the optional
+        #   response of the proxied request and the body received
+        #   from the proxied request.
+        #
+        # @yieldparam [Net::HTTP::Response] response
+        #   The response.
+        #
+        # @yieldparam [String] body
+        #   The body from the response.
         #
         # @example
-        #   MyApp.public_dir 'path/to/another/public'
+        #   proxy '/login.php' do |response,body|
+        #     body.gsub(/https/,'http')
+        #   end
+        #
+        # @see Middleware::Proxy
         #
         # @since 0.2.0
         #
-        def self.public_dir(directory)
-          directory = File.expand_path(directory)
-
-          before do
-            sub_path = File.expand_path(File.join('',request.path_info))
-            full_path = File.join(directory,sub_path)
-
-            return_file(full_path) if File.file?(full_path)
-          end
+        # @api public
+        #
+        def self.proxy(path,options={},&block)
+          use(Middleware::Proxy,options,&block)
         end
 
         protected
@@ -315,18 +445,15 @@ module Ronin
         #
         # @since 0.2.0
         #
+        # @api semipublic
+        #
         def default_response
           halt 404, ''
         end
 
         enable :sessions
 
-        helpers Helpers::Rendering
-        helpers Helpers::Proxy
-
-        not_found do
-          default_response
-        end
+        any('*') { default_response }
 
       end
     end