ronin-web 0.3.0.rc1 → 1.0.0.beta1

data/lib/ronin/web/middleware/helpers.rb

@@ -1,145 +0,0 @@
-#
-# Ronin Web - A Ruby library for Ronin that provides support for web
-# scraping and spidering functionality.
-#
-# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
-#
-# This file is part of Ronin Web.
-#
-# Ronin is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ronin is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ronin.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-require 'ronin/web/middleware/request'
-require 'ronin/web/middleware/response'
-
-require 'rack'
-
-module Ronin
-  module Web
-    module Middleware
-      #
-      # A module containing helper methods that can be used in Rack
-      # Middleware and Rack Apps.
-      #
-      module Helpers
-        include Rack::Utils
-
-        alias h escape_html
-
-        #
-        # Sanitizes a path received by the middleware.
-        #
-        # @param [String] path
-        #   The unsanitized path.
-        #
-        # @return [String]
-        #   The unescaped and absolute path.
-        #
-        # @since 0.3.0
-        #
-        # @api semipublic
-        #
-        def sanitize_path(path)
-          File.expand_path(unescape(path))
-        end
-
-        #
-        # Returns the MIME type for a path.
-        #
-        # @param [String] path
-        #   The path to determine the MIME type for.
-        #
-        # @return [String]
-        #   The MIME type for the path.
-        #
-        # @since 0.3.0
-        #
-        # @api semipublic
-        #
-        def mime_type_for(path)
-          Rack::Mime.mime_type(File.extname(path))
-        end
-
-        alias content_type_for mime_type_for
-
-        #
-        # Creates a new response.
-        #
-        # @param [String, Array, IO] body
-        #   The body for the response.
-        #
-        # @param [Hash] headers
-        #   Additional headers for the response.
-        #
-        # @param [Integer] status
-        #   The HTTP Status Code for the response.
-        #
-        # @yield [response]
-        #   If a block is given, it will be passed the new response.
-        #
-        # @yieldparam [Response] response
-        #   The new response.
-        #
-        # @return [Array]
-        #   The new response.
-        #
-        # @example Create a response.
-        #   response ['Hello'], {'Content-Type' => 'text/txt'}, 200
-        #
-        # @example Create a response with just a String.
-        #   response 'Hello'
-        #
-        # @since 0.3.0
-        #
-        # @api semipublic
-        #
-        def response(body=[],headers={},status=200)
-          response = Response.new(body,status,headers)
-
-          yield(response) if block_given?
-          return response
-        end
-
-        #
-        # Creates a new response for a file.
-        #
-        # @param [String] path
-        #   The path to the file.
-        #
-        # @param [Hash] headers
-        #   Additional headers for the response.
-        #
-        # @param [Integer] status
-        #   The HTTP Status Code for the response.
-        #
-        # @return [Response]
-        #   The new response object.
-        #
-        # @see #response
-        #
-        # @since 0.3.0
-        #
-        # @api semipublic
-        #
-        def response_for(path,headers={},status=200)
-          response(
-            File.new(path),
-            headers.merge('Content-Type' => mime_type_for(path)),
-            status
-          )
-        end
-      end
-    end
-  end
-end

data/lib/ronin/web/middleware/proxy.rb

@@ -1,265 +0,0 @@
-#
-# Ronin Web - A Ruby library for Ronin that provides support for web
-# scraping and spidering functionality.
-#
-# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
-#
-# This file is part of Ronin Web.
-#
-# Ronin is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ronin is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ronin.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-require 'ronin/web/middleware/base'
-require 'ronin/web/middleware/rule'
-require 'ronin/web/middleware/proxy_request'
-
-require 'ronin/network/http'
-require 'set'
-
-module Ronin
-  module Web
-    module Middleware
-      #
-      # A Rack middleware for proxying requests.
-      #
-      #     use Ronin::Web::Middleware::Proxy do |proxy|
-      #       proxy.every_request do |request|
-      #         puts request.url
-      #       end
-      #
-      #       proxy.every_response do |response|
-      #         response.headers.each do |name,value|
-      #           puts "#{name}: #{value}"
-      #         end
-      #
-      #         puts response.body
-      #       end
-      #     end
-      #
-      class Proxy < Base
-
-        # Blacklisted HTTP response Headers.
-        HEADERS_BLACKLIST = Set[
-          'Transfer-Encoding'
-        ]
-
-        #
-        # Creates a new {Proxy} middleware.
-        #
-        # @param [#call] app
-        #   The application that the proxy middleware sits in front of.
-        #
-        # @param [Hash] options
-        #   Additional options.
-        #
-        # @option options [String] :campaign
-        #   The name of the campaign who's targetted hosts will be
-        #   filtered by.
-        #
-        # @option options [String, Regexp] :vhost
-        #   The Virtual-Host to filter.
-        #
-        # @option options [String, IPAddr] :ip
-        #   The IP address or IP range to filter.
-        #
-        # @option options [String, Regexp] :referer
-        #   The Referer URL or pattern to filter.
-        #
-        # @option options [String, Regexp] :user_agent
-        #   The User-Agent string to filter.
-        #
-        # @option options [Proc] :when
-        #   Custom logic to filter requests by.
-        #
-        # @yield [proxy]
-        #   If a block is given, it will be passed the new proxy middleware.
-        #
-        # @yieldparam [Proxy] proxy
-        #   The new proxy middleware object.
-        #
-        # @since 0.3.0
-        #
-        # @api public
-        #
-        def initialize(app,options={},&block)
-          intercept(options)
-
-          super(app,options,&block)
-        end
-
-        #
-        # Specifies which requests will be intercepted.
-        #
-        # @param [Hash] options
-        #   Additional options.
-        #
-        # @option options [String] :campaign
-        #   The name of the campaign who's targetted hosts will be
-        #   filtered by.
-        #
-        # @option options [String, Regexp] :vhost
-        #   The Virtual-Host to filter.
-        #
-        # @option options [String, IPAddr] :ip
-        #   The IP address or IP range to filter.
-        #
-        # @option options [String, Regexp] :referer
-        #   The Referer URL or pattern to filter.
-        #
-        # @option options [String, Regexp] :user_agent
-        #   The User-Agent string to filter.
-        #
-        # @option options [Proc] :when
-        #   Custom logic to filter requests by.
-        #
-        # @since 0.3.0
-        #
-        # @api public
-        #
-        def intercept(options={})
-          @rule = Rule.new(options)
-        end
-
-        #
-        # Uses the given block to intercept incoming requests.
-        #
-        # @yield [request]
-        #   The given block will receive every incoming request, before it
-        #   is proxied.
-        #
-        # @yieldparam [ProxyRequest] request
-        #   A proxied request.
-        #
-        # @return [Proxy]
-        #   The proxy middleware.
-        #
-        # @since 0.3.0
-        #
-        # @api public
-        #
-        def every_request(&block)
-          @every_request_block = block
-          return self
-        end
-
-        #
-        # Uses the given block to intercept proxied responses.
-        #
-        # @yield [response]
-        #   The given block will receive every proxied response.
-        #
-        # @yieldparam [Response] response
-        #   A proxied response.
-        #
-        # @return [Proxy]
-        #   The proxy middleware.
-        #
-        # @since 0.3.0
-        #
-        # @api public
-        #
-        def every_response(&block)
-          @every_response_block = block
-          return self
-        end
-
-        #
-        # Receives incoming requests, proxies them, allowing manipulation
-        # of the requests and their responses.
-        #
-        # @param [Hash, Rack::Request] env
-        #   The request.
-        #
-        # @return [Array, Response]
-        #   The response.
-        #
-        # @since 0.3.0
-        #
-        # @api public
-        #
-        def call(env)
-          request = ProxyRequest.new(env)
-
-          if @rule.match?(request)
-            @every_request_block.call(request) if @every_request_block
-          else
-            return super(env)
-          end
-
-          print_info "Proxying #{request.url} for #{request.address}"
-          request.headers.each do |name,value|
-            print_debug "  #{name}: #{value}"
-          end
-
-          response = proxy(request)
-
-          @every_response_block.call(response) if @every_response_block
-
-          print_info "Returning proxied response for #{request.address}"
-          response.headers.each do |name,value|
-            print_debug "  #{name}: #{value}"
-          end
-
-          return response
-        end
-
-        protected
-
-        #
-        # Proxies a request.
-        #
-        # @param [ProxyRequest] request
-        #   The request to send.
-        #
-        # @return [Response]
-        #   The response from the request.
-        #
-        # @api private
-        #
-        def proxy(request)
-          options = {
-            :ssl => (request.scheme == 'https'),
-            :host => request.host,
-            :port => request.port,
-            :method => request.request_method,
-            :path => request.path_info,
-            :query => request.query_string,
-            :content_type => request.content_type,
-            :headers => request.headers
-          }
-
-          if request.form_data?
-            options[:form_data] = request.POST
-          end
-
-          http_response = Net.http_request(options)
-          http_headers = {}
-
-          http_response.each_capitalized do |name,value|
-            unless HEADERS_BLACKLIST.include?(name)
-              http_headers[name] = value
-            end
-          end
-
-          return Response.new(
-            (http_response.body || ''),
-            http_response.code,
-            http_headers
-          )
-        end
-
-      end
-    end
-  end
-end

data/lib/ronin/web/middleware/proxy_request.rb

@@ -1,262 +0,0 @@
-#
-# Ronin Web - A Ruby library for Ronin that provides support for web
-# scraping and spidering functionality.
-#
-# Copyright (c) 2006-2011 Hal Brodigan (postmodern.mod3 at gmail.com)
-#
-# This file is part of Ronin Web.
-#
-# Ronin is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ronin is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ronin.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-require 'ronin/web/middleware/request'
-
-module Ronin
-  module Web
-    module Middleware
-      class ProxyRequest < Request
-
-        #
-        # Changes the HTTP Host header of the request.
-        #
-        # @param [String] new_host
-        #   The new value of the HTTP Host header.
-        #
-        # @return [String]
-        #   The new HTTP Host header.
-        #
-        # @since 0.3.0
-        #
-        # @api public
-        #
-        def host=(new_host)
-          @env['HTTP_HOST'] = new_host
-        end
-
-        #
-        # Changes the port the request is being sent to.
-        #
-        # @param [Integer] new_port
-        #   The new port the request will be sent to.
-        #
-        # @return [Integer]
-        #   The new port the request will be sent to.
-        #
-        # @since 0.3.0
-        #   
-        # @api public
-        #
-        def port=(new_port)
-          @env['SERVER_PORT'] = new_port
-        end
-
-        #
-        # Changes the URI scheme of the request.
-        #
-        # @param [String] new_port
-        #   The new URI scheme for the request.
-        #
-        # @return [String]
-        #   The new URI scheme of the request.
-        #
-        # @since 0.3.0
-        #   
-        # @api public
-        #
-        def scheme=(new_scheme)
-          @env['rack.url_scheme'] = new_scheme
-        end
-
-        #
-        # Causes the request to be proxied over SSL.
-        #
-        # @return [ProxyRequest]
-        #   The proxy request.
-        #
-        # @since 0.3.0
-        #
-        # @api public
-        #
-        def ssl!
-          self.port = 443
-          self.scheme = 'https'
-
-          return self
-        end
-
-        #
-        # Changes the HTTP Request Method of the request.
-        #
-        # @param [String] new_method
-        #   The new HTTP Request Method.
-        #
-        # @return [String]
-        #   The new HTTP Request Method.
-        #
-        # @since 0.3.0
-        #
-        # @api public
-        #
-        def request_method=(new_method)
-          @env['REQUEST_METHOD'] = new_method
-        end
-
-        alias path= path_info=
-
-        #
-        # Changes the HTTP Query String of the request.
-        #
-        # @param [String] new_query
-        #   The new HTTP Query String for the request.
-        #
-        # @return [String]
-        #   The new HTTP Query String of the request.
-        #
-        # @since 0.3.0
-        #
-        # @api public
-        #
-        def query_string=(new_query)
-          @env['QUERY_STRING'] = new_query
-        end
-
-        #
-        # Determines if the request is an XML-HTTP Request.
-        #
-        # @return [Boolean]
-        #   Specifies whether the request is XML-HTTP.
-        #
-        # @since 0.3.0
-        #
-        # @api public
-        #
-        def xhr?
-          @env['HTTP_X_REQUESTED_WITH'] == 'XMLHttpRequest'
-        end
-
-        #
-        # Specifies that the request is a XML HTTP Request.
-        #
-        # @yield [request]
-        #   If a block is given, it will be passed the request
-        #   for further modification.
-        #
-        # @yieldparam [ProxyRequest] request
-        #   The proxy request.
-        #
-        # @return [ProxyRequest]
-        #   The request.
-        #
-        # @since 0.3.0
-        #
-        # @api public
-        #
-        def xhr!
-          @env['HTTP_X_REQUESTED_WITH'] = 'XMLHttpRequest'
-
-          yield self if block_given?
-          return self
-        end
-
-        #
-        # Changes the HTTP Content-Type header of the request.
-        #
-        # @param [String] new_content_type
-        #   The new HTTP Content-Type for the request.
-        #
-        # @return [String]
-        #   The new HTTP Content-Type of the request.
-        #
-        # @since 0.3.0
-        #
-        # @api public
-        #
-        def content_type=(new_content_type)
-          @env['CONTENT_TYPE'] = new_content_type
-        end
-
-        #
-        # Changes the HTTP Accept-Encoding header of the request.
-        #
-        # @param [String] new_encoding
-        #   The new HTTP Accept-Encoding for the request.
-        #
-        # @return [String]
-        #   The new HTTP Accept-Encoding of the request.
-        #
-        # @since 0.3.0
-        #
-        # @api public
-        #
-        def accept_encoding=(new_encoding)
-          @env['HTTP_ACCEPT_ENCODING'] = new_encoding
-        end
-
-        #
-        # Sets the HTTP User-Agent header of the request.
-        #
-        # @param [String] new_user_agent
-        #   The new User-Agent header to use.
-        #
-        # @return [String]
-        #   The new User-Agent header.
-        #
-        # @since 0.3.0
-        #
-        # @api public
-        #
-        def user_agent=(new_user_agent)
-          @env['HTTP_USER_AGENT'] = new_user_agent
-        end
-
-        #
-        # Changes the HTTP Referer header of the request.
-        #
-        # @param [String] new_referer
-        #   The new HTTP Referer for the request.
-        #
-        # @return [String]
-        #   The new HTTP Referer of the request.
-        #
-        # @since 0.3.0
-        #
-        # @api public
-        #
-        def referer=(new_referer)
-          @env['HTTP_REFERER'] = new_referer
-        end
-
-        alias referrer= referer=
-
-        #
-        # Changes the body of the request.
-        #
-        # @param [String] new_body
-        #   The new body for the request.
-        #
-        # @return [String]
-        #   The new body of the request.
-        #
-        # @since 0.3.0
-        #
-        # @api public
-        #
-        def body=(new_body)
-          @env['rack.input'] = new_body
-        end
-
-      end
-    end
-  end
-end