ronin-web 0.1.1 → 0.1.2

data/History.txt

@@ -1,3 +1,13 @@
+=== 0.1.2 / 2009-03-28
+
+* Added Ronin::Web::Proxy.
+* Added diagnostic messages to Ronin::Web::Spider.
+* Fixed Rack 0.9.1 compatibility bugs in Ronin::Web::Server.
+  * Server#response now uses Rack::Response.
+  * Ronin::Web::Server now uses a Rack::Request object instead of the
+    standard env Hash.
+* Updated specs for Ronin::Web::Server.
+
 === 0.1.1 / 2009-02-23
 
 * Added a git style sub-command (<tt>ronin-web</tt>) which starts the Ronin

data/Manifest.txt

@@ -14,6 +14,7 @@ lib/ronin/web/extensions/nokogiri/xml/attr.rb
 lib/ronin/web/extensions/nokogiri/xml/element.rb
 lib/ronin/web/extensions/nokogiri/xml/document.rb
 lib/ronin/web/server.rb
+lib/ronin/web/proxy.rb
 lib/ronin/web/spider.rb
 lib/ronin/web/web.rb
 lib/ronin/web/version.rb

data/README.txt

@@ -41,8 +41,9 @@ of Ronin.
 
 * Web access (utilizing Mechanize and Nokogiri).
 * Integrates Spidr into Ronin::Web::Spider.
-* Provides Ronin::Web::Server, a customizable Rack web server that supports
-  path and host-name routing.
+* Provides Ronin::Web::Server, a customizable Rack based Web Server that
+  supports path and host-name routing.
+* Provides Ronin::Web::Proxy, a configurable Rack based Web Proxy.
 
 == REQUIREMENTS:
 
@@ -50,7 +51,7 @@ of Ronin.
 * {mechanize}[http://mechanize.rubyforge.org/] >= 0.9.0
 * {spidr}[http://spidr.rubyforge.org/] >= 0.1.3
 * {rack}[http://rack.rubyforge.org/] >= 0.9.1
-* {ronin}[http://ronin.rubyforge.org/] >= 0.1.4
+* {ronin}[http://ronin.rubyforge.org/] >= 0.2.2
 
 == INSTALL:
 

data/Rakefile

@@ -14,7 +14,7 @@ Hoe.new('ronin-web', Ronin::Web::VERSION) do |p|
     ['mechanize', '>=0.9.0'],
     ['spidr', '>=0.1.3'],
     ['rack', '>=0.9.1'],
-    ['ronin', '>=0.2.1']
+    ['ronin', '>=0.2.2']
   ]
 end
 

data/bin/ronin-web

@@ -2,6 +2,11 @@
 
 require 'rubygems'
 
+lib_dir = File.expand_path(File.join(File.dirname(__FILE__),'..','lib'))
+unless $LOAD_PATH.include?(lib_dir)
+  $LOAD_PATH << lib_dir
+end
+
 require 'ronin/ui/command_line/commands/console'
 require 'ronin/ui/console'
 

data/lib/ronin/web/proxy.rb

@@ -0,0 +1,124 @@
+#
+#--
+# Ronin Web - A Ruby library for Ronin that provides support for web
+# scraping and spidering functionality.
+#
+# Copyright (c) 2006-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#++
+#
+
+require 'ronin/web/server'
+require 'ronin/network/http'
+
+require 'net/http'
+
+module Ronin
+  module Web
+    class Proxy < Server
+
+      # The default HTTP Request to use
+      DEFAULT_HTTP_REQUEST = Net::HTTP::Get
+
+      #
+      # Creates a new Web Proxy using the given configuration _block_.
+      #
+      # _options_ may contain the following keys:
+      # <tt>:host</tt>:: The host to bind to.
+      # <tt>:port</tt>:: The port to listen on.
+      # <tt>:config</tt>:: A +Hash+ of configurable variables to be used
+      #                    in responses.
+      #
+      def initialize(options={},&block)
+        super(options)
+
+        @default = method(:proxy)
+
+        instance_eval(&block) if block
+      end
+
+      #
+      # Proxies the specified Rack _request_ and returns a corresponding
+      # Rack response.
+      #
+      def proxy(request)
+        server_response = http_response(request)
+        server_headers = server_response.to_hash
+        body = (server_response.body || '')
+
+        return response(body,server_headers)
+      end
+
+      protected
+
+      #
+      # Returns the Net::HTTP Request class that represents the specified
+      # HTTP _request_method_.
+      #
+      #   http_class('POST')
+      #   # => Net::HTTP::Post
+      #
+      def http_class(request_method)
+        http_method = request_method.downcase.capitalize
+        http_class = DEFAULT_HTTP_REQUEST
+
+        if Net::HTTP.const_defined?(http_method)
+          http_class = Net::HTTP.const_get(http_method)
+
+          unless http_class.kind_of?(Net::HTTPRequest)
+            http_class = DEFAULT_HTTP_REQUEST
+          end
+        end
+
+        return http_class
+      end
+
+      #
+      # Converts the Rack headers within the specified _request_ to
+      # Net::HTTP formatted HTTP headers.
+      #
+      def http_headers(request)
+        client_headers = {}
+
+        request.env.each do |name,value|
+          if name =~ /^HTTP_/
+            header_name = name.gsub(/^HTTP_/,'').split('_').map { |word|
+              word.capitalize
+            }.join('-')
+
+            client_headers[header_name] = value
+          end
+        end
+
+        return client_headers
+      end
+
+      #
+      # Returns the Net::HTTP response for the specified Rack _request_.
+      # 
+      def http_response(request)
+        path = request.fullpath
+        http_method = http_class(request.request_method)
+        client_request = http_method.new(path,http_headers(request))
+
+        Net.http_session(:host => request.host, :port => request.port) do |http|
+          return http.request(client_request)
+        end
+      end
+
+    end
+  end
+end

data/lib/ronin/web/server.rb

@@ -206,8 +206,8 @@ module Ronin
       #
       # Returns the HTTP 404 Not Found message for the requested path.
       #
-      def not_found(env)
-        path = env['PATH_INFO']
+      def not_found(request)
+        path = request.path_info
         body = %{<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
 <html>
   <head>
@@ -228,14 +228,14 @@ module Ronin
       # an index file. If no index file can be found or _path_ points to a
       # non-existant file, a "404 Not Found" response will be returned.
       #
-      def return_file(path,env)
+      def return_file(path,request)
         if !(File.exists?(path))
-          return not_found(env)
+          return not_found(request)
         end
 
         if File.directory?(path)
           unless (path = index_of(path))
-            return not_found(env)
+            return not_found(request)
           end
         end
 
@@ -251,8 +251,8 @@ module Ronin
       #
       #   response("<data>lol</data>", :content_type => 'text/xml')
       #
-      def response(body=[],options={},&block)
-        status = (options.delete(:status) || 200)
+      def response(body='',options={},&block)
+        status = (options.delete(:status) || options.delete('Status') || 200)
         headers = {}
 
         options.each do |name,value|
@@ -263,7 +263,7 @@ module Ronin
           headers[header_name] = value.to_s
         end
 
-        return Rack::Response.new(body,status,headers,&block)
+        return Rack::Response.new(body,status,headers,&block).finish
       end
 
       #
@@ -288,7 +288,7 @@ module Ronin
       # Use the given _server_ or _block_ as the default route for all
       # other requests.
       #
-      #   default do |env|
+      #   default do |request|
       #     [200, {'Content-Type' => 'text/html'}, 'lol train']
       #   end
       #
@@ -302,7 +302,7 @@ module Ronin
       # requests to host names which match the specified _pattern_.
       #
       #   hosts_like(/^a[0-9]\./) do
-      #     map('/download/') do |env|
+      #     map('/download/') do |request|
       #       ...
       #     end
       #   end
@@ -315,7 +315,7 @@ module Ronin
       # Registers the given _server_ or _block_ to be called when receiving
       # requests for paths which match the specified _pattern_.
       #
-      #   paths_like(/\.xml$/) do |env|
+      #   paths_like(/\.xml$/) do |request|
       #     ...
       #   end
       #
@@ -340,7 +340,7 @@ module Ronin
       #
       # Binds the specified URL _path_ to the given _server_ or _block_.
       #
-      #   bind '/secrets.xml' do |env|
+      #   bind '/secrets.xml' do |request|
       #     [200, {'Content-Type' => 'text/xml'}, "Made you look."]
       #   end
       #
@@ -353,7 +353,7 @@ module Ronin
       # Binds the specified URL directory _path_ to the given 
       # _server_ or _block_.
       #
-      #   map '/downloads' do |env|
+      #   map '/downloads' do |request|
       #     response(
       #       "Your somewhere inside the downloads directory",
       #       :content_type' => 'text/xml'
@@ -375,11 +375,11 @@ module Ronin
         file = File.expand_path(file)
         content_type = (options[:content_type] || content_type_for(file))
 
-        bind(path) do |env|
+        bind(path) do |request|
           if File.file?(file)
-            return_file(file,env)
+            return_file(file,request)
           else
-            not_found(env)
+            not_found(request)
           end
         end
       end
@@ -394,14 +394,14 @@ module Ronin
         sub_dirs = path.split('/')
         directory = File.expand_path(directory)
 
-        map(path) do |env|
-          http_path = File.expand_path(env['PATH_INFO'])
+        map(path) do |request|
+          http_path = File.expand_path(request.path_info)
           http_dirs = http_path.split('/')
 
           sub_path = http_dirs[sub_dirs.length..-1].join('/')
           absolute_path = File.join(directory,sub_path)
 
-          return_file(absolute_path,env)
+          return_file(absolute_path,request)
         end
       end
 
@@ -419,6 +419,7 @@ module Ronin
       def call(env)
         http_host = env['HTTP_HOST']
         http_path = File.expand_path(env['PATH_INFO'])
+        request = Rack::Request.new(env)
 
         if http_host
           if (server = virtual_host(http_host))
@@ -428,12 +429,12 @@ module Ronin
 
         if http_path
           if (block = @paths[http_path])
-            return block.call(env)
+            return block.call(request)
           end
 
           @path_patterns.each do |pattern,block|
             if http_path.match(pattern)
-              return block.call(env)
+              return block.call(request)
             end
           end
 
@@ -446,11 +447,11 @@ module Ronin
           }.sort.last
 
           if (sub_dir && (block = @directories[sub_dir]))
-            return block.call(env)
+            return block.call(request)
           end
         end
 
-        return @default.call(env)
+        return @default.call(request)
       end
 
       #

data/lib/ronin/web/spider.rb

@@ -22,12 +22,16 @@
 #
 
 require 'ronin/web/web'
+require 'ronin/ui/diagnostics'
 
 require 'spidr/agent'
 
 module Ronin
   module Web
     class Spider < Spidr::Agent
+
+      include UI::Diagnostics
+
       #
       # Creates a new Spider object with the given _options_ and
       # _block_. If a _block_ is given, it will be passed the newly created
@@ -52,28 +56,14 @@ module Ronin
       # <tt>:ignore_exts</tt>:: An +Array+ of File extension patterns to not
       #                         visit.
       #
-      def self.agent(options={},&block)
-        self.new(self.default_options.merge(options),&block)
-      end
+      def initialize(options={},&block)
+        super(default_options.merge(options))
 
-      #
-      # Creates a new Spider object with the given _options_ and will begin
-      # spidering the specified host _name_. If a _block_ is given it
-      # will be passed the newly created Spider object, before the agent
-      # begins spidering.
-      #
-      def self.host(name,options={},&block)
-        super(name,self.default_options.merge(options),&block)
-      end
+        every_url do |url|
+          print_info("Spidering #{url}")
+        end
 
-      #
-      # Creates a new Spider object with the given _options_ and will begin
-      # spidering the host of the specified _url_. If a _block_ is
-      # given it will be passed the newly created Spider object, before
-      # the agent begins spidering.
-      #
-      def self.site(url,options={},&block)
-        super(url,self.default_options.merge(options),&block)
+        block.call(self) if block
       end
 
       protected
@@ -81,7 +71,7 @@ module Ronin
       #
       # Returns the default options for Spider.
       #
-      def self.default_options
+      def default_options
         {:proxy => Web.proxy, :user_agent => Web.user_agent}
       end
     end

data/lib/ronin/web/version.rb

@@ -24,6 +24,6 @@
 module Ronin
   module Web
     # Ronin Web Version
-    VERSION = '0.1.1'
+    VERSION = '0.1.2'
   end
 end

data/spec/spec_helper.rb

@@ -1,5 +1,5 @@
 require 'rubygems'
-gem 'rspec', '>=1.1.3'
+gem 'rspec', '>=1.1.12'
 require 'spec'
 
 require 'ronin/web/version'

data/spec/web/helpers/server.rb

@@ -1,2 +1,10 @@
 # Web Server root directory
 WEB_SERVER_ROOT = File.expand_path(File.join(File.dirname(__FILE__),'root'))
+
+def get_path(server,path)
+  server.route_path(path).last
+end
+
+def get_url(server,url)
+  server.route(url).last
+end

data/spec/web/server_spec.rb

@@ -75,68 +75,68 @@ describe Web::Server do
   it "should have a default response for un-matched paths" do
     path = '/test/default'
 
-    @server.route_path(path).body.should == ['This is default.']
+    get_path(@server,path).body.should == ['This is default.']
   end
 
   it "should bind a path to a certain response" do
     path = '/test/bind.xml'
 
-    @server.route_path(path).body.should == ['<secret/>']
+    get_path(@server,path).body.should == ['<secret/>']
   end
 
   it "should match paths with patterns" do
     path = '/test/path_patterns/secret.pdf'
 
-    @server.route_path(path).body.should == ['No secrets here.']
+    get_path(@server,path).body.should == ['No secrets here.']
   end
 
   it "should match paths to sub-directories" do
     path = '/test/map/impossible.html'
 
-    @server.route_path(path).body.should == ['mapped']
+    get_path(@server,path).body.should == ['mapped']
   end
 
   it "should return a response for a file" do
     path = '/test/file.txt'
 
-    @server.route_path(path).body.should == ["This is a test.\n"]
+    get_path(@server,path).body.should == ["This is a test.\n"]
   end
 
   it "should return files from bound directories" do
     path = '/test/directory/test.txt'
 
-    @server.route_path(path).body.should == ["This is a test.\n"]
+    get_path(@server,path).body.should == ["This is a test.\n"]
   end
 
   it "should return the index file for a bound directory" do
     path = '/test/directory/'
 
-    @server.route_path(path).body.should == ["Index of files.\n"]
+    get_path(@server,path).body.should == ["Index of files.\n"]
   end
 
   it "should match virtual hosts" do
     url = 'http://virtual.host.com/test/virtual_host.xml'
 
-    @server.route(url).body.should == ['<virtual/>']
+    get_url(@server,url).body.should == ['<virtual/>']
   end
 
   it "should match virtual hosts with patterns" do
     url = 'http://virtual0.host.com/test/virtual_host_patterns.xml'
 
-    @server.route(url).body.should == ['<virtual-patterns/>']
+    get_url(@server,url).body.should == ['<virtual-patterns/>']
   end
 
   it "should provide access to servers via their host-names" do
     virtual_host = @server.virtual_host('virtual.host.com')
     url = 'http://virtual.host.com/test/virtual_host.xml'
 
-    virtual_host.route(url).body.should == ['<virtual/>']
+    get_url(virtual_host,url).body.should == ['<virtual/>']
   end
 
   it "should provide access to servers via their host-names that match virtual host patterns" do
     virtual_host = @server.virtual_host('virtual1.host.com')
     url = 'http://virtual0.host.com/test/virtual_host_patterns.xml'
 
-    virtual_host.route(url).body.should == ['<virtual-patterns/>']
+    get_url(virtual_host,url).body.should == ['<virtual-patterns/>']
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: ronin-web
 version: !ruby/object:Gem::Version 
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors: 
 - Postmodern
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-02-23 00:00:00 -08:00
+date: 2009-03-28 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -60,7 +60,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 0.2.1
+        version: 0.2.2
     version: 
 - !ruby/object:Gem::Dependency 
   name: hoe
@@ -70,7 +70,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 1.8.3
+        version: 1.11.0
     version: 
 description: Ronin Web is a Ruby library for Ronin that provides support for web scraping and spidering functionality.  Ronin is a Ruby platform designed for information security and data exploration tasks. Ronin allows for the rapid development and distribution of code over many of the common Source-Code-Management (SCM) systems.
 email: 
@@ -83,7 +83,6 @@ extra_rdoc_files:
 - History.txt
 - Manifest.txt
 - README.txt
-- spec/web/helpers/root/test.txt
 files: 
 - History.txt
 - Manifest.txt
@@ -101,6 +100,7 @@ files:
 - lib/ronin/web/extensions/nokogiri/xml/element.rb
 - lib/ronin/web/extensions/nokogiri/xml/document.rb
 - lib/ronin/web/server.rb
+- lib/ronin/web/proxy.rb
 - lib/ronin/web/spider.rb
 - lib/ronin/web/web.rb
 - lib/ronin/web/version.rb