ronin-web 0.2.0 → 0.2.1

data/History.txt

@@ -1,3 +1,11 @@
+=== 0.2.1 / 2009-10-18
+
+* Require spidr >= 0.2.0.
+* Added Ronin::Scanners::Web.
+* Added more specs for Ronin::Web and Ronin::Scanners::Web.
+* Renamed Ronin::Web.proxy to Ronin::Web.proxy_server, to not conflict
+  with the original Ronin::Web.proxy method.
+
 === 0.2.0 / 2009-09-24
 
 * Require ronin >= 0.3.0.

data/Manifest.txt

@@ -34,12 +34,14 @@ lib/ronin/web/spider.rb
 lib/ronin/web/web.rb
 lib/ronin/web/version.rb
 lib/ronin/network/helpers/web.rb
+lib/ronin/scanners/web.rb
 tasks/spec.rb
 tasks/yard.rb
 spec/spec_helper.rb
 spec/web/helpers/server.rb
 spec/web/helpers/root/index.html
 spec/web/helpers/root/test.txt
+spec/web/web_spec.rb
 spec/web/extensions/nokogiri_spec.rb
 spec/web/server/classes/public1/static1.txt
 spec/web/server/classes/public2/static2.txt
@@ -59,3 +61,4 @@ spec/web/server/files_spec.rb
 spec/web/server/hosts_spec.rb
 spec/web/server/proxy_spec.rb
 spec/web/proxy/base_spec.rb
+spec/scanners/web_spec.rb

data/README.txt

@@ -53,11 +53,11 @@ install or update Overlays.
 
 * Get a web-page:
 
-    Web.get('http://www.example.com/')
+    Web.get('http://www.rubyinside.com/')
 
 * Get only the body of the web-page:
 
-    Web.get_body('http://www.example.com/')
+    Web.get_body('http://www.rubyinside.com/')
 
 * Get a WWW::Mechanize agent:
 
@@ -151,7 +151,7 @@ install or update Overlays.
 == REQUIREMENTS:
 
 * {mechanize}[http://mechanize.rubyforge.org/] >= 0.9.3
-* {spidr}[http://spidr.rubyforge.org/] >= 0.1.9
+* {spidr}[http://spidr.rubyforge.org/] >= 0.2.0
 * {sinatra}[http://www.sinatrarb.com/] >= 0.9.4
 * {ronin}[http://ronin.rubyforge.org/] >= 0.3.0
 

data/Rakefile

@@ -12,7 +12,7 @@ Hoe.spec('ronin-web') do
   self.remote_rdoc_dir = 'docs/ronin-web'
   self.extra_deps = [
     ['mechanize', '>=0.9.3'],
-    ['spidr', '>=0.1.9'],
+    ['spidr', '>=0.2.0'],
     ['sinatra', '>=0.9.4'],
     ['ronin', '>=0.3.0']
   ]

data/lib/ronin/scanners/web.rb

@@ -0,0 +1,52 @@
+#
+# Ronin Web - A Ruby library for Ronin that provides support for web
+# scraping and spidering functionality.
+#
+# Copyright (c) 2006-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#
+
+require 'ronin/web/spider'
+require 'ronin/scanners/scanner'
+
+module Ronin
+  module Scanners
+    class Web < Web::Spider
+
+      include Scanner
+
+      protected
+
+      def each_target(&block)
+        print_info "Started web spidering ..."
+
+        history.clear
+
+        unless visit_hosts.empty?
+          enqueue("http://#{visit_hosts.first}/")
+        end
+
+        run do |page|
+          print_info "Scanning page: #{page.url}"
+          block.call(page)
+        end
+
+        print_info "Finished web spidering."
+      end
+
+    end
+  end
+end

data/lib/ronin/web/proxy/web.rb

@@ -25,10 +25,11 @@ module Ronin
   module Web
     #
     # Returns the Ronin Web Proxy. When called for the first time
-    # the proxy will be started in the background with the given
-    # _options_.
+    # the proxy will be started in the background.
     #
-    def Web.proxy(options={},&block)
+    # @see Server::Base.run!
+    #
+    def Web.proxy_server(options={},&block)
       unless class_variable_defined?('@@ronin_web_proxy')
         @@ronin_web_proxy = Proxy::App
         @@ronin_web_proxy.run!(options.merge(:background => true))

data/lib/ronin/web/version.rb

@@ -22,6 +22,6 @@
 module Ronin
   module Web
     # Ronin Web Version
-    VERSION = '0.2.0'
+    VERSION = '0.2.1'
   end
 end

data/lib/ronin/web/web.rb

@@ -61,9 +61,11 @@ module Ronin
     #
     # @example
     #   Web.build_html do
-    #     body {
-    #       div(:style => 'display:none;') {
-    #         object(:classid => 'blabla')
+    #     html {
+    #       body {
+    #         div(:style => 'display:none;') {
+    #           object(:classid => 'blabla')
+    #         }
     #       }
     #     }
     #   end
@@ -106,8 +108,8 @@ module Ronin
     # @example
     #  Web.build_xml do
     #    post(:id => 2) {
-    #      title { text("some example) }
-    #      body { text("this is one contrived example.") }
+    #      title { text('some example') }
+    #      body { text('this is one contrived example.') }
     #    }
     #  end
     #
@@ -130,15 +132,33 @@ module Ronin
     #
     # Creates a HTTP URI based on a Hash of proxy information.
     #
-    # @param [Network::HTTP::Proxy, Hash] proxy_info
+    # @param [Network::HTTP::Proxy, Hash, String] proxy_info
     #   The proxy information.
     #
     # @return [URI::HTTP, nil]
     #   The HTTP URI that represents the proxy. If the proxy is diabled,
     #   +nil+ will be returned.
     #
+    # @example
+    #   Web.proxy_url
+    #   # => "http://www.example.com:8080"
+    #
+    # @example
+    #   Web.proxy_url({:host => 'www.example.com', :port => 8081})
+    #   # => "http://www.example.com:8081"
+    #
+    # @example
+    #   Web.proxy_url('www.example.com:9000')
+    #   # => "http://www.example.com:9000"
+    #
     def Web.proxy_url(proxy_info=Web.proxy)
-      Network::HTTP::Proxy.new(proxy_info).url
+      proxy = if proxy_info.kind_of?(Hash)
+                Network::HTTP::Proxy.new(proxy_info)
+              else
+                Network::HTTP::Proxy.parse(proxy_info)
+              end
+
+      return proxy.url
     end
 
     #
@@ -195,7 +215,8 @@ module Ronin
     # @option options [String] :user_agent
     #   The User-Agent string to use.
     #
-    # @option options [Network::HTTP::Proxy, Hash] :proxy (Web.proxy)
+    # @option options [Network::HTTP::Proxy, Hash, String] :proxy
+    #   (Web.proxy)
     #   Proxy information.
     #
     # @option options [String] :user
@@ -216,7 +237,7 @@ module Ronin
     #   The contents of the URL.
     #
     # @example Open a given URL.
-    #   Web.open('http://www.hackety.org/')
+    #   Web.open('http://rubyflow.com/')
     #
     # @example Open a given URL, using a custom User-Agent alias.
     #   Web.open('http://tenderlovemaking.com/',
@@ -270,7 +291,8 @@ module Ronin
     # @option options [String] :user_agent
     #   The User-Agent string to use.
     #
-    # @option options [Network::HTTP::Proxy, Hash] :proxy (Web.proxy)
+    # @option options [Network::HTTP::Proxy, Hash, String] :proxy
+    #   (Web.proxy)
     #   Proxy information.
     #
     # @yield [agent]
@@ -305,7 +327,16 @@ module Ronin
         agent.user_agent = Web.user_agent
       end
 
-      proxy = (options[:proxy] || Web.proxy)
+      proxy = if options[:proxy].kind_of?(Hash)
+                options[:proxy]
+              elsif options[:proxy].kind_of?(String)
+                Network::HTTP::Proxy.parse(options[:proxy])
+              elsif options[:proxy].nil?
+                Web.proxy
+              else
+                raise(RuntimeError,"the given :proxy option is neither a Proxy, Hash or String",caller)
+              end
+
       if proxy[:host]
         agent.set_proxy(proxy[:host],proxy[:port],proxy[:user],proxy[:password])
       end
@@ -343,7 +374,7 @@ module Ronin
     #   The requested page.
     #
     # @example
-    #   Web.get('http://www.0x000000.com')
+    #   Web.get('http://www.rubyinside.com')
     #   # => WWW::Mechanize::Page
     #
     # @example

data/spec/scanners/web_spec.rb

@@ -0,0 +1,24 @@
+require 'ronin/scanners/web'
+
+require 'spec_helper'
+
+describe Scanners::Web do
+  before(:all) do
+    Scanners::Web.class_eval do
+      scanner(:test) do |page,results,options|
+        results.call(page.url)
+      end
+    end
+
+    @scanner = Scanners::Web.new(:host => 'www.example.com')
+  end
+
+  it "should spider every page on a website" do
+    @scanner.enqueue('http://www.example.com/')
+    @scanner.scan.should == {:test => [URI('http://www.example.com/')]}
+  end
+
+  it "should start spidering the first acceptable host" do
+    @scanner.scan.should == {:test => [URI('http://www.example.com/')]}
+  end
+end

data/spec/web/server/classes/proxy_app.rb

@@ -30,9 +30,9 @@ class ProxyApp < Ronin::Web::Server::Base
     end
   end
 
-  get '/rss.php' do
+  get '/feed/vulnerabilities/latest.rss' do
     proxy_doc do |response,doc|
-      for_host('milworm.com') do
+      for_host('osvdb.org') do
         doc.search('//item').each do |item|
           if item.inner_text =~ /(XSS|SQLi|SQL\s+Injection)/i
             item.remove

data/spec/web/server/proxy_spec.rb

@@ -41,7 +41,7 @@ describe Web::Server::Helpers::Proxy do
   end
 
   it "should allow modification of proxied XML documents" do
-    get_host '/rss.php', 'milw0rm.com'
+    get_host '/feed/vulnerabilities/latest.rss', 'osvdb.org'
 
     last_response.should be_ok
     last_response.body.should_not =~ /(XSS|SQLi|SQL\s+Injection)/i

data/spec/web/web_spec.rb

@@ -0,0 +1,167 @@
+require 'ronin/web/web'
+
+require 'spec_helper'
+
+describe Web do
+  it "should have a VERSION constant" do
+    Web.const_defined?('VERSION').should == true
+  end
+
+  it "should be able to parse HTML" do
+    doc = Web.html(%{
+      <html>
+        <body>Hello</body>
+      </html>
+    })
+
+    doc.at('body').inner_text.should == "Hello"
+  end
+
+  it "should be able to build HTML documents" do
+    doc = Web.build_html do
+      html {
+        body {
+          div { text("hello") }
+        }
+      }
+    end
+
+    doc.to_html.include?("<html><body><div>hello</div></body></html>").should == true
+  end
+
+  it "should be able to parse XML" do
+    doc = Web.html(%{
+      <?xml version="1.0"?>
+      <root>
+        <stuff>Hello</stuff>
+      </root>
+    })
+
+    doc.at('stuff').inner_text.should == "Hello"
+  end
+
+  it "should be able to build XML documents" do
+    doc = Web.build_xml do
+      root {
+        stuff(:name => 'bla') { text("hello") }
+      }
+    end
+
+    doc.to_xml.include?("<root>\n  <stuff name=\"bla\">hello</stuff>\n</root>").should == true
+  end
+
+  it "should have a default proxy" do
+    Web.proxy.should_not be_nil
+  end
+
+  it "should disable the proxy by default" do
+    Web.proxy.should_not be_enabled
+  end
+
+  describe "proxy_url" do
+    before(:all) do
+      @uri = URI('http://www.example.com:9001')
+    end
+
+    it "should convert Network::HTTP::Proxy objects into a URI" do
+      Web.proxy_url(Network::HTTP::Proxy.new(
+        :host => 'www.example.com',
+        :port => 9001
+      )).should == @uri
+    end
+
+    it "should convert a Hash into a URI" do
+      Web.proxy_url(
+        :host => 'www.example.com', 
+        :port => 9001
+      ).should == @uri
+    end
+
+    it "should convert a String into a valid URI" do
+      Web.proxy_url("www.example.com:9001").should == @uri
+    end
+  end
+
+  it "should provide User-Agent aliases" do
+    Web.user_agent_aliases.should_not be_empty
+  end
+
+  it "should provide a default User-Agent" do
+    Web.user_agent.should be_nil
+  end
+
+  it "should allow setting of the User-Agent string using an alias" do
+    Web.user_agent_alias = 'Mac FireFox'
+
+    Web.user_agent.should == "Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.8.0.3) Gecko/20060426 Firefox/1.5.0.3"
+  end
+
+  it "should open URLs as temporary files" do
+    file = Web.open('http://www.example.com/')
+    
+    file.read.should =~ /Example Web Page/
+  end
+
+  describe "agent" do
+    after(:each) do
+      Web.user_agent = nil
+    end
+
+    it "should provide WWW::Mechanize agents" do
+      Web.agent.class.should == WWW::Mechanize
+    end
+
+    it "should use the Ronin User-Agent string" do
+      Web.user_agent = 'test'
+      Web.agent.user_agent.should == 'test'
+    end
+
+    it "should support using a custom User-Agent string" do
+      agent = Web.agent(:user_agent => 'test2')
+
+      agent.user_agent.should == 'test2'
+    end
+
+    it "should support using a custom User-Agent alias" do
+      agent = Web.agent(:user_agent_alias => 'iPhone')
+      
+      agent.user_agent.should == 'Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 Mobile/1C28 Safari/419.3'
+    end
+
+    describe ":proxy" do
+      it "should accept Proxy values" do
+        pending "WWW::Mechanize needs reader methods for the proxy settings"
+      end
+
+      it "should accept Hash values" do
+      end
+
+      it "should accept String values" do
+        pending "WWW::Mechanize needs reader methods for the proxy settings"
+      end
+
+      it "should default to Web.proxy" do
+        pending "WWW::Mechanize needs reader methods for the proxy settings"
+      end
+
+      it "should raise a RuntimeError exception for bad :proxy options" do
+        lambda {
+          Web.agent(:proxy => 42)
+        }.should raise_error(RuntimeError)
+      end
+    end
+  end
+
+  it "should be able to get WWW::Mechanize pages" do
+    page = Web.get('http://www.example.com/')
+
+    page.class.should == WWW::Mechanize::Page
+    page.at('title').inner_text.should == 'Example Web Page'
+  end
+
+  it "should be able to get the bodies of WWW::Mechanize pages" do
+    body = Web.get_body('http://www.example.com/')
+
+    body.should =~ /Example Web Page/
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: ronin-web
 version: !ruby/object:Gem::Version 
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors: 
 - Postmodern
@@ -30,7 +30,7 @@ cert_chain:
   pDj+ws7QjtH/Qcrr1l9jfN0ehDs=
   -----END CERTIFICATE-----
 
-date: 2009-09-25 00:00:00 -07:00
+date: 2009-10-18 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -51,7 +51,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 0.1.9
+        version: 0.2.0
     version: 
 - !ruby/object:Gem::Dependency 
   name: sinatra
@@ -165,12 +165,14 @@ files:
 - lib/ronin/web/web.rb
 - lib/ronin/web/version.rb
 - lib/ronin/network/helpers/web.rb
+- lib/ronin/scanners/web.rb
 - tasks/spec.rb
 - tasks/yard.rb
 - spec/spec_helper.rb
 - spec/web/helpers/server.rb
 - spec/web/helpers/root/index.html
 - spec/web/helpers/root/test.txt
+- spec/web/web_spec.rb
 - spec/web/extensions/nokogiri_spec.rb
 - spec/web/server/classes/public1/static1.txt
 - spec/web/server/classes/public2/static2.txt
@@ -190,6 +192,7 @@ files:
 - spec/web/server/hosts_spec.rb
 - spec/web/server/proxy_spec.rb
 - spec/web/proxy/base_spec.rb
+- spec/scanners/web_spec.rb
 has_rdoc: yard
 homepage: http://ronin.rubyforge.org/web/
 licenses: []