arachni 0.3 → 0.4

data/lib/{rpc/xml → arachni/rpc}/server/options.rb

@@ -1,6 +1,6 @@
 =begin
                   Arachni
-  Copyright (c) 2010-2011 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+  Copyright (c) 2010-2012 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
 
   This is free software; you can copy and distribute and modify
   this program under the term of the GPL v2.0 License
@@ -13,7 +13,7 @@ module Arachni
 #
 # Overrides the Options class adding support for direct options parsing.
 #
-# Not much to look at but it streamlines XML-RPC server option handling.
+# Not much to look at but it streamlines RPC server option handling.
 #
 # @author: Tasos "Zapotek" Laskos
 #                                      <tasos.laskos@gmail.com>
@@ -28,44 +28,15 @@ class Options
             begin
                 send( "#{k.to_s}=", v )
             rescue Exception => e
-                ap e
-                ap e.backtrace
+                # ap e
+                # ap e.backtrace
             end
         }
         true
     end
 
-    #
-    # Resets all important options that can affect the scan
-    # during framework reuse.
-    #
-    def reset
-        # nil everything out
-        self.instance_variables.each {
-            |var|
-
-            # do *NOT* nil out @dir, we'll loose our paths!
-            next if var.to_s == '@dir'
-
-            begin
-                instance_variable_set( var.to_s, nil )
-            rescue Exception
-            end
-        }
-
-
-        @exclude    = []
-        @include    = []
-        @redundant  = []
-        @lsmod      = []
-        @exclude_cookies    = []
-
-        # set some defaults
-        @redirect_limit = 20
-
-        # relatively low but will give good performance without bottleneck
-        # on low bandwidth conections
-        @http_req_limit = 20
+    def datastore=( hash )
+        @datastore = hash
     end
 
     #

data/lib/{rpc/xml → arachni/rpc}/server/output.rb

@@ -1,6 +1,6 @@
 =begin
                   Arachni
-  Copyright (c) 2010-2011 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+  Copyright (c) 2010-2012 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
 
   This is free software; you can copy and distribute and modify
   this program under the term of the GPL v2.0 License
@@ -13,7 +13,7 @@ module Arachni
 module UI
 
 #
-# XML-RPC deamon Output module
+# RPC deamon Output module
 #
 # It basically classifies and buffers all system messages until it's time to
 # flush the buffer and send them over the wire.
@@ -21,7 +21,7 @@ module UI
 # @author: Tasos "Zapotek" Laskos
 #                                      <tasos.laskos@gmail.com>
 #                                      <zapotek@segfault.gr>
-# @version: 0.1
+# @version: 0.1.1
 #
 module Output
 
@@ -42,6 +42,8 @@ module Output
 
     @@reroute_to_file = false
 
+    @@buffer_cap = 30
+
     @@buffer ||= []
 
     #
@@ -70,9 +72,16 @@ module Output
             }
         else
             @@buffer << msg
+            if @@buffer_cap.is_a? Integer
+                @@buffer.slice!( (@@buffer.size - @@buffer_cap)..@@buffer.size  )
+            end
         end
     end
 
+    def uncap_buffer!
+        @@buffer_cap = nil
+    end
+
     # Prints an error message
     #
     # It ignores all flags, error messages will be output under all
@@ -85,6 +94,21 @@ module Output
         buffer( :error => str )
         print_color( '[-]', 31, str, $stderr )
     end
+    def print_error_backtrace( e )
+        e.backtrace.each { |line| print_error( line ) }
+    end
+
+    #
+    # Same as print_error but the message won't be printed to stderr.
+    #
+    # Used mainly to draw attention to something that didn't behave as expected
+    # rather than display an actual error.
+    #
+    # @param    [String]    error string
+    #
+    def print_bad( str = '' )
+        buffer( :red => str )
+    end
 
     # Prints a status message
     #
@@ -293,6 +317,8 @@ module Output
         out.print "\033[1;#{color.to_s}m #{sign}\033[1;00m #{string}\n";
     end
 
+    extend self
+
 end
 
 end

data/lib/{rpc/xml → arachni/rpc}/server/plugin/manager.rb

@@ -1,6 +1,6 @@
 =begin
                   Arachni
-  Copyright (c) 2010-2011 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+  Copyright (c) 2010-2012 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
 
   This is free software; you can copy and distribute and modify
   this program under the term of the GPL v2.0 License
@@ -13,13 +13,12 @@ module Arachni
 require Options.instance.dir['lib'] + 'plugin/manager'
 
 module RPC
-module XML
-module Server
+class Server
 module Plugin
 
 #
 # We need to extend the original Manager and redeclare its inherited methods
-# which are required over XMLRPC.
+# which are required over RPC.
 #
 # @author: Tasos "Zapotek" Laskos
 #                                      <tasos.laskos@gmail.com>
@@ -39,9 +38,10 @@ class Manager < ::Arachni::Plugin::Manager
     end
 
     def load( plugins )
-
         @plugin_opts.merge!( plugins )
         super( plugins.keys )
+
+        @framework.opts.plugins = plugins
     end
 
     def create( name )
@@ -55,4 +55,3 @@ end
 end
 end
 end
-end

data/lib/{ruby.rb → arachni/ruby.rb}

@@ -1,5 +1,4 @@
-
 opts = Arachni::Options.instance
 require opts.dir['lib'] +  'ruby/object'
+require opts.dir['lib'] +  'ruby/array'
 require opts.dir['lib'] +  'ruby/string'
-require opts.dir['lib'] +  'ruby/xmlrpc/server'

data/lib/arachni/ruby/array.rb

@@ -0,0 +1,31 @@
+=begin
+                  Arachni
+  Copyright (c) 2010-2012 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+
+  This is free software; you can copy and distribute and modify
+  this program under the term of the GPL v2.0 License
+  (See LICENSE file for details)
+
+=end
+
+class Array
+
+    def chunk( pieces = 2 )
+        return self if pieces <= 0
+
+        len    = self.length;
+        mid    = ( len / pieces )
+        chunks = []
+        start  = 0
+
+        1.upto( pieces ) do |i|
+            last = start + mid
+            last = last - 1 unless len % pieces >= i
+            chunks << self[ start..last ] || []
+            start = last + 1
+        end
+
+        return chunks
+    end
+
+end

data/lib/{ruby → arachni/ruby}/object.rb

@@ -1,6 +1,6 @@
 =begin
                   Arachni
-  Copyright (c) 2010-2011 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+  Copyright (c) 2010-2012 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
 
   This is free software; you can copy and distribute and modify
   this program under the term of the GPL v2.0 License

data/lib/{ruby → arachni/ruby}/string.rb

@@ -1,6 +1,6 @@
 =begin
                   Arachni
-  Copyright (c) 2010-2011 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+  Copyright (c) 2010-2012 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
 
   This is free software; you can copy and distribute and modify
   this program under the term of the GPL v2.0 License

data/lib/{spider.rb → arachni/spider.rb}

@@ -1,6 +1,6 @@
 =begin
                   Arachni
-  Copyright (c) 2010-2011 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+  Copyright (c) 2010-2012 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
 
   This is free software; you can copy and distribute and modify
   this program under the term of the GPL v2.0 License
@@ -22,7 +22,7 @@ module Arachni
 # @author: Tasos "Zapotek" Laskos
 #                                      <tasos.laskos@gmail.com>
 #                                      <zapotek@segfault.gr>
-# @version: 0.2
+# @version: 0.2.3
 #
 class Spider
 
@@ -36,18 +36,18 @@ class Spider
     attr_reader :opts
 
     #
-    # Sitemap, array of links
+    # Discovered paths
     #
     # @return [Array]
     #
     attr_reader :sitemap
 
     #
-    # Code block to be executed on each page
+    # URLs that caused redirects
     #
-    # @return [Proc]
+    # @return [Array]
     #
-    attr_reader :on_every_page_blocks
+    attr_reader :redirects
 
     #
     # Constructor <br/>
@@ -58,14 +58,32 @@ class Spider
     def initialize( opts )
         @opts = opts
 
-        @sitemap = []
+        @sitemap   = []
+        @redirects = []
         @on_every_page_blocks = []
 
+        @seed_url = @opts.url.to_s
+
+        @extend_paths   = @opts.extend_paths   || []
+        @restrict_paths = @opts.restrict_paths || []
+
+        @paths = [ @seed_url ]
+
+        if restricted_to_paths?
+            @paths |= @sitemap = @restrict_paths
+        else
+            @paths |= @extend_paths
+        end
+
         # if we have no 'include' patterns create one that will match
         # everything, like '.*'
         @opts.include =[ Regexp.new( '.*' ) ] if @opts.include.empty?
     end
 
+    def restricted_to_paths?
+        !@restrict_paths.empty?
+    end
+
     #
     # Runs the Spider and passes parsed page to the block
     #
@@ -73,87 +91,101 @@ class Spider
     #
     # @return [Arachni::Parser::Page]
     #
-    def run( &block )
+    def run( parse = true, &block )
         return if @opts.link_count_limit == 0
 
-        paths = []
-        paths << @opts.url.to_s
-
         visited = []
 
-        while( !paths.empty? )
-            while( !paths.empty? && url = paths.pop )
-                url = url_sanitize( url )
-                next if skip?( url ) || !in_domain?( url )
+        opts = {
+            :timeout    => nil,
+            :remove_id  => true,
+            :follow_location => true,
+            :update_cookies  => true
+        }
+
+        # we need a parser in order to have access to skip() in case
+        # there's a redirect that shouldn't be followed
+        seed_page = http.get( @seed_url, opts.merge( :async => false ) ).response
+
+        print_status( "[HTTP: #{seed_page.code}] " + seed_page.effective_url )
+
+        parser = Parser.new( @opts, seed_page )
+        parser.url = @seed_url
+        @paths = parser.paths | [@seed_url]
+
+        while( !@paths.empty? )
+            while( !@paths.empty? && url = parser.to_absolute( @paths.pop ) )
+                next if skip?( url )
 
                 wait_if_paused
 
                 visited << url
 
-                opts = {
-                    :timeout => nil,
-                    :remove_id => true,
-                    :async => @opts.spider_first
-                }
-
-                Arachni::HTTP.instance.get( url, opts ).on_complete {
+                http.get( url, opts ).on_complete {
                     |res|
 
-                    print_line
+                    next if parser.skip?( res.effective_url )
+
                     print_status( "[HTTP: #{res.code}] " + res.effective_url )
 
-                    page = Arachni::Parser.new( @opts, res ).run
-                    page.url = url_sanitize( res.effective_url )
+                    if parse
+                        page = Arachni::Parser::Page.from_http_response( res, @opts )
+                        paths = page.paths
+                        check_url = page.url
+                    else
+                        c_parser = Parser.new( @opts, res )
+                        paths = c_parser.text? ? c_parser.paths : []
+                        check_url = c_parser.url
+                    end
+
+                    if !restricted_to_paths?
+                        @sitemap |= paths
 
-                    @sitemap |= page.paths.map { |path| url_sanitize( path ) }
-                    paths    |= @sitemap - visited
+                        if !res.headers_hash['Location'].empty?
+                            @redirects << res.request.url
+                        end
 
+                        @paths   |= @sitemap - visited
+                    end
 
                     # call the block...if we have one
                     if block
                         exception_jail{
-                            block.call( page.clone )
+                            if !skip?( check_url )
+                                block.call( parse ? page.clone : res )
+                            else
+                                print_info( 'Matched skip rule.' )
+                            end
                         }
                     end
-
-                    # run blocks specified later
-                    @on_every_page_blocks.each {
-                        |block|
-                        block.call( page )
-                    }
-
                 }
 
-                Arachni::HTTP.instance.run if !@opts.spider_first
-
                 # make sure we obey the link count limit and
                 # return if we have exceeded it.
                 if( @opts.link_count_limit &&
-                    @opts.link_count_limit <= visited.size )
-                    Arachni::HTTP.instance.run if @opts.spider_first
+                    @opts.link_count_limit > 0 &&
+                    visited.size >= @opts.link_count_limit )
+                    http.run
                     return @sitemap.uniq
                 end
 
-
-            end
-
-            if @opts.spider_first
-                Arachni::HTTP.instance.run
-            else
-                break
             end
 
+            http.run
         end
 
         return @sitemap.uniq
     end
 
+    def http
+        Arachni::HTTP.instance
+    end
+
     def skip?( url )
-        @opts.exclude.each {
-            |regexp|
-            return true if regexp =~ url
-        }
+        redundant?( url )
+    end
 
+    def redundant?( url )
         @opts.redundant.each_with_index {
             |redundant, i|
 
@@ -173,19 +205,10 @@ class Spider
                 @opts.redundant[i]['count'] -= 1
             end
         }
-
-
-        skip_cnt = 0
-        @opts.include.each {
-            |regexp|
-            skip_cnt += 1 if !(regexp =~ url)
-        }
-
-        return false if skip_cnt > 1
-
         return false
     end
 
+
     def wait_if_paused
         while( paused? )
             ::IO::select( nil, nil, nil, 1 )
@@ -205,55 +228,5 @@ class Spider
         return @pause
     end
 
-    #
-    # Checks if the uri is in the same domain
-    #
-    # @param [URI] url
-    #
-    # @return [String]
-    #
-    def in_domain?( uri )
-
-        uri_1 = URI( uri.to_s )
-        uri_2 = URI( @opts.url.to_s )
-
-        if( @opts.follow_subdomains )
-            return extract_domain( uri_1 ) ==  extract_domain( uri_2 )
-        end
-
-        uri_1.host == uri_2.host
-    end
-
-    #
-    # Extracts the domain from a URI object
-    #
-    # @param [URI] url
-    #
-    # @return [String]
-    #
-    def extract_domain( url )
-
-        if !url.host then return false end
-
-        splits = url.host.split( /\./ )
-
-        if splits.length == 1 then return true end
-
-        splits[-2] + "." + splits[-1]
-    end
-
-
-    #
-    # Hook for further analysis of pages, statistics etc.
-    #
-    # @param [Proc] block code to be executed for every page
-    #
-    # @return [self]
-    #
-    def on_every_page( &block )
-        @on_every_page_blocks.push( block )
-        self
-    end
-
 end
 end