arachni 0.2.4 → 0.3

data/lib/rpc/xml/server/dispatcher.rb

@@ -178,6 +178,10 @@ class Dispatcher < Base
         }
     end
 
+    def proc_info
+        unnil( proc( Process.pid ) )
+    end
+
     #
     # Outputs the Arachni banner.<br/>
     # Displays version number, revision number, author details etc.
@@ -235,6 +239,23 @@ USAGE
 
     private
 
+    #
+    # Recursively removes nils.
+    #
+    # @param    [Hash]  hash
+    #
+    # @return   [Hash]
+    #
+    def unnil( hash )
+        hash.each_pair {
+            |k, v|
+            hash[k] = '' if v.nil?
+            hash[k] = unnil( v ) if v.is_a? Hash
+        }
+
+        return hash
+    end
+
     #
     # Initializes and updates the pool making sure that the number of
     # available server processes stays constant for any given moment

data/lib/spider.rb

@@ -8,8 +8,9 @@
 
 =end
 
-require Arachni::Options.instance.dir['lib'] + 'anemone'
 require Arachni::Options.instance.dir['lib'] + 'module/utilities'
+require 'nokogiri'
+require Arachni::Options.instance.dir['lib'] + 'nokogiri/xml/node'
 
 module Arachni
 
@@ -21,7 +22,7 @@ module Arachni
 # @author: Tasos "Zapotek" Laskos
 #                                      <tasos.laskos@gmail.com>
 #                                      <zapotek@segfault.gr>
-# @version: 0.1
+# @version: 0.2
 #
 class Spider
 
@@ -34,8 +35,6 @@ class Spider
     #
     attr_reader :opts
 
-    attr_reader :pages
-
     #
     # Sitemap, array of links
     #
@@ -59,31 +58,6 @@ class Spider
     def initialize( opts )
         @opts = opts
 
-        @anemone_opts = {
-            :threads              =>  1,
-            :discard_page_bodies  =>  false,
-            :delay                =>  0,
-            :obey_robots_txt      =>  false,
-            :depth_limit          =>  false,
-            :link_count_limit     =>  false,
-            :redirect_limit       =>  false,
-            :storage              =>  nil,
-            :cookies              =>  nil,
-            :accept_cookies       =>  true,
-            :proxy_addr           =>  nil,
-            :proxy_port           =>  nil,
-            :proxy_user           =>  nil,
-            :proxy_pass           =>  nil
-        }
-
-        hash_opts = @opts.to_h
-        @anemone_opts.each_pair {
-            |k, v|
-            @anemone_opts[k] = hash_opts[k.to_s] if hash_opts[k.to_s]
-        }
-
-        @anemone_opts = @anemone_opts.merge( hash_opts )
-
         @sitemap = []
         @on_every_page_blocks = []
 
@@ -102,85 +76,170 @@ class Spider
     def run( &block )
         return if @opts.link_count_limit == 0
 
-        i = 1
-        # start the crawl
-        Anemone.crawl( @opts.url, @anemone_opts ) {
-            |anemone|
+        paths = []
+        paths << @opts.url.to_s
 
-            # apply 'exclude' patterns
-            anemone.skip_links_like( @opts.exclude ) if @opts.exclude
+        visited = []
 
-            # apply 'include' patterns and grab matching pages
-            # as they are discovered
-            anemone.on_pages_like( @opts.include ) {
-                |page|
+        while( !paths.empty? )
+            while( !paths.empty? && url = paths.pop )
+                url = url_sanitize( url )
+                next if skip?( url ) || !in_domain?( url )
 
-                @pages = anemone.pages.keys || []
+                wait_if_paused
 
-                url = url_sanitize( page.url.to_s )
+                visited << url
 
-                # something went kaboom, tell the user and skip the page
-                if page.error
-                    print_error( "[Error: " + (page.error.to_s) + "] " + url )
-                    print_debug_backtrace( page.error )
-                    next
-                end
+                opts = {
+                    :timeout => nil,
+                    :remove_id => true,
+                    :async => @opts.spider_first
+                }
 
-                # push the url in the sitemap
-                @sitemap.push( url )
-
-                print_line
-                print_status( "[HTTP: #{page.code}] " + url )
-
-                # call the block...if we have one
-                if block
-                    exception_jail{
-                        new_page = Arachni::Parser.new( @opts,
-                            Typhoeus::Response.new(
-                                :effective_url => url,
-                                :body          => page.body,
-                                :headers_hash  => page.headers
-                            )
-                        ).run
-                        new_page.code   = page.code
-                        new_page.method = 'GET'
-                        block.call( new_page.clone )
+                Arachni::HTTP.instance.get( url, opts ).on_complete {
+                    |res|
+
+                    print_line
+                    print_status( "[HTTP: #{res.code}] " + res.effective_url )
+
+                    page = Arachni::Parser.new( @opts, res ).run
+                    page.url = url_sanitize( res.effective_url )
+
+                    @sitemap |= page.paths.map { |path| url_sanitize( path ) }
+                    paths    |= @sitemap - visited
+
+
+                    # call the block...if we have one
+                    if block
+                        exception_jail{
+                            block.call( page.clone )
+                        }
+                    end
+
+                    # run blocks specified later
+                    @on_every_page_blocks.each {
+                        |block|
+                        block.call( page )
                     }
-                end
 
-                # run blocks specified later
-                @on_every_page_blocks.each {
-                    |block|
-                    block.call( page )
                 }
 
-                # we don't need the HTML doc anymore
-                page.discard_doc!( )
+                Arachni::HTTP.instance.run if !@opts.spider_first
 
                 # make sure we obey the link count limit and
                 # return if we have exceeded it.
                 if( @opts.link_count_limit &&
-                    @opts.link_count_limit <= i )
+                    @opts.link_count_limit <= visited.size )
+                    Arachni::HTTP.instance.run if @opts.spider_first
                     return @sitemap.uniq
                 end
 
-                i+=1
-            }
-        }
+
+            end
+
+            if @opts.spider_first
+                Arachni::HTTP.instance.run
+            else
+                break
+            end
+
+        end
 
         return @sitemap.uniq
     end
 
+    def skip?( url )
+        @opts.exclude.each {
+            |regexp|
+            return true if regexp =~ url
+        }
+
+        @opts.redundant.each_with_index {
+            |redundant, i|
+
+            if( url =~ redundant['regexp'] )
+
+                if( @opts.redundant[i]['count'] == 0 )
+                    print_verbose( 'Discarding redundant page: \'' + url + '\'' )
+                    return true
+                end
+
+                print_info( 'Matched redundancy rule: ' +
+                redundant['regexp'].to_s + ' for page \'' +
+                url + '\'' )
+
+                print_info( 'Count-down: ' + @opts.redundant[i]['count'].to_s )
+
+                @opts.redundant[i]['count'] -= 1
+            end
+        }
+
+
+        skip_cnt = 0
+        @opts.include.each {
+            |regexp|
+            skip_cnt += 1 if !(regexp =~ url)
+        }
+
+        return false if skip_cnt > 1
+
+        return false
+    end
+
+    def wait_if_paused
+        while( paused? )
+            ::IO::select( nil, nil, nil, 1 )
+        end
+    end
+
+    def pause!
+        @pause = true
+    end
+
+    def resume!
+        @pause = false
+    end
+
+    def paused?
+        @pause ||= false
+        return @pause
+    end
+
+    #
+    # Checks if the uri is in the same domain
     #
-    # Decodes URLs to reverse multiple encodes and removes NULL characters
+    # @param [URI] url
     #
-    def url_sanitize( url )
+    # @return [String]
+    #
+    def in_domain?( uri )
+
+        uri_1 = URI( uri.to_s )
+        uri_2 = URI( @opts.url.to_s )
 
-        while( url =~ /%/ )
-            url = ( URI.decode( url ).to_s.unpack( 'A*' )[0] )
+        if( @opts.follow_subdomains )
+            return extract_domain( uri_1 ) ==  extract_domain( uri_2 )
         end
 
-        return url
+        uri_1.host == uri_2.host
+    end
+
+    #
+    # Extracts the domain from a URI object
+    #
+    # @param [URI] url
+    #
+    # @return [String]
+    #
+    def extract_domain( url )
+
+        if !url.host then return false end
+
+        splits = url.host.split( /\./ )
+
+        if splits.length == 1 then return true end
+
+        splits[-2] + "." + splits[-1]
     end
 
 

data/lib/ui/cli/cli.rb

@@ -123,16 +123,15 @@ class CLI
 
         audited = stats[:auditmap_size]
         mapped  = stats[:sitemap_size]
-        progress = ( Float( audited ) / mapped ) * 100
 
         print_line
-        print_info( "Audit progress: #{progress.to_s[0...5]}% ( #{audited}/#{mapped} pages )" )
+        print_info( "Audit progress: #{stats[:progress]}% ( Discovered #{mapped} pages )" )
         print_line
         print_info( "Sent #{stats[:requests]} requests." )
         print_info( "Received and analyzed #{stats[:responses]} responses." )
         print_info( 'In ' + stats[:time] )
 
-        avg = 'Average: ' + stats[:avg] + ' requests/second.'
+        avg = 'Average: ' + stats[:avg].to_s + ' requests/second.'
         print_info( avg )
 
         print_line

data/lib/ui/web/addon_manager.rb

@@ -0,0 +1,273 @@
+=begin
+                  Arachni
+  Copyright (c) 2010-2011 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+
+  This is free software; you can copy and distribute and modify
+  this program under the term of the GPL v2.0 License
+  (See LICENSE file for details)
+
+=end
+
+
+module Arachni
+module UI
+module Web
+
+module Addons
+
+    #
+    # Base class for all add-ons.
+    #
+    #
+    # @author: Tasos "Zapotek" Laskos
+    #                                      <tasos.laskos@gmail.com>
+    #                                      <zapotek@segfault.gr>
+    # @version: 0.1
+    #
+    class Base
+
+        def initialize( settings, route )
+            @settings = settings
+            @route    = '/addons/' + route
+
+            @settings.helpers do
+
+                def present( tpl, args )
+                    views = current_addon.path_views
+                    trv = ( '../' * views.split( '/' ).size ) + views + tpl.to_s
+
+                    erb_args = []
+                    erb_args << { :layout => true }
+                    erb_args << { :tpl => trv.to_sym, :addon => addons.by_name( current_addon_name ), :tpl_args => args }
+
+                    erb :addon, *erb_args
+                end
+
+                def partial( tpl, args )
+                    views = current_addon.path_views
+                    trv = ( '../' * views.split( '/' ).size ) + views + tpl.to_s
+
+                    erb_args = []
+                    erb_args << { :layout => false }
+                    erb_args << args
+
+                    erb trv.to_sym, *erb_args
+                end
+
+                def current_addon_name
+                    env['PATH_INFO'].scan( /\/addons\/(.*?)\// ).flatten[0]
+                end
+
+                def current_addon
+                    addons.running[current_addon_name]
+                end
+
+            end
+
+        end
+
+        def path_root
+            @route
+        end
+
+        def path_views
+            path_addon + '/views/'
+        end
+
+        def path_addon
+            Options.instance.dir['lib'] + 'ui/web' + path_root
+        end
+
+        def run
+
+        end
+
+        #
+        # This optional method allows you to specify the title which will be
+        # used for the menu (in case you want it to be dynamic).
+        #
+        # @return   [String]
+        #
+        def title
+            ''
+        end
+
+
+        #
+        #
+        # *DO NOT MESS WITH THE FOLLOWING METHODS*
+        #
+        #
+
+
+        def settings
+           @settings
+        end
+
+        def get( path, &block )
+            settings.get( @route + path, &block )
+        end
+
+        def post( path, &block )
+            settings.post( @route + path, &block )
+        end
+
+        def put( path, &block )
+            settings.put( @route + path, &block )
+        end
+
+        def delete( path, &block )
+            settings.delete( @route + path, &block )
+        end
+
+    end
+end
+
+
+#
+# Add-on manager.
+#
+#
+# @author: Tasos "Zapotek" Laskos
+#                                      <tasos.laskos@gmail.com>
+#                                      <zapotek@segfault.gr>
+# @version: 0.1
+#
+class AddonManager
+
+    include Utilities
+
+    class Addon
+        include DataMapper::Resource
+
+        property :id,   Serial
+        property :name, String
+    end
+
+    class RestrictedComponentManager < Arachni::ComponentManager
+        def paths
+            cpaths = paths = Dir.glob( File.join( "#{@lib}", "*.rb" ) )
+            return paths.reject { |path| helper?( path ) }
+        end
+    end
+
+    def initialize( opts, settings )
+        @opts     = opts
+        @settings = settings
+
+        lib = @opts.dir['lib'] + 'ui/web/addons/'
+        @@manager ||= RestrictedComponentManager.new( lib, Addons )
+
+        @@running ||= {}
+
+        DataMapper::setup( :default, "sqlite3://#{@settings.db}/default.db" )
+        DataMapper.finalize
+
+        Addon.auto_upgrade!
+
+        run( enabled )
+    end
+
+    #
+    # Runs addons.
+    #
+    # @param    [Array]     addons  array holding the names of the addons
+    #
+    def run( addons )
+
+        begin
+            addons.each {
+                |name|
+                @@running[name] = @@manager[name].new( @settings, name )
+                @@running[name].run
+            }
+
+        rescue ::Exception => e
+            ap e.to_s
+            ap e.backtrace
+        end
+    end
+
+    def running
+        @@running
+    end
+
+    #
+    # Gets add-on info by name.
+    #
+    # @param    [String]    name
+    #
+    # @return   [Hash]
+    #
+    def by_name( name )
+        available.each { |addon| return addon if addon['filename'] == name }
+        return nil
+    end
+
+    #
+    # Gets all available add-ons.
+    #
+    # @return   [Array]
+    #
+    def available
+        @@available ||= populate_available
+
+        @@available.each {
+            |addon|
+
+            if @@running[addon['filename']] && !@@running[addon['filename']].title.empty?
+                addon['title'] = @@running[addon['filename']].title
+            else
+                addon['title'] = addon['name']
+            end
+        }
+
+        return @@available
+    end
+
+    #
+    # Enables and runs add-ons.
+    #
+    # @param    [Array]     addons  array holding the names of the addons
+    #
+    def enable!( addons )
+        Addon.all.destroy
+        addons.each { |addon| Addon.create( :name => addon ); run( [addon] ) }
+    end
+
+    #
+    # Gets all enabled add-ons.
+    #
+    # @return   [Array]
+    #
+    def enabled
+        Addon.all.map { |addon| addon.name }
+    end
+
+    private
+    def populate_available
+        @@available ||= []
+        return @@available if !@@available.empty?
+
+        @@available_classes ||= {}
+        @@manager.available.each {
+            |avail|
+
+            @@available << {
+                'name'        => @@manager[avail].info[:name],
+                'filename'    => avail,
+                'description' => @@manager[avail].info[:description],
+                'version'     => @@manager[avail].info[:version],
+                'author'      => @@manager[avail].info[:author]
+            }
+
+            @@available_classes[avail] = @@manager[avail]
+
+        }
+        return @@available
+    end
+
+end
+end
+end
+end