arachni 0.2.4 → 0.3

data/lib/module/base.rb

@@ -115,6 +115,19 @@ class Base
     def clean_up( )
     end
 
+    #
+    # ABSTRACT - OPTIONAL
+    #
+    # Prevents auditting elements that have been previously
+    # logged by any of the modules returned by this method.
+    #
+    # @return   [Array]     module names
+    #
+    def redundant
+        # [ 'sqli', 'sqli_blind_rdiff' ]
+        []
+    end
+
     #
     # ABSTRACT - REQUIRED
     #
@@ -169,6 +182,10 @@ class Base
         Arachni::Module::Manager.register_results( results )
     end
 
+    def set_framework( framework )
+        @framework = framework
+    end
+
 end
 end
 end

data/lib/module/manager.rb

@@ -36,7 +36,7 @@ module Module
 # @author: Tasos "Zapotek" Laskos
 #                                      <tasos.laskos@gmail.com>
 #                                      <zapotek@segfault.gr>
-# @version: 0.1
+# @version: 0.1.1
 #
 class Manager < Arachni::ComponentManager
 
@@ -48,7 +48,8 @@ class Manager < Arachni::ComponentManager
     def initialize( opts )
         super( opts.dir['modules'], Arachni::Modules )
         @opts = opts
-        @@results  = []
+        @@results    = []
+        @@issue_set  = Set.new
     end
 
     #
@@ -60,8 +61,31 @@ class Manager < Arachni::ComponentManager
     #
     def self.register_results( results )
         @@results |= results
+        results.each { |issue| @@issue_set << self.issue_set_id_from_issue( issue ) }
     end
 
+    def self.issue_set_id_from_issue( issue )
+        issue_url = URI( issue.url )
+        issue_url_str = issue_url.scheme + "://" + issue_url.host + issue_url.path
+        return "#{issue.mod_name}:#{issue.elem}:#{issue.var}:#{issue_url_str}"
+    end
+
+    def self.issue_set_id_from_elem( mod_name, elem )
+        elem_url  = URI( elem.action )
+        elem_url_str  = elem_url.scheme + "://" + elem_url.host + elem_url.path
+
+        return "#{mod_name}:#{elem.type}:#{elem.altered}:#{elem_url_str}"
+    end
+
+    def self.issue_set
+        @@issue_set
+    end
+
+    def issue_set
+        @@issue_set
+    end
+
+
     #
     # Class method
     #

data/lib/module/trainer.rb

@@ -27,6 +27,7 @@ class Trainer
 
     include Output
     include ElementDB
+    include Utilities
 
     attr_writer   :page
     attr_accessor :http
@@ -70,18 +71,6 @@ class Trainer
 
     end
 
-    #
-    # Decodes URLs to reverse multiple encodes and removes NULL characters
-    #
-    def url_sanitize( url )
-
-        while( url =~ /%/ )
-            url = ( URI.decode( url ).to_s.unpack( 'A*' )[0] )
-        end
-
-        return URI.encode( url )
-    end
-
     def follow?( url )
         @parser.url = @page.url
 

data/lib/module/utilities.rb

@@ -24,6 +24,18 @@ module Module
 #
 module Utilities
 
+    #
+    # Decodes URLs to reverse multiple encodes and removes NULL characters
+    #
+    def url_sanitize( url )
+
+        while( url =~ /%/ )
+            url = ( URI.decode( url ).to_s.unpack( 'A*' )[0] )
+        end
+
+        return URI.encode( url )
+    end
+
     #
     # Gets path from URL
     #

data/lib/parser/auditable.rb

@@ -49,6 +49,11 @@ class Auditable
         @auditor = auditor
     end
 
+    def get_auditor
+        @auditor
+    end
+
+
     #
     # Delegate output related methods to the auditor
     #
@@ -142,7 +147,7 @@ class Auditable
             return if skip?( elem )
 
             # inform the user about what we're auditing
-            print_status( get_status_str( opts[:altered] ) )
+            print_status( get_status_str( opts[:altered] ) )  if !opts[:silent]
 
             # submit the element with the injection values
             req = elem.submit( opts )
@@ -179,7 +184,7 @@ class Auditable
         var_combo = []
         if( !hash || hash.size == 0 ) then return [] end
 
-        if( self.is_a?( Arachni::Parser::Element::Form ) )
+        if( self.is_a?( Arachni::Parser::Element::Form ) && !opts[:skip_orig] )
 
             if !audited?( audit_id( Arachni::Parser::Element::Form::FORM_VALUES_ORIGINAL ) )
                 # this is the original hash, in case the default values
@@ -295,7 +300,7 @@ class Auditable
                 print_error( 'Failed to get responses, backing out... ' )
                 next
             else
-                print_status( 'Analyzing response #' + res.request.id.to_s + '...' )
+                print_status( 'Analyzing response #' + res.request.id.to_s + '...' )  if elem.opts && !elem.opts[:silent]
             end
 
             # call the block, if there's one

data/lib/parser/elements.rb

@@ -48,6 +48,8 @@ class Base < Arachni::Element::Auditable
 
     attr_accessor :auditable
 
+    attr_accessor :orig
+
     #
     # Relatively 'raw' hash holding the element's attributes, values, etc.
     #
@@ -113,6 +115,8 @@ class Link < Base
         @method = 'get'
 
         @auditable = @raw['vars']
+        @orig      = @auditable.deep_clone
+        @orig.freeze
     end
 
     def http_request( url, opts )
@@ -155,6 +159,8 @@ class Form < Base
         @method = @raw['attrs']['method']
 
         @auditable = simple['auditable'] || {}
+        @orig      = @auditable.deep_clone
+        @orig.freeze
     end
 
     def http_request( url, opts )
@@ -248,6 +254,9 @@ class Cookie < Base
             |cookie|
             Options.instance.exclude_cookies.include?( cookie )
         }
+
+        @orig      = @auditable.deep_clone
+        @orig.freeze
     end
 
     def http_request( url, opts )
@@ -274,6 +283,8 @@ class Header < Base
         @method = 'header'
 
         @auditable = @raw
+        @orig      = @auditable.deep_clone
+        @orig.freeze
     end
 
     def http_request( url, opts )

data/lib/parser/page.rb

@@ -19,7 +19,7 @@ class Parser
 # @author: Tasos "Zapotek" Laskos
 #                                      <tasos.laskos@gmail.com>
 #                                      <zapotek@segfault.gr>
-# @version: 0.2
+# @version: 0.2.1
 #
 class Page
 
@@ -60,6 +60,8 @@ class Page
     #
     attr_accessor :response_headers
 
+    attr_accessor :paths
+
     #
     # @see Parser#links
     #

data/lib/parser/parser.rb

@@ -10,9 +10,11 @@
 module Arachni
 
 opts = Arachni::Options.instance
+require 'webrick'
 require opts.dir['lib'] + 'parser/elements'
 require opts.dir['lib'] + 'parser/page'
 require opts.dir['lib'] + 'module/utilities'
+require opts.dir['lib'] + 'component_manager'
 
 #
 # Analyzer class
@@ -41,12 +43,41 @@ require opts.dir['lib'] + 'module/utilities'
 # @author: Tasos "Zapotek" Laskos
 #                                      <tasos.laskos@gmail.com>
 #                                      <zapotek@segfault.gr>
-# @version: 0.2
+# @version: 0.2.1
 #
 class Parser
-
+    include Arachni::UI::Output
     include Arachni::Module::Utilities
 
+    module Extractors
+    #
+    # Base Spider parser class for modules.
+    #
+    # The aim of such modules is to extract paths from a webpage for the Spider to follow.
+    #
+    #
+    # @author: Tasos "Zapotek" Laskos
+    #                                      <tasos.laskos@gmail.com>
+    #                                      <zapotek@segfault.gr>
+    # @version: 0.1
+    # @abstract
+    #
+    class Paths
+
+        #
+        # This method must be implemented by all modules and must return an array
+        # of paths as plain strings
+        #
+        # @param    [Nokogiri]  Nokogiri document
+        #
+        # @return   [Array<String>]  paths
+        #
+        def run( doc )
+
+        end
+    end
+    end
+
     #
     # @return    [String]    the url of the page
     #
@@ -68,7 +99,7 @@ class Parser
     def initialize( opts, res )
         @opts = opts
 
-        @url  = res.effective_url
+        @url  = url_sanitize( res.effective_url )
         @html = res.body
         @response_headers = res.headers_hash
     end
@@ -89,6 +120,7 @@ class Parser
                 :html        => @html,
                 :headers     => [],
                 :response_headers     => @response_headers,
+                :paths       => [],
                 :forms       => [],
                 :links       => [],
                 :cookies     => [],
@@ -108,14 +140,25 @@ class Parser
 
         jar = preped.merge( jar )
 
+        c_links = links
+
+        if !( vars = link_vars( @url ) ).empty?
+            url = to_absolute( @url )
+            c_links << Arachni::Parser::Element::Link.new( url, {
+                'href' => url,
+                'vars' => vars
+            } )
+        end
+
         return Page.new( {
             :url         => @url,
             :query_vars  => link_vars( @url ),
             :html        => @html,
             :headers     => headers(),
             :response_headers     => @response_headers,
+            :paths       => paths(),
             :forms       => @opts.audit_forms ? forms() : [],
-            :links       => @opts.audit_links ? links() : [],
+            :links       => @opts.audit_links ? c_links : [],
             :cookies     => merge_with_cookiestore( merge_with_cookiejar( cookies_arr ) ),
             :cookiejar   => jar
         } )
@@ -258,7 +301,7 @@ class Parser
             if( !elements[i]['attrs'] || !elements[i]['attrs']['action'] )
                 action = @url.to_s
             else
-                action = elements[i]['attrs']['action']
+                action = url_sanitize( elements[i]['attrs']['action'] )
             end
             action = URI.escape( action ).to_s
 
@@ -322,8 +365,17 @@ class Parser
             if( !include?( link['href'] ) ) then next end
             if !in_domain?( URI.parse( link['href'] ) ) then next end
 
-            link['vars'] = link_vars( link['href'] )
+            link['vars'] = {}
+            link_vars( link['href'] ).each_pair {
+                |key, val|
+                begin
+                    link['vars'][key] = url_sanitize( val )
+                rescue
+                    link['vars'][key] = val
+                end
+            }
 
+            link['href'] = url_sanitize( link['href'] )
 
             link_arr << Element::Link.new( @url, link )
 
@@ -356,11 +408,12 @@ class Parser
         rescue
         end
 
+
         # don't ask me why....
         if @response_headers.to_s.substring?( 'set-cookie' )
             begin
-                cookies << WEBrick::Cookie.parse_set_cookies( @response_headers['Set-Cookie'].to_s )
-                cookies << WEBrick::Cookie.parse_set_cookies( @response_headers['set-cookie'].to_s )
+                cookies << ::WEBrick::Cookie.parse_set_cookies( @response_headers['Set-Cookie'].to_s )
+                cookies << ::WEBrick::Cookie.parse_set_cookies( @response_headers['set-cookie'].to_s )
             rescue
                 return cookies_arr
             end
@@ -390,6 +443,32 @@ class Parser
         return cookies_arr
     end
 
+    def dir( url )
+        URI( File.dirname( URI( url.to_s ).path ) + '/' )
+    end
+
+    #
+    # Array of distinct links to follow
+    #
+    # @return   [Array<URI>]
+    #
+    def paths
+      return @paths unless @paths.nil?
+      @paths = []
+      return @paths if !doc
+
+      run_extractors( ).each {
+          |path|
+          next if path.nil? or path.empty?
+          abs = to_absolute( path ) rescue next
+
+          @paths << abs if in_domain?( abs )
+      }
+
+      @paths.uniq!
+      return @paths
+    end
+
     #
     # Extracts variables and their values from a link
     #
@@ -434,26 +513,36 @@ class Parser
             end
         rescue Exception => e
             return nil if link.nil?
-            #      return link
         end
 
         # remove anchor
-        link = URI.encode( link.to_s.gsub( /#[a-zA-Z0-9_-]*$/, '' ) )
+        link = URI.encode( link.to_s.gsub( /#[a-zA-Z0-9_-]*$/,'' ) )
 
-        begin
-            relative = URI(link)
-            url = URI.parse( @url )
+        if url = base
+            base_url = URI( url )
+        else
+            base_url = URI( @url )
+        end
 
-            absolute = url.merge(relative)
+        relative = URI( link )
+        absolute = base_url.merge( relative )
 
-            absolute.path = '/' if absolute.path.empty?
-        rescue Exception => e
-            return
-        end
+        absolute.path = '/' if absolute.path && absolute.path.empty?
 
         return absolute.to_s
     end
 
+
+    def base
+        begin
+            tmp = doc.search( '//base[@href]' )
+            return tmp[0]['href'].dup
+        rescue
+            return
+        end
+    end
+
+
     #
     # Returns +true+ if *uri* is in the same domain as the page, returns
     # +false+ otherwise
@@ -508,6 +597,29 @@ class Parser
 
     private
 
+    #
+    # Runs all Spider (path extraction) modules and returns an array of paths
+    #
+    # @return   [Array]   paths
+    #
+    def run_extractors
+        lib = @opts.dir['root'] + 'path_extractors/'
+
+
+        begin
+            @@manager ||= ::Arachni::ComponentManager.new( lib, Extractors )
+
+            return @@manager.available.map {
+                |name|
+                @@manager[name].new.run( doc )
+            }.flatten.uniq
+
+        rescue ::Exception => e
+            print_error( e.to_s )
+            print_debug_backtrace( e )
+        end
+    end
+
     #
     # Merges an array of form inputs with an array of form selects
     #