arachni 0.3 → 0.4

data/lib/{parser → arachni/parser}/auditable.rb

@@ -53,6 +53,13 @@ class Auditable
         @auditor
     end
 
+    def override_instance_scope!
+        @override_instance_scope = true
+    end
+
+    def override_instance_scope?
+        @override_instance_scope ||= false
+    end
 
     #
     # Delegate output related methods to the auditor
@@ -142,6 +149,7 @@ class Auditable
         injection_sets( injection_str, opts ).each {
             |elem|
 
+            elem.auditor( get_auditor )
             opts[:altered] = elem.altered.dup
 
             return if skip?( elem )
@@ -168,11 +176,14 @@ class Auditable
     end
 
     #
-    # Injectes the injecton_str in self's values according to formatting options
-    # and returns an array of hashes in the form of:
-    #  <altered_variable>   => <new element>
+    # Injects the injecton_str in self's values according to formatting options
+    # and returns an array of Element permutations.
     #
     # @param    [String]  injection_str  the string to inject
+    # @param    [Hash]    opts           formatting and permutation options
+    #                                       * :skip_orig => skip submission with default/original values (for {Arachni::Parser::Element::Form} elements)
+    #                                       * :format => {Format}
+    #                                       * :param_flip => flip injection value and input name
     #
     # @return    [Array]
     #
@@ -224,13 +235,51 @@ class Auditable
 
         }
 
-        if opts[:param_flip]
+        if opts[:param_flip] #&& !self.is_a?( Arachni::Parser::Element::Cookie )
             elem = self.dup
+
+            # when under HPG mode element auditing is strictly regulated
+            # and when we flip params we essentially create a new element
+            # which won't be on the whitelist
+            elem.override_instance_scope!
+
             elem.altered = 'Parameter flip'
             elem.auditable[injection_str] = seed
             var_combo << elem
         end
 
+        # if there are two password type fields in the form there's a good
+        # chance that it's a 'please retype your password' thing so make sure
+        # that we have a variation which has identical password values
+        if self.is_a?( Arachni::Parser::Element::Form )
+            chash = hash.dup
+            chash = Arachni::Module::KeyFiller.fill( chash )
+            delem = self.dup
+
+            add = false
+            @raw['auditable'].each {
+                |input|
+
+                if input['type'] == 'password'
+                    delem.altered = input['name']
+
+                    opts[:format].each {
+                        |format|
+                        chash[input['name']] =
+                            format_str( injection_str, chash[input['name']], format )
+                    }
+
+                    add = true
+                end
+            } if @raw['auditable']
+
+            if add
+                delem.auditable = chash
+                var_combo << delem
+            end
+        end
+
+
         print_debug_injection_set( var_combo, opts )
 
         return var_combo
@@ -258,6 +307,34 @@ class Auditable
         return "Auditing #{self.type} variable '" + altered + "' of " + @action
     end
 
+    #
+    # Returns am audit identifier string to be registered using {#audited}.
+    #
+    # @param  [Hash]  input
+    # @param  [Hash]  opts
+    #
+    # @return  [String]
+    #
+    def audit_id( injection_str = '', opts = {} )
+        vars = auditable.keys.sort.to_s
+
+        str = ''
+        str += !opts[:no_auditor] ? "#{@auditor.class.info[:name]}:" : ''
+
+        str += "#{@action}:" + "#{self.type}:#{vars}"
+        str += "=#{injection_str.to_s}" if !opts[:no_injection_str]
+        str += ":timeout=#{opts[:timeout]}" if !opts[:no_timeout]
+
+        return str
+    end
+
+    def self.restrict_to_elements!( elements )
+        @@restrict_to_elements = Set.new
+        elements.each {
+            |elem|
+            @@restrict_to_elements << elem
+        }
+    end
 
     private
 
@@ -297,7 +374,7 @@ class Auditable
 
             # make sure that we have a response before continuing
             if !res
-                print_error( 'Failed to get responses, backing out... ' )
+                print_error( 'Failed to get response, backing out...' )
                 next
             else
                 print_status( 'Analyzing response #' + res.request.id.to_s + '...' )  if elem.opts && !elem.opts[:silent]
@@ -353,6 +430,8 @@ class Auditable
     end
 
     def match_regexp_and_log( regexp, res, opts )
+        regexp = regexp.is_a?( Regexp ) ? regexp :
+            Regexp.new( regexp.to_s, Regexp::IGNORECASE )
 
         match_data = res.body.scan( regexp )[0]
         match_data = match_data.to_s
@@ -380,35 +459,36 @@ class Auditable
     end
 
     #
-    # Returns am audit identifier string to be registered using {#audited}.
-    #
-    # @param  [Hash]  input
-    # @param  [Hash]  opts
+    # Checks whether or not an audit has been already performed.
     #
-    # @return  [String]
+    # @param  [String]  elem_audit_id  a string returned by {#audit_id}
     #
-    def audit_id( injection_str, opts = {} )
-        vars = auditable.keys.sort.to_s
+    def audited?( elem_audit_id )
 
-        timeout = opts[:timeout] || ''
-        return "#{@auditor.class.info[:name]}:" +
-          "#{@action}:" + "#{self.type}:" +
-          "#{vars}=#{injection_str.to_s}:timeout=#{timeout}"
-    end
+        opts = {
+            :no_auditor => true,
+            :no_timeout => true,
+            :no_injection_str => true
+        }
 
-    #
-    # Checks whether or not an audit has been already performed.
-    #
-    # @param  [String]  audit_id  a string returned by {#audit_id}
-    #
-    def audited?( audit_id )
-      ret =  @@audited.include?( audit_id )
+        @@restrict_to_elements ||= Set.new
+
+        if @@audited.include?( elem_audit_id )
+            msg = 'Skipping, already audited: ' + elem_audit_id
+            ret = true
+        elsif !get_auditor.override_instance_scope? && !override_instance_scope? &&
+            !@@restrict_to_elements.empty? &&
+            !@@restrict_to_elements.include?( audit_id( nil, opts ) )
+            msg = 'Skipping, out of instance scope.'
+            ret = true
+        else
+            msg = 'Current audit ID: ' + elem_audit_id
+            ret = false
+        end
 
-      msg = 'Current audit ID: ' if !ret
-      msg = 'Skipping, already audited: ' if ret
-      print_debug( msg + audit_id )
+        print_debug( msg )
 
-      return ret
+        return ret
     end
 
     #
@@ -441,8 +521,7 @@ class Auditable
         append = default_str if ( format & Format::APPEND )   != 0
         semicolon = append = null = ''   if ( format & Format::STRAIGHT ) != 0
 
-
-        return semicolon + append + injection_str + null
+        return semicolon + append + injection_str.to_s + null
     end
 
     def print_debug_injection_set( var_combo, opts )
@@ -504,8 +583,8 @@ class Auditable
           print_debug( "|--> Combo: " )
 
           combo.each {
-              |combo|
-              print_debug( "|------> " + combo.to_s )
+              |c_combo|
+              print_debug( "|------> " + c_combo.to_s )
           }
 
         }

data/lib/{parser → arachni/parser}/elements.rb

@@ -1,6 +1,6 @@
 =begin
                   Arachni
-  Copyright (c) 2010-2011 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+  Copyright (c) 2010-2012 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
 
   This is free software; you can copy and distribute and modify
   this program under the term of the GPL v2.0 License
@@ -78,7 +78,7 @@ class Base < Arachni::Element::Auditable
     #
     def initialize( url, raw = {} )
         @raw   = raw.dup
-        @url   = url.dup
+        @url   = url.to_s
     end
 
     #
@@ -104,6 +104,10 @@ class Base < Arachni::Element::Auditable
 
     end
 
+    def dup
+        self.class.new( @url.dup, @raw.dup )
+    end
+
 end
 
 class Link < Base
@@ -111,10 +115,10 @@ class Link < Base
     def initialize( url, raw = {} )
         super( url, raw )
 
-        @action = @raw['href']
+        @action = @raw['href'] || @raw[:href] || @raw['action'] || @raw[:action]
         @method = 'get'
 
-        @auditable = @raw['vars']
+        @auditable = @raw['vars'] || @raw[:vars] || @raw['inputs'] || @raw[:inputs]
         @orig      = @auditable.deep_clone
         @orig.freeze
     end
@@ -131,14 +135,18 @@ class Link < Base
         Arachni::Module::Auditor::Element::LINK
     end
 
-    def audit_id( injection_str, opts = {} )
+    def audit_id( injection_str = '', opts = {} )
         vars = auditable.keys.sort.to_s
-        url = URI( @auditor.page.url ).merge( URI( @action ).path ).to_s
+        url = @action.gsub( /\?.*/, '' )
+
+        str = ''
+        str += !opts[:no_auditor] ? "#{@auditor.class.info[:name]}:" : ''
 
-        timeout = opts[:timeout] || ''
-        return "#{@auditor.class.info[:name]}:" +
-          "#{url}:" + "#{self.type}:" +
-          "#{vars}=#{injection_str.to_s}:timeout=#{timeout}"
+        str += "#{url}:" + "#{self.type}:#{vars}"
+        str += "=#{injection_str.to_s}" if !opts[:no_injection_str]
+        str += ":timeout=#{opts[:timeout]}" if !opts[:no_timeout]
+
+        return str
     end
 
 
@@ -155,17 +163,15 @@ class Form < Base
     def initialize( url, raw = {} )
         super( url, raw )
 
-        @action = @raw['attrs']['action']
-        @method = @raw['attrs']['method']
+        @action = @raw['action'] || @raw[:action] || @raw['attrs']['action']
+        @method = @raw['method'] || @raw[:method] || @raw['attrs']['method']
 
-        @auditable = simple['auditable'] || {}
+        @auditable = @raw[:inputs] || @raw['inputs'] || simple['auditable'] || {}
         @orig      = @auditable.deep_clone
         @orig.freeze
     end
 
     def http_request( url, opts )
-
-
         params   = opts[:params]
         altered  = opts[:altered]
 
@@ -203,7 +209,6 @@ class Form < Base
     end
 
     def id
-
         id = simple['attrs'].to_s
 
         auditable.map {
@@ -213,12 +218,10 @@ class Form < Base
         }
 
         return id
-
     end
 
     def simple
-
-        form = Hash.new
+        form = {}
 
         return form if !@raw || !@raw['auditable'] || @raw['auditable'].empty?
 
@@ -248,7 +251,12 @@ class Cookie < Base
         @action = @url
         @method = 'cookie'
 
-        @auditable = { @raw['name'] => @raw['value'] }
+        if @raw['name']
+            @auditable = { @raw['name'] => @raw['value'] }
+        else
+            @auditable = @raw
+        end
+
         @simple = @auditable.dup
         @auditable.reject! {
             |cookie|

data/lib/{parser → arachni/parser}/page.rb

@@ -1,6 +1,6 @@
 =begin
                   Arachni
-  Copyright (c) 2010-2011 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+  Copyright (c) 2010-2012 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
 
   This is free software; you can copy and distribute and modify
   this program under the term of the GPL v2.0 License
@@ -19,7 +19,7 @@ class Parser
 # @author: Tasos "Zapotek" Laskos
 #                                      <tasos.laskos@gmail.com>
 #                                      <zapotek@segfault.gr>
-# @version: 0.2.1
+# @version: 0.2.2
 #
 class Page
 
@@ -90,12 +90,29 @@ class Page
     #
     attr_accessor :cookiejar
 
+    def self.from_http_response( res, opts )
+        page = Arachni::Parser.new( opts, res ).run
+        page.url = Arachni::Module::Utilities.url_sanitize( res.effective_url )
+        return page
+    end
+
     def initialize( opts = {} )
+
+        @forms = []
+        @links = []
+        @cookies = []
+        @headers = []
+
+        @cookiejar = {}
+        @paths = []
+
+        @response_headers = {}
+        @query_vars       = {}
+
         opts.each {
             |k, v|
             send( "#{k}=", v )
         }
-
     end
 
     def body

data/lib/{parser → arachni/parser}/parser.rb

@@ -1,6 +1,6 @@
 =begin
                   Arachni
-  Copyright (c) 2010-2011 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+  Copyright (c) 2010-2012 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
 
   This is free software; you can copy and distribute and modify
   this program under the term of the GPL v2.0 License
@@ -50,33 +50,33 @@ class Parser
     include Arachni::Module::Utilities
 
     module Extractors
-    #
-    # Base Spider parser class for modules.
-    #
-    # The aim of such modules is to extract paths from a webpage for the Spider to follow.
-    #
-    #
-    # @author: Tasos "Zapotek" Laskos
-    #                                      <tasos.laskos@gmail.com>
-    #                                      <zapotek@segfault.gr>
-    # @version: 0.1
-    # @abstract
-    #
-    class Paths
-
         #
-        # This method must be implemented by all modules and must return an array
-        # of paths as plain strings
+        # Base Spider parser class for modules.
         #
-        # @param    [Nokogiri]  Nokogiri document
+        # The aim of such modules is to extract paths from a webpage for the Spider to follow.
         #
-        # @return   [Array<String>]  paths
         #
-        def run( doc )
+        # @author: Tasos "Zapotek" Laskos
+        #                                      <tasos.laskos@gmail.com>
+        #                                      <zapotek@segfault.gr>
+        # @version: 0.1
+        # @abstract
+        #
+        class Paths
+
+            #
+            # This method must be implemented by all modules and must return an array
+            # of paths as plain strings
+            #
+            # @param    [Nokogiri]  Nokogiri document
+            #
+            # @return   [Array<String>]  paths
+            #
+            def run( doc )
+            end
 
         end
     end
-    end
 
     #
     # @return    [String]    the url of the page
@@ -99,9 +99,13 @@ class Parser
     def initialize( opts, res )
         @opts = opts
 
+        @code = res.code
         @url  = url_sanitize( res.effective_url )
         @html = res.body
         @response_headers = res.headers_hash
+
+        @doc   = nil
+        @paths = nil
     end
 
     #
@@ -112,9 +116,9 @@ class Parser
     def run
 
         # non text files won't contain any auditable elements
-        type = Arachni::HTTP.content_type( @response_headers )
-        if type.is_a?( String) && !type.substring?( 'text' )
+        if !text?
             return Page.new( {
+                :code        => @code,
                 :url         => @url,
                 :query_vars  => link_vars( @url ),
                 :html        => @html,
@@ -151,6 +155,7 @@ class Parser
         end
 
         return Page.new( {
+            :code        => @code,
             :url         => @url,
             :query_vars  => link_vars( @url ),
             :html        => @html,
@@ -165,6 +170,12 @@ class Parser
 
     end
 
+    def text?
+        type = Arachni::HTTP.content_type( @response_headers )
+        return false if !type
+        return type.to_s.substring?( 'text' )
+    end
+
     def doc
       return @doc if @doc
       @doc = Nokogiri::HTML( @html ) if @html rescue nil
@@ -303,7 +314,7 @@ class Parser
             else
                 action = url_sanitize( elements[i]['attrs']['action'] )
             end
-            action = URI.escape( action ).to_s
+            action = uri_encode( action ).to_s
 
             elements[i]['attrs']['action'] = to_absolute( action.clone ).to_s
 
@@ -314,10 +325,7 @@ class Parser
                     elements[i]['attrs']['method'].downcase
             end
 
-            url = URI.parse( URI.escape( elements[i]['attrs']['action'] ) )
-            if !in_domain?( url )
-                next
-            end
+            next if skip?( elements[i]['attrs']['action'] )
 
             elements[i]['textarea'] = form_textareas( form )
             elements[i]['select']   = form_selects( form )
@@ -361,9 +369,7 @@ class Parser
             link['href'] = to_absolute( link['href'] )
 
             if !link['href'] then next end
-            if( exclude?( link['href'] ) ) then next end
-            if( !include?( link['href'] ) ) then next end
-            if !in_domain?( URI.parse( link['href'] ) ) then next end
+            next if skip?( link['href'] )
 
             link['vars'] = {}
             link_vars( link['href'] ).each_pair {
@@ -405,16 +411,20 @@ class Parser
                 k, v = elem['content'].split( ';' )[0].split( '=', 2 )
                 cookies_arr << Element::Cookie.new( @url, { 'name' => k, 'value' => v } )
             }
-        rescue
+        rescue Exception => e
+            # ap e
+            # ap e.backtrace
         end
 
 
         # don't ask me why....
-        if @response_headers.to_s.substring?( 'set-cookie' )
+        if @response_headers.to_s.downcase.substring?( 'set-cookie' )
             begin
                 cookies << ::WEBrick::Cookie.parse_set_cookies( @response_headers['Set-Cookie'].to_s )
                 cookies << ::WEBrick::Cookie.parse_set_cookies( @response_headers['set-cookie'].to_s )
-            rescue
+            rescue Exception => e
+                # ap e
+                # ap e.backtrace
                 return cookies_arr
             end
         end
@@ -457,15 +467,7 @@ class Parser
       @paths = []
       return @paths if !doc
 
-      run_extractors( ).each {
-          |path|
-          next if path.nil? or path.empty?
-          abs = to_absolute( path ) rescue next
-
-          @paths << abs if in_domain?( abs )
-      }
-
-      @paths.uniq!
+      @paths = run_extractors
       return @paths
     end
 
@@ -508,28 +510,37 @@ class Parser
     def to_absolute( link )
 
         begin
-            if URI.parse( link ).host
+            link = normalize_url( link )
+            if uri_parser.parse( link ).host
                 return link
             end
         rescue Exception => e
-            return nil if link.nil?
+            # ap e
+            # ap e.backtrace
+            return nil
         end
 
-        # remove anchor
-        link = URI.encode( link.to_s.gsub( /#[a-zA-Z0-9_-]*$/,'' ) )
+        begin
+            # remove anchor
+            link = uri_encode( link.to_s.gsub( /#[a-zA-Z0-9_-]*$/,'' ) )
 
-        if url = base
-            base_url = URI( url )
-        else
-            base_url = URI( @url )
-        end
+            if url = base
+                base_url = uri_parser.parse( url )
+            else
+                base_url = uri_parser.parse( @url )
+            end
 
-        relative = URI( link )
-        absolute = base_url.merge( relative )
+            relative = uri_parser.parse( link )
+            absolute = base_url.merge( relative )
 
-        absolute.path = '/' if absolute.path && absolute.path.empty?
+            absolute.path = '/' if absolute.path && absolute.path.empty?
 
-        return absolute.to_s
+            return absolute.to_s
+        rescue Exception => e
+            # ap e
+            # ap e.backtrace
+            return nil
+        end
     end
 
 
@@ -543,11 +554,20 @@ class Parser
     end
 
 
+    def too_deep?( url )
+        if @opts.depth_limit > 0 && (@opts.depth_limit + 1) <= URI(url.to_s).path.count( '/' )
+            return true
+        else
+            return false
+        end
+    end
+
     #
     # Returns +true+ if *uri* is in the same domain as the page, returns
     # +false+ otherwise
     #
     def in_domain?( uri )
+
         curi = URI.parse( normalize_url( uri.to_s ) )
 
         if( @opts.follow_subdomains )
@@ -589,11 +609,24 @@ class Parser
 
         @opts.include.each {
             |pattern|
+            pattern = Regexp.new( pattern ) if pattern.is_a?( String )
             return true if url.to_s =~ pattern
         }
         return false
     end
 
+    def skip?( path )
+        return true if !path
+
+        begin
+            return true if !include?( path )
+            return true if exclude?( path )
+            return true if too_deep?( path )
+            return true if !in_domain?( path )
+        rescue
+            true
+        end
+    end
 
     private
 
@@ -603,20 +636,20 @@ class Parser
     # @return   [Array]   paths
     #
     def run_extractors
-        lib = @opts.dir['root'] + 'path_extractors/'
-
-
         begin
-            @@manager ||= ::Arachni::ComponentManager.new( lib, Extractors )
+            @@manager ||=
+                ::Arachni::ComponentManager.new( @opts.dir['path_extractors'], Extractors )
 
             return @@manager.available.map {
                 |name|
                 @@manager[name].new.run( doc )
-            }.flatten.uniq
+            }.flatten.uniq.compact.
+            map { |path| to_absolute( url_sanitize( path ) ) }.
+            reject { |path| skip?( path ) }
 
         rescue ::Exception => e
             print_error( e.to_s )
-            print_debug_backtrace( e )
+            print_error_backtrace( e )
         end
     end
 
@@ -641,7 +674,11 @@ class Parser
         i = new_arr.size
         selects.each {
             |select|
-            select['attrs']['value'] = select['options'][0]['value']
+
+            begin
+                select['attrs']['value'] = select['options'][0]['value']
+            rescue
+            end
             new_arr << select['attrs']
         }