arachni 0.4.0.4 → 0.4.1

data/lib/arachni/session.rb

@@ -0,0 +1,279 @@
+=begin
+    Copyright 2010-2012 Tasos Laskos <tasos.laskos@gmail.com>
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+=end
+
+module Arachni
+
+#
+# Session management class
+#
+# Handles logins, provided log-out detection, stores and executes login sequences
+# and provided general webapp session related helpers.
+#
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+#
+class Session
+    include UI::Output
+    include Utilities
+
+    # @return   [Options]  options
+    attr_reader :opts
+
+    #
+    # A block used to login to the webapp.
+    #
+    # The block should log the framework into the webapp and return +true+ on
+    # success, +false+ on failure.
+    #
+    # @return   [Block]
+    #
+    attr_accessor :login_sequence
+
+    #
+    # A block used to check whether or not we're logged in to the webapp.
+    #
+    # The block should return +true+ on success, +false+ on failure.
+    #
+    # The proc should expect 2 parameters, the first one being a hash of HTTP
+    # options and the second one an optional block.
+    #
+    # If a block has been set, the check should work async and pass the result
+    # to the block, otherwise it should simply return the result.
+    #
+    # The result of the check should be +true+ or +false+.
+    #
+    # A good example of this can be found in {#set_login_check}.
+    #
+    # @return   [Block]
+    #
+    # @see #set_login_check
+    #
+    attr_accessor :login_check
+
+    #
+    # Sets a login form and generates a login sequence from it.
+    #
+    # The form must be kosher, best be generated by one of the {Arachni::Element::Form}
+    # helpers, {Parser} or {#find_login_form}.
+    #
+    # Once you get the right form you need to update it with the appropriate values
+    # before passing it to this accessor.
+    #
+    # @return   [Element::Form]
+    #
+    attr_accessor :login_form
+
+    def initialize( opts = Arachni::Options.instance )
+        @opts = opts
+    end
+
+    # @return   [Array<Element::Cookie>]    session cookies
+    def cookies
+        http.cookies.select{ |c| c.session? }
+    end
+
+    #
+    # Tries to find the main session (login/ID) cookie.
+    #
+    # @param    [Block] block   block to be passed the cookie
+    #
+    def cookie( &block )
+        return block.call( @session_cookie ) if @session_cookie
+        fail 'No login-check has been configured.' if !has_login_check?
+
+        cookies.each do |cookie|
+            logged_in?( cookies: { cookie.name => '' } ) do |bool|
+                next if bool
+                block.call( @session_cookie = cookie )
+            end
+        end
+    end
+
+    #
+    # Finds a login forms based on supplied location, collection and criteria.
+    #
+    # @param    [Hash] opts
+    # @option opts [Bool] :requires_password Does the login form include a password field? (Defaults to +true+)
+    # @option opts [Array, Regexp] :action Regexp to match or String to compare against the form action.
+    # @option opts [String, Array, Hash, Symbol] :inputs Inputs that the form must contain.
+    # @option opts [Array<Element::Form>] :forms Collection of forms to look through.
+    # @option opts [Page, Array<Page>] :pages Pages to look through.
+    # @option opts [String] :url URL to fetch and look for forms.
+    #
+    # @param    [Block] block   if a block and a :url are given, the request
+    #                               will run async and the block will be called
+    #                               with the result of this method.
+    #
+    def find_login_form( opts = {}, &block )
+        async = block_given?
+
+        requires_password = (opts[:requires_password].nil? ? true : opts[:requires_password])
+
+        find = proc do |cforms|
+            cforms.select do |f|
+                next if requires_password && !f.requires_password?
+
+                oks   = []
+
+                if action = opts[:action]
+                    oks << !!(action.is_a?( Regexp ) ? f.action =~ action : f.action == action)
+                end
+
+                if inputs = opts[:inputs]
+                    oks << f.has_inputs?( inputs )
+                end
+
+                oks.count( true ) == oks.size
+            end.first
+        end
+
+        forms = if opts[:pages]
+                    [opts[:pages]].flatten.map { |p| p.forms }.flatten
+                elsif opts[:forms]
+                    opts[:forms]
+                elsif url = opts[:url]
+                    if async
+                        page_from_url( url ) { |p| block.call find.call( p.forms ) }
+                    else
+                        page_from_url( url ).forms
+                    end
+                end
+
+        find.call( forms || [] ) if !async
+    end
+
+    # @return   [Bool]  +true+ if there is log-in capability, +false+ otherwise
+    def can_login?
+        @login_sequence && @login_check
+    end
+
+    # @return   [Bool, nil] +true+ if logged-in, +false+ otherwise, +nil+ if
+    #                           there's no log-in capability
+    def ensure_logged_in
+        return if !can_login?
+        return true if logged_in?
+
+        print_bad 'The scanner has been logged out.'
+        print_info 'Trying to re-login...'
+
+        login
+
+        if !logged_in?
+            print_bad 'Could not re-login.'
+            false
+        else
+            print_ok 'Logged-in successfully.'
+            true
+        end
+    end
+
+    #
+    # Uses the block in {#login_sequence} to login to the webapp.
+    #
+    # @return   [Bool, nil]     +true+ if login was successful, +false+ if not,
+    #                               +nil+ if no {#login_sequence} has been set.
+    #
+    def login
+        login_sequence.call if has_login_sequence?
+    end
+
+    # @return   [Bool]  +true+ if a login sequence exists, +false+ otherwise
+    def has_login_sequence?
+        !!login_sequence
+    end
+
+    #
+    # Uses the block in {#login_check} to check in we're logged in to the webapp.
+    #
+    # @param    [Hash]   http_opts   extra HTTP options to use for the check
+    # @param    [Block]  block       if a block has been provided the check
+    #                                   will be async and the result will be passed
+    #                                   to it, otherwise the method will return
+    #                                   the result.
+    #
+    #
+    # @return   [Bool, nil]     +true+ if we're logged-in, +false+ if not,
+    #                               +nil+ if no {#login_sequence} has been set.
+    #
+    def logged_in?( http_opts = {}, &block )
+        login_check.call( http_opts, block ) if has_login_check?
+    end
+
+    # @return   [Bool]  +true+ if a login check exists, +false+ otherwise
+    def has_login_check?
+        !!login_check
+    end
+
+    def login_check( &block )
+        return @login_check = block if block_given?
+
+        if @opts.login_check_url && @opts.login_check_pattern
+            set_login_check( @opts.login_check_url, @opts.login_check_pattern )
+        end
+
+        @login_check
+    end
+
+    #
+    # A block used to login to the webapp.
+    #
+    # The block should log the framework into the webapp and return +true+ on
+    # success, +false+ on failure.
+    #
+    # @param    [Block] block  if a block has been given it will be set as
+    #                               the login sequence
+    #
+    # @return   [Block]
+    #
+    def login_sequence( &block )
+        if @login_form && !block_given?
+            @login_sequence = proc do
+                @login_form.refresh( update_cookies: true ).
+                    submit( async: false,
+                            update_cookies: true,
+                            follow_location: false ).
+                    response
+            end
+        end
+
+        return @login_sequence if !block_given?
+        @login_sequence = block
+    end
+
+    #
+    # Sets a login check using the provided +url+ and +regexp+.
+    #
+    # @param    [String, #to_s]  url        URL to request
+    # @param    [String, Regexp] pattern   pattern to match against the body of the response
+    #
+    def set_login_check( url, pattern )
+        login_check do |opts, block|
+            bool = nil
+            http.get( url.to_s, opts.merge( async: !!block ) ) do |res|
+                bool = !!res.body.match( pattern )
+                block.call( bool ) if block
+            end
+
+            bool
+        end
+    end
+
+    # @return   [HTTP]  http interface
+    def http
+        HTTP
+    end
+
+end
+end

data/lib/arachni/spider.rb

@@ -1,231 +1,377 @@
 =begin
-                  Arachni
-  Copyright (c) 2010-2012 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+    Copyright 2010-2012 Tasos Laskos <tasos.laskos@gmail.com>
 
-  This is free software; you can copy and distribute and modify
-  this program under the term of the GPL v2.0 License
-  (See LICENSE file for details)
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
 
-=end
+        http://www.apache.org/licenses/LICENSE-2.0
 
-require Arachni::Options.instance.dir['lib'] + 'module/utilities'
-require 'nokogiri'
-require Arachni::Options.instance.dir['lib'] + 'nokogiri/xml/node'
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+=end
 
 module Arachni
 
+lib = Options.dir['lib']
+
+require lib + 'bloom_filter'
+require lib + 'module/utilities'
+require 'nokogiri'
+require lib + 'nokogiri/xml/node'
+
 #
-# Spider class
-#
-# Crawls the URL in opts[:url] and grabs the HTML code and headers.
+# Crawls the target webapp until there are no new paths left.
 #
-# @author: Tasos "Zapotek" Laskos
-#                                      <tasos.laskos@gmail.com>
-#                                      <zapotek@segfault.gr>
-# @version: 0.2.3
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
 #
 class Spider
+    include UI::Output
+    include Utilities
 
-    include Arachni::UI::Output
-    include Arachni::Module::Utilities
-
-    #
-    #
-    # @return [Options]
-    #
+    # @return [Arachni::Options]
     attr_reader :opts
 
-    #
-    # Discovered paths
-    #
-    # @return [Array]
-    #
-    attr_reader :sitemap
-
-    #
-    # URLs that caused redirects
-    #
-    # @return [Array]
-    #
+    # @return [Array<String>]   URLs that caused redirects
     attr_reader :redirects
 
     #
-    # Constructor <br/>
     # Instantiates Spider class with user options.
     #
-    # @param  [Options] opts
+    # @param  [Arachni::Options] opts
     #
-    def initialize( opts )
+    def initialize( opts = Options.instance )
         @opts = opts
 
-        @sitemap   = []
+        @sitemap   = {}
         @redirects = []
-        @on_every_page_blocks = []
+        @paths     = []
+        @visited   = Set.new
 
-        @seed_url = @opts.url.to_s
+        @on_each_page_blocks     = []
+        @on_each_response_blocks = []
+        @on_complete_blocks      = []
 
-        @extend_paths   = @opts.extend_paths   || []
-        @restrict_paths = @opts.restrict_paths || []
+        @pass_pages       = true
+        @pending_requests = 0
 
-        @paths = [ @seed_url ]
+        seed_paths
+    end
 
-        if restricted_to_paths?
-            @paths |= @sitemap = @restrict_paths
-        else
-            @paths |= @extend_paths
-        end
+    def url
+        @opts.url
+    end
+
+    # @return   [Array<String>]  Working paths, paths that haven't yet been followed.
+    #                                You'll actually get a copy of the working paths
+    #                                and not the actual object itself;
+    #                                if you want to add more paths use {#push}.
+    def paths
+        @paths.clone
+    end
 
-        # if we have no 'include' patterns create one that will match
-        # everything, like '.*'
-        @opts.include =[ Regexp.new( '.*' ) ] if @opts.include.empty?
+    # @return   [Array<String>] list of crawled URLs
+    def sitemap
+        @sitemap.keys
     end
 
-    def restricted_to_paths?
-        !@restrict_paths.empty?
+    # @return   [Hash<Integer, String>] list of crawled URLs with their HTTP codes
+    def fancy_sitemap
+        @sitemap
     end
 
     #
-    # Runs the Spider and passes parsed page to the block
+    # Runs the Spider and passes the requested object to the block.
     #
-    # @param [Block] block
+    # @param [Bool] pass_pages_to_block  decides weather the block should be passed [Arachni::Page]s
+    #                           or [Typhoeus::Response]s
+    # @param [Block] block  to be passed each page as visited
     #
-    # @return [Arachni::Parser::Page]
+    # @return [Array<String>]   sitemap
     #
-    def run( parse = true, &block )
-        return if @opts.link_count_limit == 0
+    def run( pass_pages_to_block = true, &block )
+        return if !@opts.crawl?
 
-        visited = []
+        # options could have changed so reseed
+        seed_paths
 
-        opts = {
-            :timeout    => nil,
-            :remove_id  => true,
-            :follow_location => true,
-            :update_cookies  => true
-        }
+        if block_given?
+            pass_pages_to_block ? on_each_page( &block ) : on_each_response( &block )
+        end
 
-        # we need a parser in order to have access to skip() in case
-        # there's a redirect that shouldn't be followed
-        seed_page = http.get( @seed_url, opts.merge( :async => false ) ).response
+        while !done?
+            wait_if_paused
+            while !done? && url = @paths.shift
+                wait_if_paused
 
-        print_status( "[HTTP: #{seed_page.code}] " + seed_page.effective_url )
+                visit( url ) do |res|
+                    obj = if pass_pages_to_block
+                        Page.from_response( res, @opts )
+                    else
+                        Parser.new( res, @opts )
+                    end
 
-        parser = Parser.new( @opts, seed_page )
-        parser.url = @seed_url
-        @paths = parser.paths | [@seed_url]
+                    if @on_each_response_blocks.any?
+                        call_on_each_response_blocks( res )
+                    end
 
-        while( !@paths.empty? )
-            while( !@paths.empty? && url = parser.to_absolute( @paths.pop ) )
-                next if skip?( url ) || visited.include?( url )
+                    if @on_each_page_blocks.any?
+                        call_on_each_page_blocks( pass_pages_to_block ? obj : Page.from_response( res, @opts ) )
+                    end
 
-                wait_if_paused
+                    push( obj.paths )
+                end
+            end
 
-                visited << url
+            http.run
+        end
 
-                http.get( url, opts ).on_complete {
-                    |res|
+        http.run
 
-                    next if parser.skip?( res.effective_url )
+        call_on_complete_blocks
 
-                    print_status( "[HTTP: #{res.code}] " + res.effective_url )
+        sitemap
+    end
 
-                    if parse
-                        page = Arachni::Parser::Page.from_http_response( res, @opts )
-                        paths = page.paths
-                        check_url = page.url
-                    else
-                        c_parser = Parser.new( @opts, res )
-                        paths = c_parser.text? ? c_parser.paths : []
-                        check_url = c_parser.url
-                    end
+    #
+    # Sets blocks to be called every time a page is visited.
+    #
+    # @param    [Block]     block
+    #
+    def on_each_page( &block )
+        fail 'Block is mandatory!' if !block_given?
+        @on_each_page_blocks << block
+        self
+    end
 
-                    if !restricted_to_paths?
-                        @sitemap |= paths
+    #
+    # Sets blocks to be called every time a response is received.
+    #
+    # @param    [Block]     block
+    #
+    def on_each_response( &block )
+        fail 'Block is mandatory!' if !block_given?
+        @on_each_response_blocks << block
+        self
+    end
 
-                        if !res.headers_hash['Location'].empty?
-                            @redirects << res.request.url
-                        end
+    #
+    # Sets blocks to be called once the crawler is done.
+    #
+    # @param    [Block]    block
+    #
+    def on_complete( &block )
+        fail 'Block is mandatory!' if !block_given?
+        @on_complete_blocks << block
+        self
+    end
 
-                        @paths   |= @sitemap - visited
-                    end
+    #
+    # Pushes new paths for the crawler to follow; if the crawler has finished
+    # it will be awaken when new paths are pushed.
+    #
+    # The paths will be sanitized and normalized (cleaned up and converted to absolute ones).
+    #
+    # @param    [String, Array<String>] paths
+    #
+    # @return   [Bool]  true if push was successful,
+    #                       false otherwise (provided empty or paths that must be skipped)
+    #
+    def push( paths )
+        paths = dedup( paths )
+        return false if paths.empty?
 
-                    # call the block...if we have one
-                    if block
-                        exception_jail{
-                            if !skip?( check_url )
-                                block.call( parse ? page.clone : res )
-                            else
-                                print_info( 'Matched skip rule.' )
-                            end
-                        }
-                    end
-                }
-
-                # make sure we obey the link count limit and
-                # return if we have exceeded it.
-                if( @opts.link_count_limit &&
-                    @opts.link_count_limit > 0 &&
-                    visited.size >= @opts.link_count_limit )
-                    http.run
-                    return @sitemap.uniq
-                end
+        @paths |= paths
+        @paths.uniq!
 
-            end
+        # REVIEW: This may cause segfaults, Typhoeus::Hydra doesn't like threads.
+        #Thread.new { run } if idle? # wake up the crawler
+        true
+    end
 
-            http.run
-        end
+    # @return [TrueClass, FalseClass] true if crawl is done, false otherwise
+    def done?
+        idle? || limit_reached?
+    end
+
+    # @return [TrueClass, FalseClass] true if the queue is empty and no
+    #                                           requests are pending, false otherwise
+    def idle?
+        @paths.empty? && @pending_requests == 0
+    end
+
+    # @return [TrueClass] pauses the system on a best effort basis
+    def pause
+        @pause = true
+    end
+
+    # @return [TrueClass] resumes the system on a best effort basis
+    def resume
+        @pause = false
+        true
+    end
+
+    # @return [Bool] true if the system it paused, false otherwise
+    def paused?
+        @pause ||= false
+    end
+
+    private
+
+    def seed_paths
+        push url
+        push @opts.extend_paths
+    end
 
-        return @sitemap.uniq
+    def call_on_each_page_blocks( obj )
+        @on_each_page_blocks.each { |b| exception_jail( false ) { b.call( obj ) } }
     end
 
+    def call_on_each_response_blocks( obj )
+        @on_each_response_blocks.each { |b| exception_jail( false ) { b.call( obj ) } }
+    end
+
+    def call_on_complete_blocks
+        @on_complete_blocks.each { |b| exception_jail( false ) { b.call } }
+    end
+
+    # @return   [Arachni::HTTP]   HTTP interface
     def http
-        Arachni::HTTP.instance
+        HTTP
     end
 
+    #
+    # Decides if a URL should be skipped based on weather it:
+    # * has previously been {#visited?}
+    # * matches a {#redundant?} filter
+    # * matches universal {#skip_path?} options like inclusion and exclusion filters
+    #
+    # @param    [String]    url to check
+    #
+    # @return   [Bool]  true if any of the 3 filters returns true, false otherwise
+    #
     def skip?( url )
-        redundant?( url )
+        visited?( url ) || skip_path?( url )
+    end
+
+    def remove_path_params( url )
+        uri = URI( url ).dup
+        uri.path = uri.path.split( ';' ).first.to_s
+        uri.to_s
+    end
+
+    #
+    # @param    [String]    url
+    #
+    # @return   [Bool]  true if the url has already been visited, false otherwise
+    #
+    def visited?( url )
+        @visited.include?( remove_path_params( url ) )
     end
 
+    # @return   [Bool]  true if the link-count-limit has been exceeded, false otherwise
+    def limit_reached?
+        @opts.link_count_limit > 0 && @visited.size >= @opts.link_count_limit
+    end
+
+    #
+    # Checks is the provided URL matches a redundant filter
+    # and decreases its counter if so.
+    #
+    # If a filter's counter has reached 0 the method returns true.
+    #
+    # @param    [String]    url
+    #
+    # @return   [Bool]  true if the url is redundant, false otherwise
+    #
     def redundant?( url )
-        @opts.redundant.each_with_index {
-            |redundant, i|
+        redundant = @opts.redundant?( url ) do |count, regexp, path|
+            print_info "Matched redundancy rule: #{regexp} for #{path}"
+            print_info "Count-down: #{count}"
+        end
 
-            if( url =~ redundant['regexp'] )
+        print_verbose "Discarding redundant page: #{url}" if redundant
+        redundant
+    end
 
-                if( @opts.redundant[i]['count'] == 0 )
-                    print_verbose( 'Discarding redundant page: \'' + url + '\'' )
-                    return true
-                end
+    def auto_redundant?( url )
+        return false if !@opts.auto_redundant?
+        @auto_redundant ||= Hash.new( 0 )
 
-                print_info( 'Matched redundancy rule: ' +
-                redundant['regexp'].to_s + ' for page \'' +
-                url + '\'' )
+        h = "#{url.split( '?' ).first}#{parse_query( url ).keys.sort}".hash
 
-                print_info( 'Count-down: ' + @opts.redundant[i]['count'].to_s )
+        if @auto_redundant[h] >= @opts.auto_redundant
+            print_verbose "Discarding auto-redundant page: #{url}"
+            return true
+        end
 
-                @opts.redundant[i]['count'] -= 1
-            end
-        }
-        return false
+        @auto_redundant[h] += 1
+        false
     end
 
+    def dedup( paths )
+        return [] if !paths || paths.empty?
+
+        [paths].flatten.uniq.compact.map { |p| to_absolute( p, url ) }.
+            reject { |p| skip?( p ) }.uniq.compact
+    end
 
     def wait_if_paused
-        while( paused? )
-            ::IO::select( nil, nil, nil, 1 )
-        end
+        ::IO::select( nil, nil, nil, 1 ) while( paused? )
     end
 
-    def pause!
-        @pause = true
+    def hit_redirect_limit?
+        @opts.redirect_limit > 0 && @opts.redirect_limit >= @followed_redirects
     end
 
-    def resume!
-        @pause = false
+    def visit( url, opts = {}, &block )
+        return if skip?( url ) || redundant?( url ) || auto_redundant?( url )
+        visited( url )
+
+        @followed_redirects ||= 0
+        @pending_requests += 1
+
+        opts = {
+            timeout:         nil,
+            follow_location: false,
+            update_cookies:  true
+        }.merge( opts )
+
+        wrap = proc do |res|
+            effective_url = normalize_url( res.effective_url )
+
+            if res.redirection?
+                @redirects << res.request.url
+                if hit_redirect_limit? || skip?( res.location )
+                    decrease_pending
+                    next
+                end
+                @followed_redirects += 1
+                push res.location
+            end
+
+            print_status( "[HTTP: #{res.code}] " + effective_url )
+            @sitemap[effective_url] = res.code
+            block.call( res )
+
+            decrease_pending
+        end
+
+        http.get( url, opts, &wrap )
+    rescue
+        decrease_pending
+        nil
     end
 
-    def paused?
-        @pause ||= false
-        return @pause
+    def decrease_pending
+        @pending_requests -= 1
+    end
+
+    def visited( url )
+        @visited << remove_path_params( url )
     end
 
 end