arachni 0.2.2.1

data/lib/module/output.rb

@@ -0,0 +1,68 @@
+=begin
+                  Arachni
+  Copyright (c) 2010-2011 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+
+  This is free software; you can copy and distribute and modify
+  this program under the term of the GPL v2.0 License
+  (See LICENSE file for details)
+
+=end
+
+
+module Arachni
+module Module
+
+
+#
+# Provides output functionality to the modules via the {Arachni::UI::Output}<br/>
+# prepending the module name to the output message.
+#
+# @author: Tasos "Zapotek" Laskos
+#                                      <tasos.laskos@gmail.com>
+#                                      <zapotek@segfault.gr>
+# @version: 0.1
+#
+module Output
+
+    include Arachni::UI::Output
+
+    alias :o_print_error    :print_error
+    alias :o_print_status   :print_status
+    alias :o_print_info     :print_info
+    alias :o_print_ok       :print_ok
+    alias :o_print_debug    :print_debug
+    alias :o_print_verbose  :print_verbose
+    alias :o_print_line     :print_line
+
+    def print_error( str = '' )
+        o_print_error( self.class.info[:name] + ": " + str )
+    end
+
+    def print_status( str = '' )
+        o_print_status( self.class.info[:name] + ": " + str )
+    end
+
+    def print_info( str = '' )
+        o_print_info( self.class.info[:name] + ": " + str )
+    end
+
+    def print_ok( str = '' )
+        o_print_ok( self.class.info[:name] + ": " + str )
+    end
+
+    def print_debug( str = '' )
+        o_print_debug( self.class.info[:name] + ": " + str )
+    end
+
+    def print_verbose( str = '' )
+        o_print_verbose( self.class.info[:name] + ": " + str )
+    end
+
+    def print_line( str = '' )
+        o_print_line( self.class.info[:name] + ": " + str )
+    end
+
+end
+
+end
+end

data/lib/module/trainer.rb

@@ -0,0 +1,240 @@
+=begin
+                  Arachni
+  Copyright (c) 2010-2011 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+
+  This is free software; you can copy and distribute and modify
+  this program under the term of the GPL v2.0 License
+  (See LICENSE file for details)
+
+=end
+
+require Arachni::Options.instance.dir['lib'] + 'module/element_db'
+require Arachni::Options.instance.dir['lib'] + 'module/output'
+
+module Arachni
+module Module
+
+#
+# Trainer class
+#
+# Analyzes all HTTP responses looking for new auditable elements.
+#
+#                                      <tasos.laskos@gmail.com>
+#                                      <zapotek@segfault.gr>
+# @version: 0.2.1
+#
+class Trainer
+
+    include Output
+    include ElementDB
+
+    attr_writer   :page
+    attr_accessor :http
+    attr_accessor :parser
+
+    def initialize
+      @opts     = Options.instance
+      @updated  = false
+    end
+
+    #
+    # Passes the reponse to {#analyze} for analysis
+    #
+    # @param  [Typhoeus::Response]  res
+    # @param  [Bool]  redir  was the response forcing a redirection?
+    #
+    def add_response( res, redir = false )
+
+        # non text files won't contain any auditable elements
+        type = @http.class.content_type( res.headers_hash )
+        if type.is_a?( String) && !type.substring?( 'text' )
+            return false
+        end
+
+        @parser = Parser.new( Options.instance, res )
+        @parser.url = @page.url
+
+        begin
+            url = res.effective_url
+            url = URI( to_absolute( url ) )
+
+            return if !follow?(  url )
+            return if ( redir && !follow?(  url ) )
+
+            analyze( [ res, redir ] )
+
+        rescue Exception => e
+            print_error( "Invalid URL, probably broken redirection. Ignoring..." )
+            raise e
+        end
+
+    end
+
+    #
+    # Decodes URLs to reverse multiple encodes and removes NULL characters
+    #
+    def url_sanitize( url )
+
+        while( url =~ /%/ )
+            url = ( URI.decode( url ).to_s.unpack( 'A*' )[0] )
+        end
+
+        return URI.encode( url )
+    end
+
+    def follow?( url )
+        @parser.url = @page.url
+
+        return false if !@parser.in_domain?( url )
+        return false if @parser.exclude?( url )
+        return false if !@parser.include?( url )
+        return true
+    end
+
+    #
+    # Returns an updated {Arachni::Parser::Page} object or nil if there waere no updates
+    #
+    # @return  [Page]
+    #
+    def page
+        if( @updated  )
+              @updated = false
+              return  @page
+          else
+              return nil
+        end
+    end
+
+
+    #
+    # Analyzes a response looking for new links, forms and cookies.
+    #
+    # @param   [Typhoeus::Response, Bool]  res
+    #
+    def analyze( res )
+
+        print_debug( 'Started for response with request ID: #' +
+          res[0].request.id.to_s )
+
+        @parser.url = res[0].effective_url.clone
+
+        train_cookies( res[0] )
+
+        # if the response body is the same as the page body and
+        # no new cookies have appeared there's no reason to analyze the page
+        if( res[0].body == @page.html && !@updated )
+            print_debug( 'Page hasn\'t changed, skipping...' )
+            return
+        end
+
+        train_forms( res[0] )
+        train_links( res[0], res[1] )
+
+        if( @updated )
+            @page.html = res[0].body.dup
+
+            begin
+                url           = res[0].request.url
+                # prepare the page url
+                @parser.url = to_absolute( url )
+            rescue Exception => e
+                print_error( "Invalid URL, probably broken redirection. Ignoring..." )
+                # raise e
+            end
+
+            @page.response_headers    = res[0].headers_hash
+            @page.query_vars = @parser.link_vars( @parser.url ).dup
+            @page.url        = @parser.url.dup
+            @page.code       = res[0].code
+            @page.method     = res[0].request.method.to_s.upcase
+
+        end
+
+        print_debug( 'Training complete.' )
+    end
+
+    private
+
+    def to_absolute( url )
+        effective_url = url_sanitize( url )
+        @page.url     = url_sanitize( @page.url )
+
+        begin
+            # prepare the page url
+            return (URI.parse( @page.url ).merge( URI( effective_url ) )).to_s.dup
+        rescue
+            # worst case scenario
+            return @page.url
+        end
+    end
+
+    def train_forms( res )
+        return [], 0 if !@opts.audit_forms
+
+        # @parser.url = res.effective_url.clone
+        forms = @parser.forms( ).clone
+        cforms, form_cnt = update_forms( forms )
+
+        if ( form_cnt > 0 )
+            @page.forms = cforms.flatten
+            @updated = true
+
+            print_debug( 'Found ' + form_cnt.to_s + ' new forms.' )
+        else
+            @page.forms = forms
+        end
+
+    end
+
+    def train_links( res, redir = false )
+        return [], 0  if !@opts.audit_links
+
+        # @parser.url = res.effective_url.clone
+
+        links   = @parser.links.clone
+
+        if( redir )
+            url = to_absolute( res.effective_url )
+            links << Arachni::Parser::Element::Link.new( url, {
+                'href' => url,
+                'vars' => @parser.link_vars( url )
+            } )
+        end
+
+        clinks, link_cnt = update_links( links )
+
+        if ( link_cnt > 0 )
+            @page.links = clinks.flatten
+            @updated = true
+
+            print_debug( 'Found ' + link_cnt.to_s + ' new links.' )
+        else
+            @page.links = links
+        end
+
+    end
+
+    def train_cookies( res )
+
+        cookies = @parser.cookies.clone
+        ccookies, cookie_cnt = update_cookies( cookies )
+
+        if ( cookie_cnt > 0 )
+            @page.cookies = ccookies.flatten
+            @updated = true
+
+            print_debug( 'Found ' + cookie_cnt.to_s + ' new cookies.' )
+        else
+            @page.cookies = cookies
+        end
+
+    end
+
+
+    def self.info
+      { :name  => 'Trainer' }
+    end
+
+end
+end
+end

data/lib/module/utilities.rb

@@ -0,0 +1,110 @@
+=begin
+                  Arachni
+  Copyright (c) 2010-2011 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+
+  This is free software; you can copy and distribute and modify
+  this program under the term of the GPL v2.0 License
+  (See LICENSE file for details)
+
+=end
+
+module Arachni
+module Module
+
+
+#
+# Utilities class
+#
+# Includes some useful methods for the system, the modules etc...
+#
+# @author: Tasos "Zapotek" Laskos
+#                                      <tasos.laskos@gmail.com>
+#                                      <zapotek@segfault.gr>
+# @version: 0.1.1
+#
+module Utilities
+
+    #
+    # Gets path from URL
+    #
+    # @param   [String]   url
+    #
+    # @return  [String]   path
+    #
+    def get_path( url )
+
+        uri  = URI( URI.escape( url ) )
+        path = uri.path
+
+        if !File.extname( path ).empty?
+            path = File.dirname( path )
+        end
+
+        path << '/' if path[-1] != '/'
+        return uri.scheme + "://" + uri.host + path
+    end
+
+    def seed
+        @@seed ||= Digest::SHA2.hexdigest( srand( 1000 ).to_s )
+    end
+
+    def normalize_url( url )
+        begin
+            return URI.encode( URI.decode( url.to_s ) ).to_s.gsub( '[', '%5B' ).gsub( ']', '%5D' )
+        rescue
+            begin
+                return URI.encode( URI.decode( url.to_s ) ).to_s
+            rescue
+                return url
+            end
+        end
+    end
+
+    #
+    # Gets module data files from 'modules/[modtype]/[modname]/[filename]'
+    #
+    # @param    [String]    filename filename, without the path
+    # @param    [Block]     the block to be passed each line as it's read
+    #
+    def read_file( filename, &block )
+
+        # the path of the module that called us
+        mod_path = block.source_location[0]
+
+        # the name of the module that called us
+        mod_name = File.basename( mod_path, ".rb")
+
+        # the path to the module's data file directory
+        path    = File.expand_path( File.dirname( mod_path ) ) +
+            '/' + mod_name + '/'
+
+        file = File.open( path + '/' + filename ).each {
+            |line|
+            yield line.strip
+        }
+
+        file.close
+
+    end
+
+    #
+    # Wraps the "block" in exception handling code and runs it.
+    #
+    # @param    [Block]
+    #
+    def exception_jail( &block )
+        begin
+            block.call
+        rescue Exception => e
+            print_error( e.to_s )
+            print_debug_backtrace( e )
+            raise e
+        end
+    end
+
+    extend self
+
+end
+
+end
+end

data/lib/options.rb

@@ -0,0 +1,547 @@
+=begin
+                  Arachni
+  Copyright (c) 2010-2011 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+
+  This is free software; you can copy and distribute and modify
+  this program under the term of the GPL v2.0 License
+  (See LICENSE file for details)
+
+=end
+
+require 'singleton'
+
+module Arachni
+
+#
+# Options class.
+#
+# Implements the Singleton pattern and formaly defines
+# all of Arachni's runtime options.
+#
+# @author: Tasos "Zapotek" Laskos
+#                                      <tasos.laskos@gmail.com>
+#                                      <zapotek@segfault.gr>
+# @version: 0.1.1
+#
+class Options
+
+    include Singleton
+
+    #
+    # The extension of the profile files.
+    #
+    # @return    [String]
+    #
+    PROFILE_EXT = '.afp'
+
+    #
+    # Holds absolute paths for the directory structure of the framework
+    #
+    # @return    [Hash]
+    #
+    attr_accessor :dir
+
+    #
+    # The URL to audit
+    #
+    # @return    [String,URI]
+    #
+    attr_accessor :url
+
+    #
+    # Show help?
+    #
+    # @return    [Bool]
+    #
+    attr_accessor :help
+
+    #
+    # Output only positive results during the audit?
+    #
+    # @return    [Bool]
+    #
+    attr_accessor :only_positives
+
+    #
+    # Be verbose?
+    #
+    # @return    [Bool]
+    #
+    attr_accessor :arachni_verbose
+
+    #
+    # Output debugging messages?
+    #
+    # @return    [Bool]
+    #
+    attr_accessor :debug
+
+    #
+    # Filters for redundant links
+    #
+    # @return    [Array]
+    #
+    attr_accessor :redundant
+
+    #
+    # Should the crawler obery robots.txt files?
+    #
+    # @return    [Bool]
+    #
+    attr_accessor :obey_robots_txt
+
+    #
+    # How deep to go in the site structure?<br/>
+    # If nil, depth_limit = inf
+    #
+    # @return    [Integer]
+    #
+    attr_accessor :depth_limit
+
+    #
+    # How many links to follow?
+    # If nil, link_count_limit = inf
+    #
+    # @return    [Integer]
+    #
+    attr_accessor :link_count_limit
+
+    #
+    # How many redirects to follow?
+    # If nil, redirect_limit = inf
+    #
+    # @return    [Integer]
+    #
+    attr_accessor :redirect_limit
+
+    #
+    # List modules, based on regexps, and exit?
+    #
+    # @return    [Array<Regexp>]
+    #
+    attr_accessor :lsmod
+
+    #
+    # List reports and exit?
+    #
+    # @return    [Bool]
+    #
+    attr_accessor :lsrep
+
+    #
+    # How many concurrent HTTP requests?
+    #
+    # @return    [Integer]
+    #
+    attr_accessor :http_req_limit
+
+    #
+    # Should Arachni audit links?
+    #
+    # @return    [Bool]
+    #
+    attr_accessor :audit_links
+
+    #
+    # Should Arachni audit forms?
+    #
+    # @return    [Bool]
+    #
+    attr_accessor :audit_forms
+
+    #
+    # Should Arachni audit cookies?
+    #
+    # @return    [Bool]
+    #
+    attr_accessor :audit_cookies
+
+    #
+    # Should Arachni audit HTTP headers?
+    #
+    # @return    [Bool]
+    #
+    attr_accessor :audit_headers
+
+    #
+    # Array of modules to load
+    #
+    # @return    [Array]
+    #
+    attr_accessor :mods
+
+    #
+    # Array of reports to load
+    #
+    # @return    [Array]
+    #
+    attr_accessor :reports
+
+    #
+    # Location of an Arachni Framework Report (.afr) file to load
+    #
+    # @return    [String]
+    #
+    attr_accessor :repload
+
+    #
+    # Where to save the Arachni Framework Profile (.afp) file
+    #
+    # @return    [String]
+    #
+    attr_accessor :save_profile
+
+    #
+    # Location of Arachni Framework Profile (.afp) files to load
+    #
+    # @return    [Array]
+    #
+    attr_accessor :load_profile
+
+
+    attr_accessor :show_profile
+
+    #
+    # The person that authorized the scan<br/>
+    # It will be added to the HTTP "user-agent" and "from" headers.
+    #
+    # @return    [String]
+    #
+    attr_accessor :authed_by
+
+    #
+    # The address of the proxy server
+    #
+    # @return    [String]
+    #
+    attr_accessor :proxy_addr
+
+    #
+    # The port to connect on the proxy server
+    #
+    # @return    [String]
+    #
+    attr_accessor :proxy_port
+
+    #
+    # The proxy password
+    #
+    # @return    [String]
+    #
+    attr_accessor :proxy_pass
+
+    #
+    # The proxy user
+    #
+    # @return    [String]
+    #
+    attr_accessor :proxy_user
+
+    #
+    # The proxy type
+    #
+    # @return    [String]     [http, socks]
+    #
+    attr_accessor :proxy_type
+
+    #
+    # To be populated by the framework
+    #
+    # Parsed cookiejar cookies
+    #
+    # @return    [Hash]     name=>value pairs
+    #
+    attr_accessor :cookies
+
+    #
+    # Location of the cookiejar
+    #
+    # @return    [String]
+    #
+    attr_accessor :cookie_jar
+
+    #
+    # The HTTP user-agent to use
+    #
+    # @return    [String]
+    #
+    attr_accessor :user_agent
+
+    #
+    # Exclude filters <br/>
+    # URL matching any of these patterns won't be followed
+    #
+    # @return    [Array]
+    #
+    attr_accessor :exclude
+
+    #
+    # Cookies to exclude from audit<br/>
+    #
+    # @return    [Array]
+    #
+    attr_accessor :exclude_cookies
+
+    #
+    # Include filters <br/>
+    # Only URLs that match any of these patterns will be followed
+    #
+    # @return    [Array]
+    #
+    attr_accessor :include
+
+    #
+    # Should the crawler follow subdomains?
+    #
+    # @return    [Bool]
+    #
+    attr_accessor :follow_subdomains
+
+    #
+    # Harvest the HTTP responses for the whole site at the end or
+    # for each page?
+    #
+    # @return    [Bool]
+    #
+    attr_accessor :http_harvest_last
+
+    # to be populated by the framework
+    attr_accessor :start_datetime
+    # to be populated by the framework
+    attr_accessor :finish_datetime
+    # to be populated by the framework
+    attr_accessor :delta_time
+
+    attr_accessor :lsplug
+    attr_accessor :plugins
+
+    attr_accessor :spider_first
+
+    attr_accessor :rpc_port
+    attr_accessor :ssl
+    attr_accessor :ssl_pkey
+    attr_accessor :ssl_cert
+    attr_accessor :ssl_ca
+
+    attr_accessor :server
+
+    attr_accessor :reroute_to_logfile
+    attr_accessor :pool_size
+
+
+    def initialize( )
+
+        # nil everything out
+        self.instance_variables.each {
+            |var|
+            instance_variable_set( var.to_s, nil )
+        }
+
+        @exclude    = []
+        @include    = []
+        @redundant  = []
+
+        @reports    = {}
+        @lsrep      = []
+
+        @lsmod      = []
+        @dir        = Hash.new
+        @exclude_cookies    = []
+        @load_profile       = []
+
+        @plugins = {}
+        @lsplug  = []
+
+        # set some defaults
+        @redirect_limit = 20
+
+        # relatively low but will give good performance without bottleneck
+        # on low bandwidth conections
+        @http_req_limit = 20
+
+    end
+
+    #
+    # Saves 'self' to file
+    #
+    # @param    [String]    file
+    #
+    def save( file )
+
+        dir           = @dir.clone
+        load_profile  = @load_profile.clone if @load_profile
+        save_profile  = @save_profile.clone if @save_profile
+        authed_by     = @authed_by.clone if @authed_by
+
+        @dir          = nil
+        @load_profile = nil
+        @save_profile = nil
+        @authed_by    = nil
+
+        begin
+            f = File.open( file + PROFILE_EXT, 'w' )
+            YAML.dump( self, f )
+        rescue
+            return
+        ensure
+            f.close
+
+            @dir          = dir
+            @load_profile = load_profile
+            @save_profile = save_profile
+            @authed_by    = authed_by
+        end
+
+        return f.path
+    end
+
+    def url=( str )
+        return if !str
+
+        require 'uri'
+        require self.dir['lib'] + 'exceptions'
+        require self.dir['lib'] + 'module/utilities'
+
+        begin
+            @url = URI( Arachni::Module::Utilities.normalize_url( str.to_s ) )
+        rescue
+            raise( Arachni::Exceptions::InvalidURL, "Invalid URL argument." )
+        end
+
+        return str
+    end
+
+    #
+    # Converts the Options object to hash
+    #
+    # @return    [Hash]
+    #
+    def to_h
+        hash = Hash.new
+        self.instance_variables.each {
+            |var|
+            hash[normalize_name( var )] = self.instance_variable_get( var )
+        }
+        hash
+    end
+
+    #
+    # Merges self with the object in 'options'
+    #
+    # @param    [Options]
+    #
+    def merge!( options )
+        options.to_h.each_pair {
+            |k, v|
+
+            next if ( v.is_a?( Array ) || v.is_a?( Hash ) ) && v.empty?
+            send( "#{k}=", v ) if v
+        }
+    end
+
+    def to_args
+
+        cli_args = ''
+
+        self.to_h.keys.each {
+            |key|
+
+            arg = self.to_arg( key )
+
+            cli_args += " #{arg.to_s}" if arg
+        }
+
+        return cli_args += " #{self.url}"
+    end
+
+    def to_arg( key )
+
+        var = self.instance_variable_get( "@#{key}" )
+
+        return if !var
+        return if ( var.is_a?( Array ) || var.is_a?( Hash ) ) && var.empty?
+        return if key == 'show_profile'
+        return if key == 'url'
+        return if key == 'dir'
+        return if key == 'include' && var == [/.*/]
+        return if key == 'reports' && var == ['stdout']
+
+        key = 'exclude_cookie' if key == 'exclude_cookies'
+        key = 'report'         if key == 'reports'
+
+        key = key.gsub( '_', '-' )
+
+        arg = ''
+
+        case key
+
+            when 'mods'
+                var = var.join( ',' )
+
+            when 'arachni-verbose'
+                key = 'verbosity'
+
+            when 'redundant'
+                var.each {
+                    |rule|
+                    arg += " --#{key}=#{rule['regexp'].source}:#{rule['count']}"
+                }
+                return arg
+
+            when 'plugins','report'
+                arg = ''
+                var.each {
+                    |opt, val|
+                    arg += " --#{key.chomp( 's' )}=#{opt}"
+                    arg += ':' if !val.empty?
+
+                    val.each {
+                        |k, v|
+                        arg += "#{k}=#{v},"
+                    }
+
+                    arg.chomp!( ',' )
+                }
+                return arg
+
+            when 'proxy-port'
+                return
+
+            when 'proxy-addr'
+                return "--proxy=#{self.proxy_addr}:#{self.proxy_port}"
+
+
+        end
+
+        if( var.is_a?( TrueClass ) )
+            arg = "--#{key}"
+        end
+
+        if( var.is_a?( String ) || var.is_a?( Fixnum ) )
+            arg = "--#{key}=#{var.to_s}"
+        end
+
+        if( var.is_a?( Array ) )
+
+            var.each {
+                |i|
+
+                i = i.source if i.is_a?( Regexp )
+
+                arg += " --#{key}=#{i}"
+            }
+
+        end
+
+        return arg
+    end
+
+    private
+
+    def normalize_name( name )
+        name.to_s.gsub( '@', '' )
+    end
+
+
+end
+end