arachni 0.2.4 → 0.3

data/lib/ui/web/addons/scheduler/views/options.erb

@@ -0,0 +1,93 @@
+
+
+        <div style="display: none" id="dialog_options_<%=job.id%>" title="Options">
+
+            <div class="left options">
+                <fieldset>
+                    <legend>Auditor</legend>
+                    <p>
+                        Audit links: <input disabled type="checkbox" name="audit_links" <% if opts['audit_links'] == true %> checked="checked" <% end %> />
+                    </p>
+                    <p>
+                        Audit forms: <input disabled type="checkbox" name="audit_forms" <% if opts['audit_forms'] == true %> checked="checked" <% end %> />
+                    </p>
+                    <p>
+                        Audit cookies: <input disabled type="checkbox" name="audit_cookies" <% if opts['audit_cookies'] == true %> checked="checked" <% end %> />
+                    </p>
+                    <p>
+                        Audit headers: <input disabled type="checkbox" name="audit_headers" <% if opts['audit_headers'] == true %> checked="checked" <% end %> />
+                    </p>
+                    <p>
+                        Cookies to exclude: <textarea disabled rows="2" cols="20" name="exclude_cookies"><%=opts['exclude_cookies']%></textarea>
+                        <br/>(Newline separated)
+                    </p>
+                </fieldset>
+
+                <fieldset>
+                    <legend>HTTP options</legend>
+                    <p>
+                        Cocurrent HTTP request limit: <label for="http_req_limit"></label>
+                                                      <input disabled id="http_req_limit" name="http_req_limit"/>
+                        <div id="slider"></div>
+                        <br/>
+                    </p>
+                    <p>
+                        HTTP harvest last: <input disabled type="checkbox" name="http_harvest_last" <% if opts['http_harvest_last'] == true %> checked="checked" <% end %> />
+                    </p>
+                    <p>
+                        Cookie jar: <input disabled type="file" name="cookiejar" size="25" />
+                    </p>
+                    <p>
+                        User agent: <input disabled name="user_agent" value="<%=opts['user_agent']%>"/>
+                    </p>
+                    <p>
+                        Authorized by: <input disabled name="authed_by" value="<%=opts['authed_by']%>"/>
+                    </p>
+
+                </fieldset>
+            </div>
+
+            <div class="right options">
+
+                <fieldset>
+                    <legend>Crawler options</legend>
+                    <p>
+                        Exclude rules: <textarea disabled rows="2" cols="20" name="exclude"><% if opts['exclude']%><%=opts['exclude'].join( "\r\n" )%><%end%></textarea>
+                        <br/>(Newline separated)
+                    </p>
+                    <p>
+                    Include rules: <textarea disabled rows="2" cols="20" name="include"><% if opts['include']%><%=opts['include'].join( "\r\n" )%><%end%></textarea>
+                        <br/>(Newline separated)
+                    </p>
+                    <p>
+                        Redundant rules: <textarea disabled rows="2" cols="20" name="redundant"><%=format_redundants( opts['redundant'] )%></textarea>
+                    <br/>(Newline separated)
+                    <br/>(<em>regexp:counter</em>)
+                    </p>
+                    <p>
+                        Depth: <input disabled name="depth_limit" value="<%=opts['depth_limit']%>"/>
+                        <br/>(Default: infinite)
+                    </p>
+                    <p>
+                        Link count limit: <input disabled name="link_count_limit" value="<%=opts['link_count_limit']%>"/>
+                        <br/>(Default: infinite)
+                    </p>
+                    <p>
+                        Redirect limit: <input disabled name="redirect_limit" value="<%=opts['redirect_limit']%>"/>
+                        <br/>(Default: infinite)
+                    </p>
+                    <p>
+                        Follow subdomain: <input disabled type="checkbox" name="follow_subdomains" <% if opts['follow_subdomains'] == true %> checked="checked" <% end %> />
+                    </p>
+                    <p>
+                        Obey robot.txt file: <input disabled type="checkbox" name="obey_robots_txt" <% if opts['obey_robots_txt'] == true %> checked="checked" <% end %> />
+                    </p>
+                    <p>
+                        Spider first: <input disabled type="checkbox" name="spider_first" <% if opts['spider_first'] == true %> checked="checked" <% end %> />
+                    </p>
+                </fieldset>
+
+            </div>
+
+        </div>
+

data/lib/ui/web/dispatcher_manager.rb

@@ -15,9 +15,7 @@ module UI
 module Web
 
 #
-#
-# Provides nice little wrapper for the Arachni::Report::Manager while also handling<br/>
-# conversions, storing etc.
+# Provides methods for dispatcher management.
 #
 # @author: Tasos "Zapotek" Laskos
 #                                      <tasos.laskos@gmail.com>
@@ -44,10 +42,23 @@ class DispatcherManager
         Dispatcher.auto_upgrade!
     end
 
-    def new( opts )
-        Dispatcher.create( :url => opts[:url] )
+    #
+    # Puts a new dispatcher in the DB.
+    #
+    # @param    [String]    url     URL of the dispatcher
+    #
+    def new( url )
+        Dispatcher.first_or_create( :url => url )
     end
 
+    #
+    # Provides an easy way to connect to a dispatcher and caches connections
+    # to reduce overhead.
+    #
+    # @param    [String]   url
+    #
+    # @return   [Arachni::RPC::XML::Client::Dispatcher]
+    #
     def connect( url )
         @@cache ||= {}
 
@@ -59,20 +70,68 @@ class DispatcherManager
                 return @@cache[url] = tmp
             end
         rescue Exception => e
-          return nil
+            # ap e
+            # ap e.backtrace
+            return nil
         end
     end
 
-    def alive?( url )
-        begin
-            return connect( url ).alive?
-        rescue
-            return false
-        end
+    #
+    # Checks wether the dispatcher is alive.
+    #
+    # @param    [String]    url     URL of the dispatcher
+    #
+    def alive?( url, tries = 5 )
+        tries.times {
+            begin
+                return connect( url ).alive?
+            rescue Exception => e
+                # ap e
+                # ap e.backtrace
+            end
+        }
+
+        return false
+    end
+
+    def first_alive
+        all.each {
+            |dispatcher|
+            return dispatcher if alive?( dispatcher.url )
+        }
+
+        return nil
     end
 
     #
-    # Returns the paths of all saved report files as an array
+    # Provides statistics about running jobs etc using the dispatcher
+    #
+    # @return   [Hash]
+    #
+    def stats
+        stats_h = {}
+        all.each {
+            |dispatcher|
+
+            begin
+                stats_h[dispatcher['url']] = connect( dispatcher['url'] ).stats
+                stats_h[dispatcher['url']]['running_jobs'].each {
+                    |job|
+                    begin
+                        instance = @settings.instances.port_to_url( job['port'], dispatcher['url'] )
+                        job['paused'] = @settings.instances.connect( instance ).framework.paused?
+                    rescue
+                    end
+                }
+            rescue
+            end
+        }
+
+        return stats_h
+    end
+
+    #
+    # Returns all dispatchers stored in the DB.
     #
     # @return    [Array]
     #
@@ -80,6 +139,9 @@ class DispatcherManager
         Dispatcher.all( *args )
     end
 
+    #
+    # Removed all dispatchers from the DB.
+    #
     def delete_all
         all.each {
             |report|
@@ -88,6 +150,11 @@ class DispatcherManager
         all.destroy
     end
 
+    #
+    # Removed a dispatcher from the DB.
+    #
+    # @param    [Integer]   id
+    #
     def delete( id )
         Dispatcher.get( id ).destroy
     end

data/lib/ui/web/instance_manager.rb

@@ -0,0 +1,87 @@
+=begin
+                  Arachni
+  Copyright (c) 2010-2011 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+
+  This is free software; you can copy and distribute and modify
+  this program under the term of the GPL v2.0 License
+  (See LICENSE file for details)
+
+=end
+
+require Arachni::Options.instance.dir['lib'] + 'rpc/xml/client/instance'
+require Arachni::Options.instance.dir['lib'] + 'ui/web/utilities'
+
+module Arachni
+module UI
+module Web
+
+#
+# Provides methods for instance management.
+#
+# @author: Tasos "Zapotek" Laskos
+#                                      <tasos.laskos@gmail.com>
+#                                      <zapotek@segfault.gr>
+# @version: 0.1
+#
+class InstanceManager
+
+    include Utilities
+
+    def initialize( opts, settings )
+        @opts     = opts
+        @settings = settings
+    end
+
+    #
+    # Provides an easy way to connect to an instance and caches connections
+    # to reduce overhead.
+    #
+    # @param    [String]   url
+    # @param    [Hash]     session  session of the current user (optional)
+    # @param    [String]   token    authentication token (optional)
+    #
+    # @return   [Arachni::RPC::XML::Client::Instance]
+    #
+    def connect( url, session = nil, token = nil )
+        url = 'https://' + url if !url.include?( 'https' )
+
+        @@connections ||= {}
+
+        begin
+            if @@connections[url] && @@connections[url].alive?
+              return @@connections[url]
+            end
+        rescue
+        end
+
+        begin
+
+            #
+            # Sync up the session authentication tokens with the ones in the
+            # class variables.
+            #
+            # This will allow users to still connect to instances even if they
+            # shutdown the WebUI or removed their cookies.
+            #
+
+            @@tokens ||= {}
+            session['tokens'] ||= {} if session
+            @@tokens[url] = token if token
+
+            session['tokens'].merge!( @@tokens ) if session
+            @@tokens.merge!( session['tokens'] ) if session
+            session['tokens'].merge!( @@tokens ) if session
+
+            tmp_token = session ? session['tokens'][url] : @@tokens[url]
+
+            return @@connections[url] =
+                Arachni::RPC::XML::Client::Instance.new( @opts, url, tmp_token )
+        rescue Exception => e
+            raise "Instance at #{url} has shutdown."
+        end
+    end
+
+end
+end
+end
+end

data/lib/ui/web/scheduler.rb

@@ -0,0 +1,166 @@
+=begin
+                  Arachni
+  Copyright (c) 2010-2011 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+
+  This is free software; you can copy and distribute and modify
+  this program under the term of the GPL v2.0 License
+  (See LICENSE file for details)
+
+=end
+
+require 'datamapper'
+require Arachni::Options.instance.dir['lib'] + 'rpc/xml/client/dispatcher'
+require Arachni::Options.instance.dir['lib'] + 'ui/web/utilities'
+
+module Arachni
+module UI
+module Web
+
+#
+# Schedules and executes scan jobs.
+#
+#
+# @author: Tasos "Zapotek" Laskos
+#                                      <tasos.laskos@gmail.com>
+#                                      <zapotek@segfault.gr>
+# @version: 0.1.1
+#
+class Scheduler
+
+    include Utilities
+
+    class Job
+        include DataMapper::Resource
+
+        property :id,           Serial
+        property :dispatcher,   String
+        property :url,          String
+        property :opts,         Text
+        property :datetime,     DateTime
+
+        property :owner_addr,   String
+        property :owner_host,   String
+
+        property :created_at,   DateTime
+    end
+
+    #
+    # Initializes the Scheduler and starts the clock.
+    #
+    #
+    def initialize( opts, settings )
+        @opts     = opts
+        @settings = settings
+
+        DataMapper::setup( :default, "sqlite3://#{@settings.db}/default.db" )
+        DataMapper.finalize
+
+        Job.auto_upgrade!
+
+        begin
+            ticktock!
+        rescue Exception => e
+            ap e
+            ap e.backtrace
+        end
+    end
+
+    #
+    # Runs a job.
+    #
+    # @param    [Job]   job
+    # @param    [Hash]  Sinatra environment
+    # @param    [Hash]  Rack session
+    #
+    # @return   [String]    URL of the laucnhed scanner instance
+    #
+    def run( job, env = nil, session = nil )
+        instance     = @settings.dispatchers.connect( job.dispatcher ).dispatch( job.url )
+        instance_url = @settings.instances.port_to_url( instance['port'], job.dispatcher )
+
+        env = {
+            'REMOTE_ADDR' => job.owner_addr,
+            'REMOTE_HOST' => job.owner_host
+        } if env.nil?
+
+        @settings.log.scheduler_instance_dispatched( env, instance_url )
+        @settings.log.scheduler_instance_owner_assigned( env, job.url )
+
+        arachni  = @settings.instances.connect( instance_url, session, instance['token'] )
+
+        opts = YAML::load( job.opts )
+
+        arachni.opts.set( opts['settings'] )
+        arachni.modules.load( opts['modules'] )
+        arachni.plugins.load( opts['plugins'] )
+
+        arachni.framework.run
+
+        @settings.log.scheduler_scan_started( env, job.url )
+
+        return instance_url
+    end
+
+    #
+    # Runs a job and removed it from the DB.
+    #
+    # @param    [Job]   job
+    #
+    def run_and_destroy( job )
+        run( job )
+        job.destroy
+    end
+
+    #
+    # Returns all scheduled jobs.
+    #
+    # @return    [Array]
+    #
+    def jobs( *args )
+        Job.all( *args )
+    end
+
+    #
+    # Removes all jobs.
+    #
+    def delete_all
+        jobs.destroy
+    end
+
+    #
+    # Removed a job.
+    #
+    # @param    [Integer]   id
+    #
+    def delete( id )
+        Job.get( id ).destroy
+    end
+
+
+    private
+
+    def ticktock!
+        @reaper ||= Thread.new {
+            while( true )
+                jobs.each {
+                    |job|
+
+                    begin
+                        run_and_destroy( job ) if job.datetime <= Time.now
+                    rescue Exception => e
+                        ap e
+                        ap e.backtrace
+                    end
+
+                }
+
+                ::IO::select( nil, nil, nil, 5 )
+            end
+        }
+
+    end
+
+end
+end
+end
+end