arachni 1.0.2 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f1ab8801aa396dbd2a6aa10c911a91777b046542
-  data.tar.gz: 2251a40388dec9fc771bc8aa51ddc7680bb4f825
+  metadata.gz: fe4bb5082b1faba59faf683145dd95841b0299a2
+  data.tar.gz: 83374892882e248fb3334b7651c7f166c99c3bc2
 SHA512:
-  metadata.gz: ccf69071cb3b2ddb1d880041979ac24d8d2f0d61662eadae5ad77544cbd226d47aa746e8505022d03b466290bece49410efc61957d0fc3fbd9359e5361d0b198
-  data.tar.gz: 8d3f66447ef08172a43b29605b1f611131a0565a06ba70c706ee199d191043ee78c93b1eb8bef80d95e7215950e54a1ad3d650e3d6d440cc9fb5ab2babbb21f1
+  metadata.gz: def0427f488fd60f1aed48b0a38d2cb3e3deff96c3270305f27a5a7e2e9a00d01074716b492169ef6e1f45984ac84f89ef39e81b1d155791b73aa560c40f5e33
+  data.tar.gz: 0edb331de9533d6f5b9a63d3fdcb3f4a7f64697ce6f0f3a98f610a53d41629037d50c01cee7037d28afa9b2489edaa82fd97dee4311c9c4188f4336e78815600

data/CHANGELOG.md

@@ -1,5 +1,62 @@
 # ChangeLog
 
+## 1.0.3 _(October 3, 2014)_
+
+- Added overrides for system write directories in `config/write_paths.yml`.
+- `OptionGroups`
+    - `Paths`
+        - Added `.config` -- Parsing `config/write_paths.yml`.
+        - `#logs` -- Can now be set via `.config`.
+        - `#snapshots` -- Can now be set via `.config`.
+    - `Snapshot`
+        - `#save_path` -- Can now be set via `Paths.config`.
+- `UI::Output`
+    - Moved default error log under `OptionGroups::Paths.logs`.
+    - Optimized file descriptor handling.
+- `UI::CLI`
+    - `OptionParser`
+        - Set default report location save-dir from `OptionGroups::Paths.config`.
+    - `Framework`
+        - Print the error-log location at the end of the scan if there were errors.
+- `Framework`
+    - Use `OptionGroups::Scope#extend_paths` to seed the crawl.
+- `Browser`
+    - `ElementLocator.supported_element_attributes_for`
+        - Fixed `nil`-error when dealing with unknown attributes.
+    - Added `:ignore_scope` option, allowing the browser to roam completely
+        unrestricted.
+    - Capped `setTimeout` waiting period to `OptionGroups::HTTP#request_timeout`.
+    - Fixed issue resulting in multiple cookies with the same name being sent
+        to the web application.
+    - Assigned unique custom IDs to DOM elements without ID attributes.
+- `BrowserCluster`
+    - Spawn browsers in series instead of in parallel to make it easier on
+        low resource systems.
+- `Session`
+    - Fallback to `Framework` DOM Level 1 handlers when no `Browser` is available.
+    - When `OptionGroups::Scope#dom_depth_limit` is 0 don't use the `Browser`.
+    - Configured its `Browser` with `:ignore_scope` to allow for SSO support.
+    - `#logged_in?` -- Follow redirections for login check HTTP request.
+- `Element`
+    - `Cookie`
+        - Added `#data` -- Providing access to raw cookie data.
+    - `Capabilities`
+        - `Analyzable`
+            - `Differential` -- Forcibly disable `OptionGroups::Audit#cookies_extensively`.
+            - `Timeout` -- Forcibly disable `OptionGroups::Audit#cookies_extensively`.
+        - `Mutable`
+            - `#each_mutation` -- Removed obsolete method-switch with default inputs.
+- Plugins
+    - `uncommon_headers`
+        - Added `keep-alive` and `content-disposition` in the common list.
+        - Ignore out-of-scope responses.
+    - `content_types`
+        - Ignore out-of-scope responses.
+    - `cookie_collector`
+        - `Set-Cookie` header is now always an `Array`.
+    - `autologin`
+        - Don't modify `OptionGroups::Session` (login-check) options if already set.
+
 ## 1.0.2 _(September 13, 2014)_
 
 - `UI::Output` -- Updated null output interface with placeholder debugging methods.

data/README.md

@@ -1,17 +1,9 @@
-**NOTICE**:
-
-* Arachni's license has changed, please see the _LICENSE_ file before working
-    with the project.
-* v1.0 is not backwards compatible with v0.4.
-
-<hr/>
-
 # Arachni - Web Application Security Scanner Framework
 
 <table>
     <tr>
         <th>Version</th>
-        <td>1.0.2</td>
+        <td>1.0.3</td>
     </tr>
     <tr>
         <th>Homepage</th>

data/bin/arachni_script

@@ -9,8 +9,8 @@
 
 $LOAD_PATH.unshift( File.expand_path( File.dirname( __FILE__ ) + '/../lib' ) )
 
-require_relative '../ui/cli/output'
 require 'arachni'
+require_relative '../ui/cli/output'
 
 include Arachni
 include Utilities

data/components/checks/active/xss_dom_script_context.rb

@@ -68,7 +68,7 @@ Injects JS taint code and checks to see if it gets executed as proof of vulnerab
             version:     '0.1',
 
             issue:       {
-                name:            %q{DOM-based Cross-Site Scripting (XSS)},
+                name:            %q{DOM-based Cross-Site Scripting (XSS) in script context},
                 description:     %q{
 Client-side scripts are used extensively by modern web applications.
 They perform from simple functions (such as the formatting of text) up to full

data/components/checks/active/xss_event.rb

@@ -94,7 +94,7 @@ class Arachni::Checks::XssEvent < Arachni::Check::Base
             version:     '0.1.5',
 
             issue:       {
-                name:            %q{Cross-Site Scripting in event tag of HTML element},
+                name:            %q{Cross-Site Scripting (XSS) in event tag of HTML element},
                 description:     %q{
 Client-side scripts are used extensively by modern web applications.
 They perform from simple functions (such as the formatting of text) up to full

data/components/checks/active/xss_script_context.rb

@@ -139,7 +139,7 @@ Injects JS taint code and check to see if it gets executed as proof of vulnerabi
             version:     '0.2',
 
             issue:       {
-                name:            %q{Cross-Site Scripting in HTML \'script\' tag},
+                name:            %q{Cross-Site Scripting (XSS) in script context},
                 description:     %q{
 Client-side scripts are used extensively by modern web applications.
 They perform from simple functions (such as the formatting of text) up to full

data/components/plugins/autologin.rb

@@ -48,8 +48,8 @@ class Arachni::Plugins::AutoLogin < Arachni::Plugin::Base
             return
         end
 
-        framework.options.session.check_url     = response.url
-        framework.options.session.check_pattern = @verifier
+        framework.options.session.check_url     ||= response.url
+        framework.options.session.check_pattern ||= @verifier
 
         if !session.logged_in?
             register_results(

data/components/plugins/content_types.rb

@@ -9,7 +9,6 @@
 # Logs content-types of all server responses.
 #
 # @author Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>
-# @version 0.1.6
 class Arachni::Plugins::ContentTypes < Arachni::Plugin::Base
 
     is_distributable
@@ -29,7 +28,7 @@ class Arachni::Plugins::ContentTypes < Arachni::Plugin::Base
     end
 
     def run
-        framework.http.on_complete do |response|
+        http.on_complete do |response|
             next if skip?( response )
 
             type = response.headers.content_type
@@ -47,8 +46,8 @@ class Arachni::Plugins::ContentTypes < Arachni::Plugin::Base
     end
 
     def skip?( response )
-        logged?( response ) || response.headers.content_type.to_s.empty? ||
-            !log?( response )
+        response.scope.out? || logged?( response ) ||
+            response.headers.content_type.to_s.empty? || !log?( response )
     end
 
     def log?( response )
@@ -97,7 +96,7 @@ It can help you categorize and identify publicly available file-types which in
 turn can help you identify accidentally leaked files.
 },
             author:      'Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>',
-            version:     '0.1.6',
+            version:     '0.1.7',
             options:     [
                 Options::String.new( :exclude,
                     description: 'Exclude content-types that match this regular expression.',

data/components/plugins/cookie_collector.rb

@@ -9,7 +9,7 @@
 # Simple cookie collector
 #
 # @author Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>
-# @version 0.2
+# @version 0.2.1
 class Arachni::Plugins::CookieCollector < Arachni::Plugin::Base
 
     is_distributable
@@ -42,9 +42,12 @@ class Arachni::Plugins::CookieCollector < Arachni::Plugin::Base
         response_hash.delete( :body )
         response_hash.delete( :headers_string )
 
+        r_h = response_hash.my_stringify_keys
+        r_h['headers']['Set-Cookie'] = [r_h['headers']['Set-Cookie']].flatten.compact
+
         @cookies << {
             'time'     => Time.now.to_s,
-            'response' => response_hash.my_stringify_keys,
+            'response' => r_h,
             'cookies'  => cookies
         }
     end
@@ -81,7 +84,7 @@ thousands of results leading to a huge report, highly increased memory
 consumption and CPU usage.
 },
             author:      'Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>',
-            version:     '0.2',
+            version:     '0.2.1',
             options:     [
                 Options::String.new( :filter,
                     description: 'Pattern to use to determine which cookies to ' +

data/components/plugins/uncommon_headers.rb

@@ -32,7 +32,9 @@ class Arachni::Plugins::UncommonHeaders < Arachni::Plugin::Base
          'proxy-authenticate',
          'set-cookie',
          'trailer',
-         'transfer-encoding'
+         'transfer-encoding',
+         'keep-alive',
+         'content-disposition'
     ])
 
     def prepare
@@ -52,6 +54,8 @@ class Arachni::Plugins::UncommonHeaders < Arachni::Plugin::Base
 
     def run
         http.on_complete do |response|
+            next if response.scope.out?
+
             headers = response.headers.
                 select { |name, _| !COMMON.include?( name.to_s.downcase ) }
             next if headers.empty?
@@ -85,7 +89,7 @@ class Arachni::Plugins::UncommonHeaders < Arachni::Plugin::Base
             name:        'Uncommon headers',
             description: %q{Intercepts HTTP responses and logs uncommon headers.},
             author:      'Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>',
-            version:     '0.1.2'
+            version:     '0.1.3'
         }
     end
 

data/lib/arachni/browser.rb

@@ -126,6 +126,8 @@ class Browser
         super()
         @options = options.dup
 
+        @ignore_scope = options[:ignore_scope]
+
         @width  = options[:width]  || 1600
         @height = options[:height] || 1200
 
@@ -310,6 +312,8 @@ class Browser
             wait_for_timers
 
             wait_for_pending_requests
+
+            javascript.set_element_ids
         end
 
         if @add_request_transitions
@@ -828,7 +832,12 @@ class Browser
     def wait_for_timers
         delay = load_delay
         return if !delay
-        sleep delay / 1000.0
+
+        sleep [Options.http.request_timeout, delay].min / 1000.0
+    end
+
+    def skip_path?( path )
+        enforce_scope? && super( path )
     end
 
     def response
@@ -998,6 +1007,13 @@ class Browser
             kill_process
         end
 
+        # Something went really bad, the browser couldn't be spawned even
+        # after our valiant efforts.
+        #
+        # Bail out for now and count on the BrowserCluster to retry to boot
+        # another process ass needed.
+        return if !@process
+
         begin
             @pid = @process.pid
         # Not supported on JRuby on MS Windows.
@@ -1077,6 +1093,8 @@ class Browser
 
         set_cookies = {}
         HTTP::Client.cookie_jar.for_url( url ).each do |cookie|
+            cookie = cookie.dup
+            cookie.data.delete :domain
             set_cookies[cookie.name] = cookie
         end
         cookies.each do |name, value|
@@ -1181,7 +1199,7 @@ class Browser
             transition.complete
         end
 
-        return if response.scope.out?
+        return if enforce_scope? && response.scope.out?
 
         intercept response
         save_response response
@@ -1203,6 +1221,8 @@ class Browser
     end
 
     def ignore_request?( request )
+        return if !enforce_scope?
+
         # Only allow CSS and JS resources to be loaded from out-of-scope domains.
         !['css', 'js'].include?( request.parsed_url.resource_extension ) &&
             request.scope.out? || request.scope.redundant?
@@ -1304,6 +1324,10 @@ class Browser
         end
     end
 
+    def enforce_scope?
+        !@ignore_scope
+    end
+
     def normalize_watir_url( url )
         normalize_url( ::URI.encode( url, ';' ) ).gsub( '%3B', '%253B' )
     end

data/lib/arachni/browser/element_locator.rb

@@ -127,8 +127,15 @@ class ElementLocator
     #   List of attributes supported by Watir.
     def self.supported_element_attributes_for( tag_name )
         @supported_element_attributes_for ||= {}
-        @supported_element_attributes_for[tag_name.to_sym] ||=
-            Set.new( Watir.tag_to_class[tag_name.to_sym].attribute_list )
+
+        tag_name = tag_name.to_sym
+
+        if (klass = Watir.tag_to_class[tag_name])
+            @supported_element_attributes_for[tag_name] ||=
+                Set.new( klass.attribute_list )
+        else
+            @supported_element_attributes_for[tag_name] ||= Set.new
+        end
     end
 
 end

data/lib/arachni/browser/javascript.rb

@@ -228,6 +228,12 @@ class Javascript
         taint_tracer.flush_data_flow_sinks
     end
 
+    # Sets a custom ID attribute to elements with events but without a proper ID.
+    def set_element_ids
+        return '' if !supported?
+        dom_monitor.setElementIds
+    end
+
     # @return   [String]
     #   Digest of the current DOM tree (i.e. node names and their attributes
     #   without text-nodes).

data/lib/arachni/browser/javascript/scripts/dom_monitor.js

@@ -142,5 +142,44 @@ var _tokenDOMMonitor = _tokenDOMMonitor || {
     registerEvent: function ( element, event, handler ) {
         if( !('events' in element) ) element['events'] = [];
         element['events'].push( [event, handler] );
+    },
+
+    // Sets a unique enough custom ID attribute to elements that lack proper IDs.
+    // This gets called externally (by the Browser) once the page is settled.
+    setElementIds: function() {
+        var elements = document.getElementsByTagName("*");
+        var length   = elements.length;
+
+        for( var i = 0; i < length; i++ ) {
+            var element = elements[i];
+
+            // Window and others don't have attributes.
+            if( typeof( element.getAttribute ) !== 'function' ||
+                typeof( element.setAttribute) !== 'function' ) continue;
+
+            // If the element has an ID we're cool, move on.
+            if( element.getAttribute('id') ) continue;
+
+            // Skip invisible elements.
+            if( element.offsetWidth <= 0 && element.offsetHeight <= 0 ) continue;
+
+            // We don't care about elements without events.
+            if( !element.events || element.events.length == 0 ) continue;
+
+            element.setAttribute( 'data-arachni-id', _tokenDOMMonitor.hashCode( element.innerHTML ) );
+        }
+    },
+
+    hashCode: function( str ) {
+        var hash = 0;
+        if( str.length == 0 ) return hash;
+
+        for( var i = 0; i < str.length; i++ ) {
+            var char = str.charCodeAt( i );
+            hash = ((hash << 5) - hash) + char;
+            hash = hash & hash; // Convert to 32bit integer
+        }
+
+        return hash;
     }
 };

data/lib/arachni/browser_cluster.rb

@@ -384,32 +384,18 @@ class BrowserCluster
     def initialize_workers
         print_status "Initializing #{pool_size} browsers..."
 
-        # Calculate maximum HTTP connection concurrency for each worker based on
-        # the HTTP request concurrency setting of the framework.
-        #
-        # Ideally, we'd throttle the collective connections of all browsers
-        # for optimal concurrency, but that would require all browsers sharing
-        # the same proxy which would make things **really** dirty and complicated
-        # so let's avoid that for as long as possible.
-        #concurrency = [(Options.http.request_concurrency / pool_size).to_i, 1].max
-
         @workers = []
-        workers  = Queue.new
-
-        pool_size.times do
-            Thread.new do
-                workers << Worker.new(
-                    javascript_token: @javascript_token,
-                    master:           self,
-                    width:            Options.browser_cluster.screen_width,
-                    height:           Options.browser_cluster.screen_height
-                    #concurrency:      concurrency
-                )
-            end
-        end
-
-        pool_size.times do
-            @workers << workers.pop.tap { |b| @consumed_pids << b.pid }
+        pool_size.times do |i|
+            worker = Worker.new(
+                javascript_token: @javascript_token,
+                master:           self,
+                width:            Options.browser_cluster.screen_width,
+                height:           Options.browser_cluster.screen_height
+            )
+            @workers << worker
+            @consumed_pids << worker.pid
+
+            print_status "Spawned ##{i+1} with PID #{worker.pid}."
         end
         @consumed_pids.compact!