arachni 1.3.2 → 1.4

data/lib/arachni/option_groups/browser_cluster.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -78,7 +78,9 @@ class BrowserCluster < Arachni::OptionGroup
         return @wait_for_elements = defaults[:wait_for_elements].dup if !rules
 
         @wait_for_elements = rules.inject({}) do |h, (regexp, value)|
-            regexp = regexp.is_a?( Regexp ) ? regexp : Regexp.new( regexp.to_s )
+            regexp = regexp.is_a?( Regexp ) ?
+                regexp :
+                Regexp.new( regexp.to_s, Regexp::IGNORECASE )
             h.merge!( regexp => value )
             h
         end
@@ -87,8 +89,10 @@ class BrowserCluster < Arachni::OptionGroup
     def to_rpc_data
         d = super
 
-        %w(wait_for_elements).each do |k|
-            d[k] = d[k].my_stringify
+        d['wait_for_elements'] = d['wait_for_elements'].dup
+
+        d['wait_for_elements'].dup.each do |k, v|
+            d['wait_for_elements'][k.source] = d['wait_for_elements'].delete(k)
         end
 
         d

data/lib/arachni/option_groups/datastore.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/option_groups/dispatcher.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/option_groups/http.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -35,7 +35,7 @@ class HTTP < Arachni::OptionGroup
 
     # @return   [Array<String>]
     #   Supported proxy types.
-    PROXY_TYPES = %w(http http_1_0 socks4 socks5 socks4a)
+    PROXY_TYPES = %w(http http_1_0 socks4 socks4a socks5 socks5h)
 
     # @return   [Array<String>]
     #   Supported SSL certificate types.

data/lib/arachni/option_groups/input.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -137,7 +137,9 @@ class Input < Arachni::OptionGroup
         return if !values
 
         values.inject({}) do |h, (regexp, value)|
-            regexp = regexp.is_a?( Regexp ) ? regexp : Regexp.new( regexp.to_s )
+            regexp = regexp.is_a?( Regexp ) ?
+                regexp :
+                Regexp.new( regexp.to_s, Regexp::IGNORECASE )
             h.merge!( regexp => value )
             h
         end
@@ -147,7 +149,8 @@ class Input < Arachni::OptionGroup
         h = super
         [:values, :default_values].each do |k|
             # We can't have blocks in there...
-            h[k] = h[k].select{ |_, v| v.is_a? String }.my_stringify
+            h[k] = h[k].select{ |_, v| v.is_a? String }.
+                inject({}) { |h2, (k2, v)| h2.merge k2.source => v }
         end
         h
     end

data/lib/arachni/option_groups/output.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/option_groups/paths.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -80,7 +80,7 @@ class Paths < Arachni::OptionGroup
     end
 
     def self.paths_config_file
-        "#{root_path}config/write_paths.yml"
+        Arachni.get_long_win32_filename "#{root_path}config/write_paths.yml"
     end
 
     def self.clear_config_cache
@@ -106,9 +106,16 @@ class Paths < Arachni::OptionGroup
                     next
                 end
 
-                dir.gsub!( '~', ENV['HOME'] )
+                dir = Arachni.get_long_win32_filename( dir )
+
+                if !Arachni.windows?
+                    dir.gsub!( '~', ENV['HOME'] )
+                end
+
                 dir << '/' if !dir.end_with?( '/' )
 
+                @config[category][subcat] = dir
+
                 FileUtils.mkdir_p dir
             end
         end

data/lib/arachni/option_groups/rpc.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/option_groups/scope.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -81,6 +81,13 @@ class Scope < Arachni::OptionGroup
     # @see URI::Scope#include?
     attr_accessor :include_path_patterns
 
+    # @return    [Array<String>]
+    #   Extension exclusion patterns, resources whose extension is in the list
+    #   will not be considered.
+    #
+    # @see URI::Scope#exclude_file_extension?
+    attr_accessor :exclude_file_extensions
+
     # @return    [Array<Regexp>]
     #   Path exclusion patterns, resources that match any of the specified
     #   patterns will not be considered.
@@ -132,6 +139,7 @@ class Scope < Arachni::OptionGroup
     set_defaults(
         redundant_path_patterns:  {},
         dom_depth_limit:          5,
+        exclude_file_extensions:  Set.new,
         exclude_path_patterns:    [],
         exclude_content_patterns: [],
         include_path_patterns:    [],
@@ -144,12 +152,27 @@ class Scope < Arachni::OptionGroup
         return @url_rewrites = defaults[:url_rewrites].dup if !rules
 
         @url_rewrites = rules.inject({}) do |h, (regexp, value)|
-            regexp = regexp.is_a?( Regexp ) ? regexp : Regexp.new( regexp.to_s )
+            regexp = regexp.is_a?( Regexp ) ?
+                regexp :
+                Regexp.new( regexp.to_s, Regexp::IGNORECASE )
             h.merge!( regexp => value )
             h
         end
     end
 
+    def exclude_file_extensions=( ext )
+        return @exclude_file_extensions =
+            defaults[:exclude_file_extensions].dup if !ext
+
+        if ext.is_a? Set
+            @exclude_file_extensions = ext
+        else
+            @exclude_file_extensions = Set.new(
+                [ext].flatten.compact.map { |s| s.to_s.downcase }
+            )
+        end
+    end
+
     # These options need to contain Array<String>.
     [ :restrict_paths, :extend_paths ].each do |m|
         define_method( "#{m}=".to_sym ) do |arg|
@@ -162,7 +185,9 @@ class Scope < Arachni::OptionGroup
     [ :exclude_content_patterns, :include_path_patterns, :exclude_path_patterns ].each do |m|
         define_method( "#{m}=".to_sym ) do |arg|
             arg = [arg].flatten.compact.
-                map { |s| s.is_a?( Regexp ) ? s : Regexp.new( s.to_s ) }
+                map { |s| s.is_a?( Regexp ) ?
+                s :
+                Regexp.new( s.to_s, Regexp::IGNORECASE ) }
             instance_variable_set( "@#{m}".to_sym, arg )
         end
     end
@@ -209,7 +234,9 @@ class Scope < Arachni::OptionGroup
 
         @redundant_path_patterns =
              filters.inject({}) do |h, (regexp, counter)|
-                 regexp = regexp.is_a?( Regexp ) ? regexp : Regexp.new( regexp.to_s )
+                 regexp = regexp.is_a?( Regexp ) ?
+                     regexp :
+                     Regexp.new( regexp.to_s, Regexp::IGNORECASE )
                  h.merge!( regexp => Integer( counter ) )
                  h
              end
@@ -218,12 +245,14 @@ class Scope < Arachni::OptionGroup
     def to_rpc_data
         d = super
 
+        d['exclude_file_extensions'] = d['exclude_file_extensions'].to_a
+
         %w(redundant_path_patterns url_rewrites).each do |k|
-            d[k] = d[k].my_stringify
+            d[k] = d[k].inject({}) { |h, (k2, v)| h.merge k2.source => v }
         end
 
         %w(exclude_path_patterns exclude_content_patterns include_path_patterns).each do |k|
-            d[k] = d[k].map(&:to_s)
+            d[k] = d[k].map(&:source)
         end
 
         d

data/lib/arachni/option_groups/session.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/option_groups/snapshot.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/options.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/page.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -275,7 +275,7 @@ class Page
         @has_javascript = nil
         clear_cache
 
-        @body = string.to_s.dup.recode.freeze
+        @body = string.to_s.freeze
     end
 
     ELEMENTS.each do |type|
@@ -304,7 +304,7 @@ class Page
     #
     # @see Parser#paths
     def paths
-        @cache[:paths] ||= parser ? parser.paths : []
+        @cache[:paths] ||= (parser ? parser.paths : [])
     end
 
     # @return   [Platform]
@@ -338,7 +338,9 @@ class Page
     # @return   [Nokogiri::HTML]
     #   Parsed {#body HTML} document.
     def document
-        @cache[:document] ||= (parser.nil? ? Nokogiri::HTML( body ) : parser.document)
+        @cache[:document] ||= (parser.nil? ?
+            Arachni::Parser.parse( body ) :
+            parser.document)
     end
 
     # @note Will preserve caches for elements which have been externally modified.
@@ -411,7 +413,7 @@ class Page
         tags.flatten.each do |tag|
             tag = tag.to_s
 
-            next if !(body =~ /<\s*#{tag}/i)
+            next if !body.has_html_tag?( tag )
 
             return false if !document
             return true  if document.css( tag ).any?
@@ -502,10 +504,17 @@ class Page
         self
     end
 
-    def to_initialization_options
+    def to_initialization_options( deep = true )
         h = {}
-        [:body, :cookie_jar, :element_audit_whitelist, :metadata].each do |m|
-            h[m] = try_dup( instance_variable_get( "@#{m}".to_sym ) )
+        h[:body] = @body if @body
+
+        [:cookie_jar, :element_audit_whitelist, :metadata].each do |m|
+            h[m] = instance_variable_get( "@#{m}".to_sym )
+
+            if deep
+                h[m] = try_dup( h[m] )
+            end
+
             h.delete( m ) if !h[m]
         end
 
@@ -525,7 +534,12 @@ class Page
         h[:do_not_audit_elements] = @do_not_audit_elements
 
         h[:dom] = dom.to_h.keys.inject({}) do |dh, k|
-            dh[k] = try_dup( dom.send( k ) )
+            dh[k] = dom.send( k )
+
+            if deep
+                dh[k] = try_dup( dh[k] )
+            end
+
             dh
         end
 
@@ -535,7 +549,7 @@ class Page
     # @return   [Hash]
     #   Data representing this instance that are suitable the RPC transmission.
     def to_rpc_data
-        data        = to_initialization_options.my_stringify_keys(false)
+        data        = to_initialization_options( false ).my_stringify_keys(false)
         data['dom'] = dom.to_rpc_data
         data['element_audit_whitelist'] = element_audit_whitelist.to_a
         data['response'] = data['response'].to_rpc_data
@@ -565,7 +579,7 @@ class Page
 
                         when *ELEMENTS.map(&:to_s)
                             value.map do |e|
-                                Element.const_get(name[0...-1].capitalize.to_sym).from_rpc_data( e )
+                                Element.type_to_class( name[0...-1].to_sym ).from_rpc_data( e )
                             end.to_a
 
                         else
@@ -583,7 +597,7 @@ class Page
     end
 
     def _dump( _ )
-        Marshal.dump( to_initialization_options )
+        Marshal.dump( to_initialization_options( false ) )
     end
 
     def self._load( data )

data/lib/arachni/page/dom.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -72,10 +72,12 @@ class DOM
     def digest=( d )
         return @digest = nil if !d
 
-        if d.include?( url ) || d.include?( page.url )
+        normalized_url = Utilities.normalize_url( url )
+
+        if d.include?( url ) || d.include?( normalized_url )
             d = d.dup
             d.gsub!( url, '' )
-            d.gsub!( page.url, '' )
+            d.gsub!( normalized_url, '' )
         end
 
         @digest = d.freeze
@@ -100,11 +102,11 @@ class DOM
 
     def print_transitions( printer, indent = '' )
         longest_event_size = 0
-        page.dom.transitions.each do |t|
+        @transitions.each do |t|
             longest_event_size = [t.event.to_s.size, longest_event_size].max
         end
 
-        page.dom.transitions.map do |t|
+        @transitions.map do |t|
             padding = longest_event_size - t.event.to_s.size + 1
             time    = sprintf( '%.4f', t.time.to_f )
 
@@ -143,23 +145,19 @@ class DOM
     # @return   [Browser, nil]
     #   Live page in the `browser` if successful, `nil` otherwise.
     def restore( browser, take_snapshot = true )
-        # Preload the associated HTTP response since we've already got it.
-        browser.preload( page )
-
         # First, try to load the page via its DOM#url in case it can restore
         # itself via its URL fragments and whatnot.
         browser.goto url, take_snapshot: take_snapshot
 
-        playables = playable_transitions
+        playables = self.playable_transitions
 
         # If we've got no playable transitions then we're done.
         return browser if playables.empty?
 
-        browser_page = browser.to_page
+        browser_dom = browser.state
 
-        # We were probably led to an out-of-scope page via a JS redirect,
-        # bail out.
-        return if browser_page.code == 0
+        # We were probably led to an out-of-scope page via a JS redirect, bail out.
+        return if !browser_dom
 
         # Check to see if just loading the DOM URL was enough.
         #
@@ -168,7 +166,7 @@ class DOM
         # the document may still be different from when our snapshot was captured.
         #
         # However, this check doesn't cost us much so it's worth a shot.
-        if browser_page.dom === self
+        if browser_dom === self
             browser.print_debug "Loaded snapshot by URL: #{url}"
             return browser
         end
@@ -176,10 +174,7 @@ class DOM
         browser.print_debug "Could not load snapshot by URL (#{url}), " <<
             'will load by replaying transitions.'
 
-        # The URL restore failed, so, navigate to the pure version of the URL and
-        # replay its transitions.
-        browser.preload( page )
-
+        # The URL restore failed, replay its transitions.
         playables.each do |transition|
             next if transition.play( browser )
 
@@ -194,6 +189,15 @@ class DOM
         browser
     end
 
+    def state
+        self.class.new(
+            url:         @url,
+            digest:      @digest,
+            transitions: @transitions.dup,
+            skip_states: @skip_states.dup
+        )
+    end
+
     # @return   [Hash]
     def to_h
         {
@@ -290,14 +294,17 @@ class DOM
 
     protected
 
-    def digest_without_urls( other )
-        if !digest.include?( other.url ) && !digest.include?( other.page.url )
+    def digest_without_urls( dom )
+        normalized_other_url = Utilities.normalize_url( dom.url )
+
+        if !digest.include?( dom.url ) &&
+            !digest.include?( normalized_other_url )
             return digest
         end
 
         d = digest.dup
-        d.gsub!( other.url, '' )
-        d.gsub!( other.page.url, '' )
+        d.gsub!( dom.url, '' )
+        d.gsub!( normalized_other_url, '' )
         d
     end