arachni 1.4 → 1.6.0

data/lib/arachni/option_groups/scope.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -14,6 +14,38 @@ module Arachni::OptionGroups
 # @author Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>
 class Scope < Arachni::OptionGroup
 
+    EXCLUDE_MIME_TYPES = {
+      # Media
+      image:       %w(
+            gif bmp tif tiff jpg jpeg jpe pjpeg png ico psd xcf 3dm max svg eps
+            drw ai
+        ),
+      video:       %w(asf rm mpg mpeg mpe 3gp 3g2  avi flv mov mp4 swf vob wmv),
+      audio:       %w(aif mp3 mpa ra wav wma mid m4a ogg flac),
+
+      # Generic data
+      archive:     %w(zip zipx tar gz 7z rar bz2),
+      disk:        %w(bin cue dmg iso mdf vcd raw),
+
+      # Executables -- or thereabouts.
+      application: %w(exe apk app jar pkg deb rpm msi),
+
+      # Assets
+      #
+      # The browsers will not check the scope for asset files, so these shouldn't
+      # mess with it, they should only narrow down the audit.
+      font:        %w(ttf otf woff woff2 fon fnt),
+      stylesheet:  %w(css),
+      script:      %w(js),
+
+      # Documents
+      #
+      # Allow rtf, ps, xls, doc, ppt, ppts since they can contain greppable text.
+      document:    %w(pdf docx xlsx pptx odt odp),
+    }
+
+    EXCLUDE_FILE_EXTENSIONS = Set.new( EXCLUDE_MIME_TYPES.values.flatten.uniq )
+
     # @note `nil` is infinite -- default is `nil`.
     #
     # @return    [Integer]
@@ -30,6 +62,22 @@ class Scope < Arachni::OptionGroup
     # @see Page::Scope#dom_depth_limit_reached?
     attr_accessor :dom_depth_limit
 
+    # @note `nil` is infinite -- default is `nil`.
+    #
+    # @return    [Integer]
+    #   How many DOM events to trigger for each snapshot.
+    #
+    # @see Browser#trigger_events
+    attr_accessor :dom_event_limit
+
+    # @note `nil` is infinite -- default is `nil`.
+    #
+    # @return    [Integer]
+    #   How many elements should inherit the DOM events of their parents.
+    #
+    # @see Browser#trigger_events
+    attr_accessor :dom_event_inheritance_limit
+
     # @note `nil` is infinite -- default is `nil`.
     #
     # @return    [Integer]
@@ -137,9 +185,11 @@ class Scope < Arachni::OptionGroup
     attr_accessor :url_rewrites
 
     set_defaults(
+        directory_depth_limit:    10,
+        auto_redundant_paths:     15,
         redundant_path_patterns:  {},
-        dom_depth_limit:          5,
-        exclude_file_extensions:  Set.new,
+        dom_depth_limit:          4,
+        exclude_file_extensions:  EXCLUDE_FILE_EXTENSIONS,
         exclude_path_patterns:    [],
         exclude_content_patterns: [],
         include_path_patterns:    [],
@@ -193,7 +243,7 @@ class Scope < Arachni::OptionGroup
     end
 
     def auto_redundant?
-        !!@auto_redundant_paths
+        @auto_redundant_paths.to_i > 0
     end
 
     def auto_redundant_counter
@@ -216,6 +266,10 @@ class Scope < Arachni::OptionGroup
         page_limit && page_limit.to_i > 0 && count >= page_limit
     end
 
+    def dom_event_limit_reached?( count )
+        dom_event_limit && count >= dom_event_limit
+    end
+
     # Sets the redundancy filters.
     #
     # Filter example:

data/lib/arachni/option_groups/session.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/option_groups/snapshot.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/option_groups.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/options.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -96,6 +96,9 @@ class Options
     #   The URL to audit.
     attr_reader   :url
 
+    # @return    [Arachni::URI]
+    attr_reader   :parsed_url
+
     # @return    [Array<String, Symbol>]
     #   Checks to load, by name.
     #
@@ -210,22 +213,36 @@ class Options
         parsed = Arachni::URI( url.to_s )
 
         if parsed.to_s.empty? || !parsed.absolute?
+
             fail Error::InvalidURL,
                  'Invalid URL argument, please provide a full absolute URL and try again.'
-        elsif %w(localhost 127.0.0.1).include? parsed.host
-            fail Error::ReservedHostname, "'#{parsed.host}' is reserved, please use a different hostname."
+
+        # PhantomJS won't proxy localhost.
+        elsif parsed.host == 'localhost' || parsed.host.start_with?( '127.' )
+
+            fail Error::ReservedHostname,
+                 "Loopback interfaces (like #{parsed.host}) are not supported," <<
+                     ' please use a different IP address or hostname.'
+
         else
+
             if scope.https_only? && parsed.scheme != 'https'
+
                 fail Error::InvalidURL,
                      "Invalid URL argument, the 'https-only' option requires"+
                          ' an HTTPS URL.'
+
             elsif !%w(http https).include?( parsed.scheme )
+
                 fail Error::InvalidURL,
                      'Invalid URL scheme, please provide an HTTP or HTTPS URL and try again.'
+
             end
+
         end
 
-        @url = parsed.to_s
+        @parsed_url = parsed
+        @url        = parsed.to_s
     end
 
     # Configures options via a Hash object.
@@ -333,6 +350,7 @@ class Options
         hash = hash.deep_clone
 
         hash.delete( 'url' ) if !hash['url']
+        hash.delete( 'parsed_url' )
 
         hash
     end
@@ -349,6 +367,7 @@ class Options
         end
 
         hash.delete( :url ) if !hash[:url]
+        hash.delete( :parsed_url )
         hash.delete(:paths)
 
         hash.deep_clone

data/lib/arachni/page/dom/transition.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -194,7 +194,10 @@ class Transition
         fail Error::NotPlayable, "Transition is not playable: #{self}" if !playable?
 
         if element == :page && event == :load
-            return browser.goto options[:url], cookies: options[:cookies]
+            return browser.goto( options[:url],
+                cookies:         options[:cookies],
+                take_snapshot:   false
+            )
         end
 
         browser.fire_event element, event, options

data/lib/arachni/page/dom.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -39,8 +39,11 @@ class DOM
     #   {Browser::Javascript::TaintTracer#execution_flow_sinks} data.
     attr_accessor :execution_flow_sinks
 
-    # @return   [String]
-    #   String digest of the DOM tree.
+    # @return   [Array<Arachni::Element::Cookie>]
+    attr_accessor :cookies
+
+    # @return   [Integer]
+    #   Digest of the DOM tree.
     attr_accessor :digest
 
     # @return   [String]
@@ -49,7 +52,7 @@ class DOM
 
     # @return   [Page]
     #   Page to which this DOM state is attached.
-    attr_reader   :page
+    attr_accessor :page
 
     # @param    [Hash]  options
     # @option   options [Page]  :page
@@ -58,6 +61,7 @@ class DOM
         @page                 = options[:page]
         self.url              = options[:url]                   || @page.url
         self.digest           = options[:digest]
+        @cookies              = options[:cookies]               || []
         @transitions          = options[:transitions]           || []
         @data_flow_sinks      = options[:data_flow_sinks]       || []
         @execution_flow_sinks = options[:execution_flow_sinks]  || []
@@ -69,20 +73,6 @@ class DOM
         @url = url.freeze
     end
 
-    def digest=( d )
-        return @digest = nil if !d
-
-        normalized_url = Utilities.normalize_url( url )
-
-        if d.include?( url ) || d.include?( normalized_url )
-            d = d.dup
-            d.gsub!( url, '' )
-            d.gsub!( normalized_url, '' )
-        end
-
-        @digest = d.freeze
-    end
-
     # @param    [Transition]    transition
     #   Push the given transition to the {#transitions}.
     def push_transition( transition )
@@ -144,14 +134,26 @@ class DOM
     #
     # @return   [Browser, nil]
     #   Live page in the `browser` if successful, `nil` otherwise.
-    def restore( browser, take_snapshot = true )
-        # First, try to load the page via its DOM#url in case it can restore
-        # itself via its URL fragments and whatnot.
-        browser.goto url, take_snapshot: take_snapshot
-
+    def restore( browser )
         playables = self.playable_transitions
 
-        # If we've got no playable transitions then we're done.
+        # First transition will always be the page load and if that's all there
+        # is then we're done.
+        if playables.size == 1
+            surl = playables.first.options[:url]
+
+            browser.print_debug "Only have a URL load transition: #{surl}"
+            browser.goto surl
+
+            return browser
+
+        # Alternatively, try to load the page via its DOM#url in case it can
+        # restore itself via its URL fragments and whatnot.
+        else
+            browser.goto url
+        end
+
+        # No transitions, nothing more to be done.
         return browser if playables.empty?
 
         browser_dom = browser.state
@@ -165,8 +167,8 @@ class DOM
         # page can restore itself via its URL (using fragment data most probably),
         # the document may still be different from when our snapshot was captured.
         #
-        # However, this check doesn't cost us much so it's worth a shot.
-        if browser_dom === self
+        # However, it doesn't cost us anything so it's worth a shot.
+        if browser_dom == self
             browser.print_debug "Loaded snapshot by URL: #{url}"
             return browser
         end
@@ -203,6 +205,7 @@ class DOM
         {
             url:                  url,
             transitions:          transitions.map(&:to_hash),
+            cookies:              cookies.map(&:to_hash),
             digest:               digest,
             skip_states:          skip_states,
             data_flow_sinks:      data_flow_sinks.map(&:to_hash),
@@ -229,6 +232,7 @@ class DOM
         {
             'url'                  => url,
             'transitions'          => transitions.map(&:to_rpc_data),
+            'cookies'              => cookies.map(&:to_rpc_data),
             'digest'               => digest,
             'skip_states'          => skip_states ? skip_states.collection.to_a : [],
             'data_flow_sinks'      => data_flow_sinks.map(&:to_rpc_data),
@@ -236,6 +240,18 @@ class DOM
         }
     end
 
+    def marshal_dump
+        instance_variables.inject({}) do |h, iv|
+            next h if iv == :@page
+            h[iv] = instance_variable_get( iv )
+            h
+        end
+    end
+
+    def marshal_load( h )
+        h.each { |k, v| instance_variable_set( k, v ) }
+    end
+
     # @param    [Hash]  data
     #   {#to_rpc_data}
     # @return   [DOM]
@@ -247,6 +263,9 @@ class DOM
                         when 'transitions'
                             value.map { |t| Transition.from_rpc_data t }
 
+                        when 'cookies'
+                            value.map { |c| Cookie.from_rpc_data c }
+
                         when 'data_flow_sinks'
                             value.map do |entry|
                                 Browser::Javascript::TaintTracer::Sink::DataFlow.from_rpc_data( entry )
@@ -274,40 +293,13 @@ class DOM
     end
 
     def hash
-        # TODO: Maybe raise error if #digest is not set?
-        digest.persistent_hash
+        digest || super
     end
 
     def ==( other )
         hash == other.hash
     end
 
-    # @note Removes the URL strings of both DOMs from each other's document
-    #        before comparing.
-    #
-    # @param    [DOM]   other
-    # @return   [Bool]
-    #   `true` if the compared DOM trees are effectively the same, `false` otherwise.
-    def ===( other )
-        digest_without_urls( other ) == other.digest_without_urls( self )
-    end
-
-    protected
-
-    def digest_without_urls( dom )
-        normalized_other_url = Utilities.normalize_url( dom.url )
-
-        if !digest.include?( dom.url ) &&
-            !digest.include?( normalized_other_url )
-            return digest
-        end
-
-        d = digest.dup
-        d.gsub!( dom.url, '' )
-        d.gsub!( normalized_other_url, '' )
-        d
-    end
-
 end
 
 end

data/lib/arachni/page/scope.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/page.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -90,8 +90,6 @@ class Page
         data[:response][:request]       ||= {}
         data[:response][:request][:url] ||= data[:response][:url]
 
-        ELEMENTS.each { |e| data[e] ||= [] }
-
         data[:cookie_jar] ||= []
 
         data[:response][:request] = Arachni::HTTP::Request.new( data[:response][:request] )
@@ -101,8 +99,8 @@ class Page
     end
 
     ELEMENTS = [
-        :links, :forms, :cookies, :headers, :link_templates, :jsons, :xmls,
-        :ui_inputs, :ui_forms
+        :links, :forms, :cookies, :nested_cookies, :headers, :link_templates,
+        :jsons, :xmls, :ui_inputs, :ui_forms
     ]
 
     METADATA = [ :nonce_name, :skip_dom ]
@@ -201,6 +199,10 @@ class Page
         @cache[:parser]
     end
 
+    def parser=( p )
+        @cache[:parser] = p
+    end
+
     # @param    [Array<Element::Capabilities::Auditable, Integer>]    list
     #   Audit whitelist based on {Element::Capabilities::Auditable elements} or
     #   {Element::Capabilities::Auditable#coverage_hash}s.
@@ -335,7 +337,7 @@ class Page
         response.request.method
     end
 
-    # @return   [Nokogiri::HTML]
+    # @return   [Arachni::Parser::Document]
     #   Parsed {#body HTML} document.
     def document
         @cache[:document] ||= (parser.nil? ?
@@ -416,7 +418,7 @@ class Page
             next if !body.has_html_tag?( tag )
 
             return false if !document
-            return true  if document.css( tag ).any?
+            return true  if document.nodes_by_name( tag ).any?
         end
 
         false
@@ -432,7 +434,7 @@ class Page
     # @return   [String]
     #   Title of the page.
     def title
-        document.css( 'title' ).first.text rescue nil
+        document.nodes_by_name( 'title' ).first.text rescue nil
     end
 
     # @return   [Hash]

data/lib/arachni/parser/document.rb

@@ -0,0 +1,34 @@
+=begin
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
+
+    This file is part of the Arachni Framework project and is subject to
+    redistribution and commercial restrictions. Please see the Arachni Framework
+    web site for more information on licensing and terms of use.
+=end
+
+require_relative 'with_children'
+require_relative 'nodes/comment'
+require_relative 'nodes/text'
+require_relative 'nodes/element'
+
+module Arachni
+class Parser
+class Document
+
+    include WithChildren
+
+    def name
+        :document
+    end
+
+    def to_html( indentation = 2, level = 0 )
+        html = "<!DOCTYPE html>\n"
+        children.each do |child|
+            html << child.to_html( indentation, level )
+        end
+        html << "\n"
+    end
+
+end
+end
+end

data/lib/arachni/parser/extractors/base.rb

@@ -0,0 +1,48 @@
+=begin
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
+
+    This file is part of the Arachni Framework project and is subject to
+    redistribution and commercial restrictions. Please see the Arachni Framework
+    web site for more information on licensing and terms of use.
+=end
+
+module Arachni
+class Parser
+module Extractors
+
+# @author Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>
+# @abstract
+class Base
+
+    attr_reader :html
+    attr_reader :parser
+    attr_reader :downcased_html
+
+    def initialize( options = {} )
+        @html           = options[:html]
+        @downcased_html = @html.downcase if @html
+        @parser         = options[:parser]
+    end
+
+    # This method must be implemented by all checks and must return an
+    # array of paths as plain strings
+    #
+    # @return   [Array<String>]  paths
+    # @abstract
+    def run
+    end
+
+    def check_for?( string_or_regexp )
+        return true if !@html
+        !!@downcased_html[string_or_regexp]
+    end
+
+    def document
+        parser.document
+    end
+
+end
+
+end
+end
+end

data/lib/arachni/parser/nodes/base.rb

@@ -0,0 +1,22 @@
+=begin
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
+
+    This file is part of the Arachni Framework project and is subject to
+    redistribution and commercial restrictions. Please see the Arachni Framework
+    web site for more information on licensing and terms of use.
+=end
+
+module Arachni
+class Parser
+module Nodes
+
+class Base
+
+    attr_accessor :parent
+    attr_accessor :document
+
+end
+
+end
+end
+end

data/lib/arachni/parser/nodes/comment.rb

@@ -0,0 +1,32 @@
+=begin
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
+
+    This file is part of the Arachni Framework project and is subject to
+    redistribution and commercial restrictions. Please see the Arachni Framework
+    web site for more information on licensing and terms of use.
+=end
+
+require_relative 'base'
+require_relative 'with_value'
+
+module Arachni
+class Parser
+module Nodes
+
+class Comment < Base
+    include WithValue
+
+    def text
+        @value
+    end
+
+    def to_html( indentation = 2, level = 0 )
+        indent = ' ' * (indentation * level)
+        "#{indent}<!-- #{value} -->\n"
+    end
+
+end
+
+end
+end
+end

data/lib/arachni/parser/nodes/element/with_attributes/attributes.rb

@@ -0,0 +1,31 @@
+=begin
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
+
+    This file is part of the Arachni Framework project and is subject to
+    redistribution and commercial restrictions. Please see the Arachni Framework
+    web site for more information on licensing and terms of use.
+=end
+
+module Arachni
+class Parser
+module Nodes
+class Element
+module WithAttributes
+
+class Attributes < Hash
+
+    def []( name )
+        super name.to_s.downcase
+    end
+
+    def []=( name, value )
+        super name.to_s.recode.downcase.freeze, value.recode.freeze
+    end
+
+end
+
+end
+end
+end
+end
+end

data/lib/arachni/parser/nodes/element/with_attributes.rb

@@ -0,0 +1,35 @@
+=begin
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
+
+    This file is part of the Arachni Framework project and is subject to
+    redistribution and commercial restrictions. Please see the Arachni Framework
+    web site for more information on licensing and terms of use.
+=end
+
+require_relative 'with_attributes/attributes'
+
+module Arachni
+class Parser
+module Nodes
+class Element
+
+module WithAttributes
+
+    def attributes
+        @attributes ||= Attributes.new
+    end
+
+    def []( name )
+        attributes[name]
+    end
+
+    def []=( name, value )
+        attributes[name] = value
+    end
+
+end
+
+end
+end
+end
+end