arachni 1.4 → 1.5

data/lib/arachni/page/dom.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -39,8 +39,11 @@ class DOM
     #   {Browser::Javascript::TaintTracer#execution_flow_sinks} data.
     attr_accessor :execution_flow_sinks
 
-    # @return   [String]
-    #   String digest of the DOM tree.
+    # @return   [Array<Arachni::Element::Cookie>]
+    attr_accessor :cookies
+
+    # @return   [Integer]
+    #   Digest of the DOM tree.
     attr_accessor :digest
 
     # @return   [String]
@@ -49,7 +52,7 @@ class DOM
 
     # @return   [Page]
     #   Page to which this DOM state is attached.
-    attr_reader   :page
+    attr_accessor :page
 
     # @param    [Hash]  options
     # @option   options [Page]  :page
@@ -58,6 +61,7 @@ class DOM
         @page                 = options[:page]
         self.url              = options[:url]                   || @page.url
         self.digest           = options[:digest]
+        @cookies              = options[:cookies]               || []
         @transitions          = options[:transitions]           || []
         @data_flow_sinks      = options[:data_flow_sinks]       || []
         @execution_flow_sinks = options[:execution_flow_sinks]  || []
@@ -69,20 +73,6 @@ class DOM
         @url = url.freeze
     end
 
-    def digest=( d )
-        return @digest = nil if !d
-
-        normalized_url = Utilities.normalize_url( url )
-
-        if d.include?( url ) || d.include?( normalized_url )
-            d = d.dup
-            d.gsub!( url, '' )
-            d.gsub!( normalized_url, '' )
-        end
-
-        @digest = d.freeze
-    end
-
     # @param    [Transition]    transition
     #   Push the given transition to the {#transitions}.
     def push_transition( transition )
@@ -144,14 +134,26 @@ class DOM
     #
     # @return   [Browser, nil]
     #   Live page in the `browser` if successful, `nil` otherwise.
-    def restore( browser, take_snapshot = true )
-        # First, try to load the page via its DOM#url in case it can restore
-        # itself via its URL fragments and whatnot.
-        browser.goto url, take_snapshot: take_snapshot
-
+    def restore( browser )
         playables = self.playable_transitions
 
-        # If we've got no playable transitions then we're done.
+        # First transition will always be the page load and if that's all there
+        # is then we're done.
+        if playables.size == 1
+            surl = playables.first.options[:url]
+
+            browser.print_debug "Only have a URL load transition: #{surl}"
+            browser.goto surl
+
+            return browser
+
+        # Alternatively, try to load the page via its DOM#url in case it can
+        # restore itself via its URL fragments and whatnot.
+        else
+            browser.goto url
+        end
+
+        # No transitions, nothing more to be done.
         return browser if playables.empty?
 
         browser_dom = browser.state
@@ -165,8 +167,8 @@ class DOM
         # page can restore itself via its URL (using fragment data most probably),
         # the document may still be different from when our snapshot was captured.
         #
-        # However, this check doesn't cost us much so it's worth a shot.
-        if browser_dom === self
+        # However, it doesn't cost us anything so it's worth a shot.
+        if browser_dom == self
             browser.print_debug "Loaded snapshot by URL: #{url}"
             return browser
         end
@@ -203,6 +205,7 @@ class DOM
         {
             url:                  url,
             transitions:          transitions.map(&:to_hash),
+            cookies:              cookies.map(&:to_hash),
             digest:               digest,
             skip_states:          skip_states,
             data_flow_sinks:      data_flow_sinks.map(&:to_hash),
@@ -229,6 +232,7 @@ class DOM
         {
             'url'                  => url,
             'transitions'          => transitions.map(&:to_rpc_data),
+            'cookies'              => cookies.map(&:to_rpc_data),
             'digest'               => digest,
             'skip_states'          => skip_states ? skip_states.collection.to_a : [],
             'data_flow_sinks'      => data_flow_sinks.map(&:to_rpc_data),
@@ -236,6 +240,18 @@ class DOM
         }
     end
 
+    def marshal_dump
+        instance_variables.inject({}) do |h, iv|
+            next h if iv == :@page
+            h[iv] = instance_variable_get( iv )
+            h
+        end
+    end
+
+    def marshal_load( h )
+        h.each { |k, v| instance_variable_set( k, v ) }
+    end
+
     # @param    [Hash]  data
     #   {#to_rpc_data}
     # @return   [DOM]
@@ -247,6 +263,9 @@ class DOM
                         when 'transitions'
                             value.map { |t| Transition.from_rpc_data t }
 
+                        when 'cookies'
+                            value.map { |c| Cookie.from_rpc_data c }
+
                         when 'data_flow_sinks'
                             value.map do |entry|
                                 Browser::Javascript::TaintTracer::Sink::DataFlow.from_rpc_data( entry )
@@ -274,40 +293,13 @@ class DOM
     end
 
     def hash
-        # TODO: Maybe raise error if #digest is not set?
-        digest.persistent_hash
+        digest || super
     end
 
     def ==( other )
         hash == other.hash
     end
 
-    # @note Removes the URL strings of both DOMs from each other's document
-    #        before comparing.
-    #
-    # @param    [DOM]   other
-    # @return   [Bool]
-    #   `true` if the compared DOM trees are effectively the same, `false` otherwise.
-    def ===( other )
-        digest_without_urls( other ) == other.digest_without_urls( self )
-    end
-
-    protected
-
-    def digest_without_urls( dom )
-        normalized_other_url = Utilities.normalize_url( dom.url )
-
-        if !digest.include?( dom.url ) &&
-            !digest.include?( normalized_other_url )
-            return digest
-        end
-
-        d = digest.dup
-        d.gsub!( dom.url, '' )
-        d.gsub!( normalized_other_url, '' )
-        d
-    end
-
 end
 
 end

data/lib/arachni/page/dom/transition.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -194,7 +194,10 @@ class Transition
         fail Error::NotPlayable, "Transition is not playable: #{self}" if !playable?
 
         if element == :page && event == :load
-            return browser.goto options[:url], cookies: options[:cookies]
+            return browser.goto( options[:url],
+                cookies:         options[:cookies],
+                take_snapshot:   false
+            )
         end
 
         browser.fire_event element, event, options

data/lib/arachni/page/scope.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/parser.rb

@@ -1,15 +1,30 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
     web site for more information on licensing and terms of use.
 =end
 
+require 'ox'
+
+Ox.default_options = {
+    indent:          4,
+    mode:            :generic,
+    effort:          :tolerant,
+    smart:           true,
+    invalid_replace: nil
+}
+
 module Arachni
 
 lib = Options.paths.lib
 
+require lib + 'parser/extractors/base'
+require lib + 'parser/document'
+require lib + 'parser/sax'
+require lib + 'parser/with_children'
+
 # Load all available element types.
 Dir.glob( lib + 'element/*.rb' ).each { |f| require f }
 
@@ -24,38 +39,6 @@ class Parser
     include UI::Output
     include Utilities
 
-    # @author Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>
-    module Extractors
-
-        # @author Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>
-        # @abstract
-        class Base
-
-            attr_reader :html
-            attr_reader :document
-            attr_reader :downcased_html
-
-            def initialize( options = {} )
-                @html           = options[:html]
-                @downcased_html = @html.downcase
-                @document       = options[:document]
-            end
-
-            # This method must be implemented by all checks and must return an
-            # array of paths as plain strings
-            #
-            # @return   [Array<String>]  paths
-            # @abstract
-            def run
-            end
-
-            def includes?( string_or_regexp )
-                !!@downcased_html[string_or_regexp]
-            end
-
-        end
-    end
-
     CACHE_SIZES = {
         parse:          50,
         parse_xml:      50,
@@ -67,27 +50,97 @@ class Parser
         CACHE[name] = Support::Cache::LeastRecentlyPushed.new( size )
     end
 
+    WHITELIST = %w(
+        title base a form frame iframe meta input select option script link area
+        textarea input select button comment !--
+    )
+
+    IGNORE_REQUEST_HEADERS = [
+        HTTP::Client::SEED_HEADER_NAME,
+        'Content-Length'
+    ]
+
     class <<self
 
-        def parse( html )
-            CACHE[__method__].fetch html do
-                Nokogiri::HTML( html ).freeze
+        def parse( html, options = {} )
+            CACHE[__method__].fetch [html, options] do
+                handler, sax_options = prepare_ox_options( options )
+
+                begin
+                    Ox.sax_html( handler, StringIO.new( html ), sax_options )
+                rescue SAX::Stop
+                end
+
+                handler.document
             end
         end
 
+        def push_parse( options = {} )
+            buffer, buffer_in = IO.pipe
+
+            document, sax_options = prepare_ox_options( options )
+
+            push_parse_pool.post do
+                begin
+                    Ox.sax_html( document, buffer, sax_options )
+                rescue SAX::Stop
+                end
+            end
+
+            [buffer_in, document]
+        end
+
         def parse_fragment( html )
             CACHE[__method__].fetch html do
-                Nokogiri::HTML.fragment( html ).children.first.freeze
+                parse( html ).children.first.tap do |o|
+                    o.parent   = nil
+                    o.document = nil
+                end
             end
         end
 
         def parse_xml( xml )
             CACHE[__method__].fetch xml do
-                Nokogiri::XML( xml ).freeze
+                Nokogiri::XML( xml )
+            end
+        end
+
+        def markup?( string )
+            begin
+                Ox.parse( string ).is_a?( Ox::Element )
+            rescue => e
+                false
             end
         end
 
+        private
+
+        def push_parse_pool
+            @push_parse_pool ||= Concurrent::CachedThreadPool.new
+        end
+
+        def prepare_ox_options( options )
+            handler = options[:handler] || SAX.new( options )
+
+            sax_options = {}
+            if options[:whitelist] && options[:whitelist].any?
+                overlay = Ox.sax_html_overlay.dup
+                overlay.each do |k, v|
+                    overlay[k] = :off
+                end
+
+                options[:whitelist].each do |e|
+                    overlay[e] = :active
+                end
+
+                sax_options[:overlay] = overlay
+            end
+
+            [handler, sax_options]
+        end
+
     end
+    push_parse_pool
 
     alias :skip? :skip_path?
 
@@ -95,25 +148,35 @@ class Parser
     attr_reader :url
 
     # @return   [HTTP::Response]
-    attr_reader :response
+    attr_accessor :response
 
-    # @param  [HTTP::Response, Array<HTTP::Response>] response
+    # @param  [Document, HTTP::Response, Array<HTTP::Response>] resource
     #   Response(s) to analyze and parse. By providing multiple responses the
     #   parser will be able to perform some preliminary differential analysis
     #   and identify nonce tokens in inputs.
-    #
-    # @param  [Options] options
-    def initialize( response, options = Options )
-        @options = options
-
-        if response.is_a? Array
-            @secondary_responses = response[1..-1]
-            @secondary_responses.compact! if @secondary_responses
-            response = response.shift
-        end
+    def initialize( resource )
+        case resource
+
+            when Document
+                @resource = :document
+                @document = resource
+
+            when HTTP::Response
+                @resource = :response
+
+                @response = resource
+                self.url = @response.url
 
-        @response = response
-        self.url  = response.url
+            when Array
+                @secondary_responses = resource[1..-1]
+                @secondary_responses.compact! if @secondary_responses
+                response = resource.shift
+
+                @resource = :response
+
+                @response = response
+                self.url = response.url
+        end
     end
 
     def url=( str )
@@ -147,7 +210,15 @@ class Parser
     # @return   [Boolean]
     #   `true` if the given HTTP response data are text based, `false` otherwise.
     def text?
-        !@body.to_s.empty? || @response.text?
+        from_response? ? @response.text? : true
+    end
+
+    def from_response?
+        @resource == :response
+    end
+
+    def from_document?
+        @resource == :document
     end
 
     # @return    [String]
@@ -158,20 +229,24 @@ class Parser
     end
 
     def body
-        @body || @response.body
+        @body || (@response.body if from_response?)
     end
 
-    # @return   [Nokogiri::HTML, nil]
+    # @return   [Arachni::Parser::Document, nil]
     #   Returns a parsed HTML document from the body of the HTTP response or
     #   `nil` if the response data wasn't {#text? text-based} or the response
     #   couldn't be parsed.
     def document
         return if !text?
-        return @document.freeze if @document
 
-        @document = self.class.parse( body )
-    rescue
-        nil
+        if from_document?
+            @document
+        else
+            @document = self.class.parse(
+                body,
+                whitelist: WHITELIST
+            )
+        end
     end
 
     # @note It will include common request headers as well headers from the HTTP
@@ -185,34 +260,37 @@ class Parser
                 '/xml;q=0.9,*/*;q=0.8',
             'Accept-Charset'  => 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
             'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
-            'From'            => @options.authorized_by  || '',
-            'User-Agent'      => @options.http.user_agent || '',
+            'From'            => Options.authorized_by  || '',
+            'User-Agent'      => Options.http.user_agent || '',
             'Referer'         => @url,
             'Pragma'          => 'no-cache'
-        }.merge( response.request.headers ).
-            map { |k, v| Header.new( url: @url, inputs: { k => v } ) }.freeze
+        }.merge(
+            response.request.headers.dup.tap do |h|
+                IGNORE_REQUEST_HEADERS.each { |k| h.delete k }
+            end
+        ).map { |k, v| Header.new( url: @url, inputs: { k => v } ) }.freeze
     end
 
     # @return [Array<Element::Form>]
     #   Forms from {#document}.
     def forms
         return @forms.freeze if @forms
-        return [] if !text? || !Form.in_html?( body )
+        return [] if !text? || (body && !Form.in_html?( body ))
 
-        f = Form.from_document( @url, document )
+        f = Form.from_parser( self )
         return f if !@secondary_responses
 
         @secondary_responses.each do |response|
             next if response.body.to_s.empty?
 
-            Form.from_document( @url, response.body ).each do |form2|
+            Form.from_parser( Parser.new( response ) ).each do |form2|
                 f.each do |form|
                     next if "#{form.coverage_id}:#{form.name_or_id}" !=
                         "#{form2.coverage_id}:#{form2.name_or_id}"
 
                     form.inputs.each do |k, v|
-                        next if !(v != form2.inputs[k] &&
-                            form.field_type_for( k ) == :hidden)
+                        next if v == form2.inputs[k] ||
+                            form.field_type_for( k ) != :hidden
 
                         form.nonce_name = k
                     end
@@ -226,7 +304,7 @@ class Parser
     # @return [Element::Link]
     #   Link to the page.
     def link
-        return if link_vars.empty? && !@response.redirection?
+        return if link_vars.empty? && (@response && !@response.redirection?)
         Link.new( url: @url, inputs: link_vars )
     end
 
@@ -248,9 +326,9 @@ class Parser
     #   Links in {#document}.
     def links
         return @links.freeze if @links
-        return @links = [link].compact if !text? || !Link.in_html?( body )
+        return @links = [link].compact if !text? || (body && !Link.in_html?( body ))
 
-        @links = [link].compact | Link.from_document( @url, document )
+        @links = [link].compact | Link.from_parser( self )
     end
 
     # @return [Array<Element::LinkTemplate>]
@@ -260,7 +338,7 @@ class Parser
         return @link_templates = [link_template].compact if !text?
 
         @link_templates =
-            [link_template].compact | LinkTemplate.from_document( @url, document )
+            [link_template].compact | LinkTemplate.from_parser( self )
     end
 
     # @return [Array<Element::JSON>]
@@ -299,7 +377,7 @@ class Parser
         @cookies = Cookie.from_headers( @url, @response.headers )
         return @cookies if !text? || !Cookie.in_html?( body )
 
-        @cookies |= Cookie.from_document( @url, document )
+        @cookies |= Cookie.from_parser( self )
     end
 
     # @return   [Array<Element::Cookie>]
@@ -356,7 +434,7 @@ class Parser
     # @return   [String]
     #   Base `href`, if there is one.
     def base
-        @base ||= document.search( '//base[@href]' ).first['href'] rescue nil
+        @base ||= document.nodes_by_name( :base ).map { |b| b['href'] }.first || @url
     end
 
     private
@@ -371,14 +449,16 @@ class Parser
             self.class.extractors.available.each do |name|
                 exception_jail false do
                     unsanitized_paths.merge self.class.extractors[name].new(
-                        document: document,
-                        html:     body
-                    ).run
+                        parser: self,
+                        html:   body
+                    ).run.flatten
                 end
             end
 
             sanitized_paths = Set.new
             unsanitized_paths.map do |path|
+                next if !path || path =~ /^mailto:/i
+
                 abs = to_absolute( path )
                 next if !abs || skip?( abs )