arachni 0.4.7 → 1.0

data/lib/arachni/page.rb

@@ -1,187 +1,436 @@
 =begin
-    Copyright 2010-2014 Tasos Laskos <tasos.laskos@gmail.com>
+    Copyright 2010-2014 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-        http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
+    This file is part of the Arachni Framework project and is subject to
+    redistribution and commercial restrictions. Please see the Arachni Framework
+    web site for more information on licensing and terms of use.
 =end
 
 module Arachni
 
-#
 # It holds page data like elements, cookies, headers, etc...
 #
-# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
-#
+# @author Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>
 class Page
+    include Utilities
 
-    # @return    [String]    URL of the page
-    attr_reader :url
-
-    # @return    [Fixnum]    HTTP response code.
-    attr_reader :code
-
-    # @return    [Hash]    URL query parameters.
-    attr_reader :query_vars
-
-    # @return    [String]    HTTP response body.
-    attr_reader :body
-
-    # @return    [Array<Element::Header>]   HTTP request headers.
-    attr_reader :headers
-
-    # @return    [Hash] HTTP request headers.
-    attr_reader :request_headers
-
-    # @return    [Hash] HTTP response headers.
-    attr_reader :response_headers
-
-    # @return    [Array<String>]    Paths contained in this page.
-    attr_reader :paths
-
-    # @see Parser#links
-    # @return    [Array<Element::Link>]
-    attr_accessor :links
-
-    # @see Parser#forms
-    # @return    [Array<Element::Form>]
-    attr_accessor :forms
-
-    # @see Parser#cookies
-    # @return    [Array<Element::Cookie>]
-    attr_accessor :cookies
+    class Error < Arachni::Error
+    end
 
-    # @return    [Array<Element::Cookie>]
-    #   Cookies extracted from the supplied cookie-jar.
-    attr_accessor :cookiejar
+    require_relative 'page/dom'
+    require_relative 'page/scope'
 
-    # @param    [String]    url URL to fetch.
+    # @param    [String]    url
+    #   URL to fetch.
     # @param    [Hash]  opts
-    # @option  opts    [Integer]   :precision  (1)
+    # @option  opts    [Integer]   :precision  (2)
     #   How many times to request the page and examine changes between requests.
     #   Used tp identify nonce tokens etc.
-    # @option  opts    [Hash]  :http   HTTP {HTTP#get request} options.
+    # @option  opts    [Hash]  :http
+    #   HTTP {HTTP::Client#get request} options.
     # @param    [Block] block
     #   Block to which to pass the page object. If given, the request will be
     #   performed asynchronously. If no block is given, the page will be fetched
     #   synchronously and be returned by this method.
+    #
     # @return   [Page]
     def self.from_url( url, opts = {}, &block )
         responses = []
 
-        opts[:precision] ||= 1
-        opts[:precision].times {
-            HTTP.get( url, opts[:http] || {} ) do |res|
+        opts[:precision] ||= 2
+        opts[:precision].times do
+            HTTP::Client.get( url, opts[:http] || {} ) do |res|
                 responses << res
                 next if responses.size != opts[:precision]
                 block.call( from_response( responses ) ) if block_given?
             end
-        }
+        end
 
         if !block_given?
-            HTTP.run
+            HTTP::Client.run
             from_response( responses )
         end
     end
 
-    # @param    [Typhoeus::Response]    res HTTP response to parse.
+    # @param    [HTTP::Response]    response
+    #   HTTP response to parse.
+    #
     # @return   [Page]
-    def self.from_response( res, opts = Options )
-        Parser.new( res, opts ).page
+    def self.from_response( response )
+        Parser.new( response ).page
+    end
+
+    # @option options  [String]    :url
+    #   URL of the page.
+    # @option options  [String]    :body
+    #   Body of the page.
+    # @option options  [Array<Link>]    :links
+    #   {Link} elements.
+    # @option options  [Array<Form>]    :forms
+    #   {Form} elements.
+    # @option options  [Array<Cookie>]    :cookies
+    #   {Cookie} elements.
+    # @option options  [Array<Header>]    :headers
+    #   {Header} elements.
+    # @option options  [Array<Cookie>]    :cookie_jar
+    #   {Cookie} elements with which to update the HTTP cookiejar before
+    #   auditing.
+    # @option options  [Array<String>]    :paths
+    #   Paths contained in the page.
+    # @option options  [Array<String>]    :request
+    #   {Request#initialize} options.
+    def self.from_data( data )
+        data = data.dup
+
+        data[:response]        ||= {}
+        data[:response][:code] ||= 200
+        data[:response][:url]  ||= data.delete( :url )
+        data[:response][:body] ||= data.delete( :body ) || ''
+
+        data[:response][:request]       ||= {}
+        data[:response][:request][:url] ||= data[:response][:url]
+
+        data[:links]   ||= []
+        data[:forms]   ||= []
+        data[:cookies] ||= []
+        data[:headers] ||= []
+
+        data[:cookie_jar] ||= []
+
+        data[:response][:request] = Arachni::HTTP::Request.new( data[:response][:request] )
+        data[:response]           = Arachni::HTTP::Response.new( data[:response] )
+
+        new data
+    end
+
+    ELEMENTS = [
+        :links, :forms, :cookies, :headers, :link_templates
+    ]
+
+    # @return   [DOM]
+    #   DOM snapshot.
+    attr_accessor :dom
+
+    # @return    [HTTP::Response]
+    #   HTTP response.
+    attr_reader :response
+
+    # @return    [Hash]
+    #
+    # @private
+    attr_reader :cache
+
+    # @return    [Hash]
+    #   Holds page data that will need to persist between {#clear_cache} calls
+    #   and other utility data.
+    attr_reader :metadata
+
+    # @return   [Set<Integer>]
+    #   Audit whitelist based on {Element::Capabilities::Auditable#coverage_hash}.
+    #
+    # @see  #update_element_audit_whitelist
+    # @see  #audit_element?
+    # @see  Check::Auditor#skip?
+    attr_reader :element_audit_whitelist
+
+    # Needs either a `:parser` or a `:response` or user provided data.
+    #
+    # @param    [Hash]  options
+    #   Hash from which to set instance attributes.
+    # @option options  [Array<HTTP::Response>, HTTP::Response]    :response
+    #   HTTP response of the page -- or array of responses for the page for
+    #   content refinement.
+    # @option options  [Parser]    :parser
+    #   An instantiated {Parser}.
+    def initialize( options )
+        fail ArgumentError, 'Options cannot be empty.' if options.empty?
+        options = options.dup
+
+        @cache = {}
+
+        @do_not_audit_elements = options.delete(:do_not_audit_elements)
+
+        @cache[:parser] = options.delete(:parser)
+        @response = @cache[:parser].response if @cache[:parser]
+
+        # We need to know whether or not the page has been dynamically updated
+        # with elements, in order to optimize #dup and #hash operations.
+        @has_custom_elements = Set.new
+
+        @metadata ||= {}
+
+        options.each do |k, v|
+            send( "#{k}=", try_dup( v ) )
+        end
+
+        @dom = DOM.new( (options[:dom] || {}).merge( page: self ) )
+
+        fail ArgumentError, 'No URL given!' if !url
+
+        Platform::Manager.fingerprint( self )
+
+        @element_audit_whitelist ||= []
+        @element_audit_whitelist   = Set.new( @element_audit_whitelist )
+    end
+
+    # @return   [Scope]
+    def scope
+        @scope = Scope.new( self )
+    end
+
+    # @return   [Object]
+    #   Object which performed the {#request} which lead to this page.
+    def performer
+        request.performer
+    end
+
+    # @return   [Arachni::URI]
+    def parsed_url
+        Arachni::URI( url )
     end
-    class << self; alias :from_http_response :from_response end
 
-    # @param    [Hash]  opts    Hash from which to set instance attributes.
-    def initialize( opts = {} )
-        opts.each { |k, v| instance_variable_set( "@#{k}".to_sym, try_dup( v ) ) }
+    # @return   [Parser]
+    def parser
+        return if !@response
+        return @cache[:parser] if @cache[:parser]
 
-        @forms ||= []
-        @links ||= []
-        @cookies ||= []
-        @headers ||= []
+        @cache[:parser] = Parser.new( @response )
 
-        @cookiejar ||= {}
-        @paths ||= []
+        # The page may have a browser-assigned body, set it as the one to parse.
+        @cache[:parser].body = body
+        @cache[:parser]
+    end
+
+    # @param    [Array<Element::Capabilities::Auditable, Integer>]    list
+    #   Audit whitelist based on {Element::Capabilities::Auditable elements} or
+    #   {Element::Capabilities::Auditable#coverage_hash}s.
+    #
+    # @return   [Set]   {#element_audit_whitelist}
+    #
+    # @see  #element_audit_whitelist
+    # @see  Check::Auditor#skip?
+    def update_element_audit_whitelist( list )
+        [list].flatten.each do |e|
+            @element_audit_whitelist <<
+                (e.is_a?( Integer ) ? e : e.coverage_hash )
+        end
+    end
+
+    # @param    [Element::Capabilities::Auditable, Integer]    element
+    #   Element or {Element::Capabilities::Auditable#coverage_hash}.
+    #
+    # @return   [Bool]
+    #   `true` if the element should be audited, `false` otherwise.
+    #
+    # @see  #element_audit_whitelist
+    # @see  Check::Auditor#skip?
+    def audit_element?( element )
+        return if @do_not_audit_elements
+        return true if @element_audit_whitelist.empty?
+        @element_audit_whitelist.include?(
+            element.is_a?( Integer ) ? element : element.coverage_hash
+        )
+    end
+
+    # It forces {#audit_element?} to always returns false.
+    def do_not_audit_elements
+        @do_not_audit_elements = true
+    end
+
+    # @return    [HTTP::Request]
+    #   HTTP request.
+    def request
+        response.request
+    end
+
+    # @return    [String]
+    #   URL of the page.
+    def url
+        @url ||= @response.url
+    end
+
+    # @return    [String]
+    #   URL of the page.
+    def code
+        return 0 if !@code && !response
+        @code ||= response.code
+    end
+
+    # @return    [Hash]
+    #   {#url URL} query parameters.
+    def query_vars
+        @cache[:query_vars] ||= uri_parse_query( url )
+    end
+
+    # @return    [String]
+    #   HTTP response body.
+    def body
+        return '' if !@body && !@response
+        @body ||= response.body
+    end
 
-        @response_headers ||= {}
-        @request_headers  ||= {}
-        @query_vars       ||= {}
+    # @param    [String]    string
+    #   Page body.
+    def body=( string )
+        @has_javascript = nil
+        clear_cache
 
-        @url    = Utilities.normalize_url( @url )
-        @body ||= ''
+        @body = string.to_s.dup.freeze
     end
 
-    # @return   [Platform] Applicable platforms for the page.
+    ELEMENTS.each do |type|
+        parser_method = type
+        parser_method = :cookies_to_be_audited if type == :cookies
+
+        define_method type do
+            @cache[type] ||=
+                assign_page_to_elements( parser ? parser.send(parser_method) : [] )
+        end
+
+        define_method "#{type}=" do |elements|
+            @has_custom_elements << type
+            @cache[type] = assign_page_to_elements( elements )
+        end
+    end
+
+    # @return    [Array<Element::Cookie>]
+    #   Cookies extracted from the supplied cookie-jar.
+    def cookie_jar
+        @cookie_jar ||= (parser ? parser.cookie_jar : [])
+    end
+
+    # @return    [Array<String>]
+    #   Paths contained in this page.
+    #
+    # @see Parser#paths
+    def paths
+        @cache[:paths] ||= parser ? parser.paths : []
+    end
+
+    # @return   [Platform]
+    #   Applicable platforms for the page.
     def platforms
-        Platform::Manager[@url]
+        Platform::Manager[url]
     end
 
-    # @return   [Array] All page elements.
+    # @return   [Array]
+    #   All page elements.
     def elements
-        @links | @forms | @cookies | @headers
+        ELEMENTS.map { |type| send( type ) }.flatten
     end
 
-    # @return    [String]    the request method that returned the page
+    # @return    [String]
+    #   The request method that returned the page
     def method( *args )
         return super( *args ) if args.any?
-        @method
+        response.request.method
     end
 
-    # @see #body
-    def html
-        @body
+    # @return   [Nokogiri::HTML]
+    #   Parsed {#body HTML} document.
+    def document
+        @cache[:document] ||= (parser.nil? ? Nokogiri::HTML( body ) : parser.document)
     end
 
-    # @return   [Nokogiri::HTML]    Parsed {#body HTML} document.
-    def document
-        @document ||= Nokogiri::HTML( @body )
+    # @note Will preserve caches for elements which have been externally modified.
+    #
+    # @return   [Page]
+    #   `self` with caches cleared.
+    def clear_cache
+        ELEMENTS.each do |type|
+            next if @has_custom_elements.include? type
+            # Remove the association to this page before clearing the elements
+            # from cache to make it easier on the GC.
+            (@cache[type] || []).each { |e| e.page = nil }
+        end
+
+        @cache.delete_if { |k, _| !@has_custom_elements.include? k }
+        self
     end
 
-    def marshal_dump
-        instance_variables.inject( {} ) do |h, iv|
-            next h if iv == :@document
-            h[iv] = instance_variable_get( iv )
-            h
+    def prepare_for_report
+        # We want a hard clear, that's why we don't call #clear_cache.
+        @cache.clear
+
+        # If we're dealing with binary data remove it before storing.
+        if !text?
+            response.body = nil
+            self.body     = nil
         end
+
+        @cookie_jar.clear if @cookie_jar
+
+        @dom.digest      = nil
+        @dom.skip_states = nil
+
+        self
     end
 
-    def marshal_load( h )
-        h.each { |k, v| instance_variable_set( k, v ) }
+    # @return   [Boolean]
+    #   `true` if the page contains client-side code, `false` otherwise.
+    def has_script?
+        return @has_javascript if !@has_javascript.nil?
+
+        if !response.headers.content_type.to_s.start_with?( 'text/html' ) ||
+            !text? || !document
+            return @has_javascript = false
+        end
+
+        # First check, quick and simple.
+        return @has_javascript = true if document.css( 'script' ).any?
+
+        # Check for event attributes, if there are any then there's JS to be
+        # executed.
+        Browser::Javascript.events.flatten.each do |event|
+            return @has_javascript = true if document.xpath( "//*[@#{event}]" ).any?
+        end
+
+        # If there's 'javascript:' in 'href' and 'action' attributes then
+        # there's JS to be executed.
+        [:action, :href].each do |candidate|
+            document.xpath( "//*[@#{candidate}]" ).each do |attribute|
+                if attribute.attributes[candidate.to_s].to_s.start_with?( 'javascript:' )
+                    return @has_javascript = true
+                end
+            end
+        end
+
+        @has_javascript = false
     end
 
     # @return   [Boolean]
     #   `true` if the body of the page is text-base, `false` otherwise.
     def text?
-        !!@text
+        return false if !response
+        response.text?
     end
 
-    # @return   [String]    Title of the page.
+    # @return   [String]
+    #   Title of the page.
     def title
         document.css( 'title' ).first.text rescue nil
     end
 
-    # @return   [Hash]  Converts the page data to a hash.
+    # @return   [Hash]
+    #   Converts the page data to a hash.
     def to_h
-        instance_variables.reduce({}) do |h, iv|
-            next h if iv == :@document
+        skip = [:@document, :@do_not_audit_elements, :@has_custom_elements, :@scope]
+
+        instance_variables.inject({}) do |h, iv|
+            next h if skip.include? iv
+
             h[iv.to_s.gsub( '@', '').to_sym] = try_dup( instance_variable_get( iv ) )
             h
-        end
+        end.merge(@cache).tap { |h| h.delete :parser }
     end
     alias :to_hash :to_h
 
+    def persistent_hash
+        digest.persistent_hash
+    end
+
     def hash
-        ((links.map { |e| e.hash } + forms.map { |e| e.hash } +
-            cookies.map { |e| e.hash } + headers.map { |e| e.hash }).sort.join +
-            body.to_s).hash
+        digest.hash
     end
 
     def ==( other )
@@ -193,11 +442,158 @@ class Page
     end
 
     def dup
-        self.deep_clone
+        self.class.new to_initialization_options
+    end
+
+    def to_initialization_options
+        h = {}
+        [:body, :cookie_jar, :element_audit_whitelist, :metadata].each do |m|
+            h[m] = try_dup( instance_variable_get( "@#{m}".to_sym ) )
+            h.delete( m ) if !h[m]
+        end
+
+        ELEMENTS.each do |type|
+            next if !@has_custom_elements.include?( type )
+            h[type] = @cache[type]
+
+            if !h[type] || h[type].empty?
+                h.delete( type )
+                next
+            end
+
+            h[type] = h[type].map { |e| c = e.dup; c.page = nil; c }
+        end
+
+        h[:response] = response
+        h[:do_not_audit_elements] = @do_not_audit_elements
+
+        h[:dom] = dom.to_h.keys.inject({}) do |dh, k|
+            dh[k] = try_dup( dom.send( k ) )
+            dh
+        end
+
+        h
+    end
+
+    # @return   [Hash]
+    #   Data representing this instance that are suitable the RPC transmission.
+    def to_rpc_data
+        data        = to_initialization_options.stringify_keys(false)
+        data['dom'] = dom.to_rpc_data
+        data['element_audit_whitelist'] = element_audit_whitelist.to_a
+        data['response'] = data['response'].to_rpc_data
+
+        %w(links forms cookies).each do |e|
+            next if !data[e]
+            data[e] = send(e).map(&:to_rpc_data)
+        end
+
+        data.delete 'cookie_jar'
+
+        data
+    end
+
+    # @param    [Hash]  data
+    #   {#to_rpc_data}
+    #
+    # @return   [Page]
+    def self.from_rpc_data( data )
+        dom = data.delete('dom')
+        normalized_data = {}
+        data.each do |name, value|
+
+            value = case name
+                        when 'response'
+                            HTTP::Response.from_rpc_data( value )
+
+                        when 'metadata'
+                            sanitized = {}
+                            %w(link form cookie header).each do |e|
+                                next if !value[e] || !value[e]['nonces']
+
+                                sanitized[e.to_sym] = {}
+                                sanitized[e.to_sym][:nonces] = value[e]['nonces']
+                            end
+                            sanitized
+
+                        when 'links', 'forms', 'cookies'
+                            value.map do |e|
+                                Element.const_get(name[0...-1].capitalize.to_sym).from_rpc_data( e )
+                            end.to_a
+
+                        else
+                            value
+                    end
+
+            normalized_data[name.to_sym] = value
+        end
+
+        instance = new( normalized_data )
+        instance.instance_variable_set(
+            '@dom', DOM.from_rpc_data( dom.merge( page: instance ) )
+        )
+        instance
+    end
+
+    def _dump( _ )
+        Marshal.dump( to_initialization_options )
+    end
+
+    def self._load( data )
+        new( Marshal.load( data ) )
     end
 
     private
 
+    def digest
+        element_hashes = []
+        ELEMENTS.each do |type|
+            next if !@has_custom_elements.include?( type ) || !(list = @cache[type])
+            element_hashes |= list.map(&:hash)
+        end
+
+        "#{dom.playable_transitions.hash}:#{body.hash}#{element_hashes.sort}"
+    end
+
+    [:url, :response, :cookie_jar, :element_audit_whitelist, :metadata].each do |attribute|
+        attr_writer attribute
+    end
+
+    def paths=( paths )
+        @cache[:paths] = paths
+    end
+
+    def assign_page_to_elements( list )
+        list.map do |e|
+            e.page = self
+            store_nonce_to_metadata e
+            restore_nonce_from_metadata e
+            e
+        end.freeze
+    end
+
+    def store_nonce_to_metadata( element )
+        ensure_metadata_nonces( element )
+
+        return if !element.respond_to?(:has_nonce?) || !element.has_nonce?
+
+        @metadata[element.type][:nonces][element.coverage_hash] =
+            element.nonce_name
+    end
+
+    def restore_nonce_from_metadata( element )
+        ensure_metadata_nonces( element )
+
+        return if !element.respond_to?(:nonce_name=) || element.has_nonce?
+
+        element.nonce_name = @metadata[element.type][:nonces][element.coverage_hash]
+    end
+
+    def ensure_metadata_nonces( element )
+        @metadata[element.type] ||= {}
+        @metadata[element.type][:nonces] ||= {}
+    end
+
     def try_dup( v )
         v.dup rescue v
     end