arachni 1.4 → 1.6.0

data/lib/arachni/element/cookie.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -27,6 +27,8 @@ class Cookie < Base
     # Generic element capabilities.
     include Arachni::Element::Capabilities::Submittable
     include Arachni::Element::Capabilities::Auditable
+    include Arachni::Element::Capabilities::Auditable::Buffered
+    include Arachni::Element::Capabilities::Auditable::LineBuffered
     include Arachni::Element::Capabilities::Analyzable
     include Arachni::Element::Capabilities::WithSource
 
@@ -172,10 +174,10 @@ class Cookie < Base
     # @return   [String]
     #   To be used in a `Cookie` HTTP request header.
     def to_s
-        # Only do encoding if we're dealing with a mutation, otherwise pass
+        # Only do encoding if we're dealing with updated inputs, otherwise pass
         # along the raw data as set in order to deal with server-side decoding
         # quirks.
-        if mutation? || !(raw_name || raw_value )
+        if updated? || !(raw_name || raw_value )
             "#{encode( name )}=#{encode( value )}"
         else
             "#{raw_name}=#{raw_value}"
@@ -185,14 +187,27 @@ class Cookie < Base
     # @return   [String]
     #   Converts self to a `Set-Cookie` string.
     def to_set_cookie
-        set_cookie = "#{self.to_s}; "
-        set_cookie << @data.map do |k, v|
+        set_cookie = "#{self.to_s}"
+
+        @data.each do |k, v|
             next if !v || !self.class.keep_for_set_cookie.include?( k )
-            "#{k.capitalize}=#{v}"
-        end.compact.join( '; ' )
+
+            set_cookie << "; #{k.capitalize}=#{v}"
+        end
 
         set_cookie << '; Secure'   if secure?
         set_cookie << '; HttpOnly' if http_only?
+
+        # If we want to set a cookie for only the domain that responded to the
+        # request, Set-Cookie should not specify a domain.
+        #
+        # If we want the cookie to apply to all subdomains, we need to either
+        # specify a dot-prefixed domain or a domain, the browser client will
+        # prefix the dot anyways.
+        #
+        # http://stackoverflow.com/questions/1062963/how-do-browser-cookie-domains-work/1063760#1063760
+        set_cookie << "; Domain=#{domain}" if domain.start_with?( '.' )
+
         set_cookie
     end
 
@@ -255,7 +270,7 @@ class Cookie < Base
                 next if (line = line.strip).empty? || line[0] == '#'
 
                 c = {}
-                c['domain'], foo, c['path'], c['secure'], c['expires'], c['name'],
+                c['domain'], _, c['path'], c['secure'], c['expires'], c['name'],
                     c['value'] = *line.split( "\t" )
 
                 # expiry date is optional so if we don't have one push everything back
@@ -266,7 +281,15 @@ class Cookie < Base
                     c['name'] = c['expires'].dup
                     c['expires'] = nil
                 end
+
                 c['secure'] = (c['secure'] == 'TRUE') ? true : false
+
+                c['raw_name'] = c['name']
+                c['name'] = decode( c['name'] )
+
+                c['raw_value'] = c['value']
+                c['value'] = decode( c['value'] )
+
                 new( { url: url }.merge( c.my_symbolize_keys ) )
             end.flatten.compact
         end
@@ -298,43 +321,35 @@ class Cookie < Base
         #
         # @return   [Array<Cookie>]
         #
-        # @see .from_document
+        # @see .from_parser
         # @see .from_headers
         def from_response( response )
-            from_document( response.url, response.body ) |
+            from_parser( Arachni::Parser.new( response ) ) +
                 from_headers( response.url, response.headers )
         end
 
         # Extracts cookies from a document based on `Set-Cookie` `http-equiv`
         # meta tags.
         #
-        # @param    [String]    url
-        #   Owner URL.
-        # @param    [String, Nokogiri::HTML::Document]    document
+        # @param    [Arachni::Parser]    parser
         #
         # @return   [Array<Cookie>]
         #
         # @see .parse_set_cookie
-        def from_document( url, document )
-            if !document.is_a?( Nokogiri::HTML::Document )
-                document = document.to_s
-
-                return [] if !in_html?( document )
-
-                document = Arachni::Parser.parse( document )
-            end
+        def from_parser( parser )
+            return [] if parser.body && !in_html?( parser.body )
 
             Arachni::Utilities.exception_jail {
-                document.search( '//meta[@http-equiv]' ).map do |elem|
+                parser.document.nodes_by_name( :meta ).map do |elem|
                     next if elem['http-equiv'].downcase != 'set-cookie'
 
-                    from_set_cookie( url, elem['content'] )
+                    from_set_cookie( parser.url, elem['content'] )
                 end.flatten.compact
             } rescue []
         end
 
         def in_html?( html )
-            html =~ /http-equiv/i && html =~ /set-cookie/i
+            html =~ /set-cookie/i
         end
 
         # Extracts cookies from the `Set-Cookie` HTTP response header field.
@@ -369,6 +384,16 @@ class Cookie < Base
                 cookie.instance_variables.each do |var|
                     cookie_hash[var.to_s.gsub( /@/, '' )] = cookie.instance_variable_get( var )
                 end
+
+                # http://stackoverflow.com/questions/1062963/how-do-browser-cookie-domains-work/1063760#1063760
+                if cookie_hash['domain']
+                    # IP addresses must be used verbatim.
+                    if !Arachni::URI( "http://#{cookie_hash['domain']}/" ).ip_address?
+                        cookie_hash['domain'] =
+                            ".#{cookie_hash['domain'].sub( /^\./, '' )}"
+                    end
+                end
+
                 cookie_hash['expires'] = cookie.expires
 
                 cookie_hash['path'] ||= '/'

data/lib/arachni/element/dom/capabilities/auditable.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -14,8 +14,49 @@ module Capabilities
 module Auditable
     include Arachni::Element::Capabilities::Auditable
 
-    def with_browser( &block )
-        auditor.with_browser( &block )
+    def submit_and_process( &block )
+        # If we're operating under the context of a check switch to class-level
+        # method callbacks to avoid registering multiple unique callbacks
+        # for the browser jobs, thus avoiding all the context that comes with
+        # closures.
+        if @auditor.class.respond_to? :check_and_log
+            submit( @audit_options[:submit] || {}, Auditable.audit_handle_submit_cb )
+        else
+            super( &block )
+        end
+    end
+
+    def self.handle_submission_result( page )
+        # In case of redirection or runtime scope changes.
+        return if !page.parsed_url.seed_in_host? && page.scope.out?
+
+        element = page.request.performer
+        if !element.audit_options[:silent]
+            element.print_status "Analyzing response ##{page.request.id} for " <<
+                "#{element.type} input '#{element.affected_input_name}'" <<
+                " pointing to: '#{element.audit_status_message_action}'"
+        end
+
+        Arachni::Utilities.exception_jail false do
+            element.auditor.check_and_log( page, element )
+        end
+    end
+
+    def self.audit_handle_submit( browser, options )
+        Submittable.prepare_browser( browser, options )
+        page = Submittable.submit_with_browser( browser, options )
+        # Failed to submit.
+        return if !page
+
+        Auditable.handle_submission_result page
+    end
+
+    def self.audit_handle_submit_cb
+        @audit_handle_submit_cb ||= Auditable.method(:audit_handle_submit)
+    end
+
+    def with_browser( *args, &block )
+        auditor.with_browser( *args, &block )
     end
 
     def with_browser_cluster( &block )

data/lib/arachni/element/dom/capabilities/inputtable.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/dom/capabilities/locatable.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/dom/capabilities/mutable.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -18,9 +18,13 @@ module Mutable
 
     def prepare_mutation_options( options )
         options = super( options )
-        # No sense in doing this for the DOM, either payload will be raw in the
-        # first place or the browser will override us.
+        # No sense in doing these for the DOM:
+        #
+        # Either payload will be raw in the first place or the browser will
+        # override us.
         options.delete :with_raw_payloads
+        # Browser handles the submission, there may not even be an HTTP request.
+        options.delete :with_both_http_methods
         options
     end
 

data/lib/arachni/element/dom/capabilities/submittable.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -14,35 +14,64 @@ module Capabilities
 module Submittable
     include Arachni::Element::Capabilities::Submittable
 
-    # @param  [Hash]  options
-    # @param  [Block]  block
-    #   Callback to be passed the evaluated {Page}.
-    def submit( options = {}, &block )
-        with_browser do |browser|
-            prepare_browser( browser, options )
-
-            # If we've wondered to an out-of-scope resource don't bother calling.
-            # Can be caused by a JS redirect or something akin to that.
-            if (transitions = self.trigger.compact).any?
-                page = browser.to_page
-                page.dom.transitions += transitions
-                block.call page.tap { |p| p.request.performer = self }
-            end
-
-            @element = nil
-            @browser = nil
+    def submit( options = {}, method = nil, &block )
+        # Remove references to the Auditor instance (the check instance) to
+        # remove references to the associated pages and HTTP responses etc.
+        #
+        # We don't know how long we'll be waiting in the queue so keeping these
+        # objects in memory can result in big leaks -- which is why we're also
+        # moving to class-level callbacks, to avoid closures capturing context.
+
+        auditor  = @auditor
+        @auditor = nil
+
+        options = options.merge(
+            element: self,
+            auditor: auditor.class,
+            page:    page
+        )
+
+        if method
+            auditor.with_browser( options, method )
+        else
+            auditor.with_browser( options, &Submittable.prepare_callback( &block ) )
         end
+
         nil
     end
 
-    private
+    def self.prepare_callback( &block )
+        lambda do |browser, options|
+            Submittable.prepare_browser( browser, options )
+            Submittable.submit_with_browser( browser, options, &block )
+        end
+    end
 
-    def prepare_browser( browser, options )
-        @browser = browser
+    def self.prepare_browser( browser, options )
         browser.javascript.custom_code = options[:custom_code]
         browser.javascript.taint       = options[:taint]
+        browser.load options[:page]
+    end
+
+    def self.submit_with_browser( browser, options, &cb )
+        element = options[:element]
+        element.browser = browser
+        element.auditor = options[:auditor]
+        element.page    = options[:page]
+
+        # If we've wandered to an out-of-scope resource don't bother calling.
+        # Can be caused by a JS redirect or something akin to that.
+        if (transitions = element.trigger.compact).any?
+            page = browser.to_page
+            page.dom.transitions  += transitions
+            page.request.performer = element
 
-        browser.load page
+            # Auditable.handle_submission_result page
+            cb.call( page ) if block_given?
+            return page
+        end
+
+        nil
     end
 
 end

data/lib/arachni/element/dom.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/form/capabilities/auditable.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/form/capabilities/mutable.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -95,15 +95,21 @@ module Mutable
             # and we don't want to parse it unless we have a select input.
             break if !node
 
-            escape = "'#{input.split( "'" ).join( "', \"'\", '" )}', ''"
-            node.xpath( "select[@name=concat(#{escape})]" ).css('option').each do |option|
-                try_input do
-                    elem = self.dup
-                    elem.mutation_with_original_values
-                    elem.affected_input_name  = input
-                    elem.affected_input_value = option['value'] || option.text
-                    yield elem if !generated.include?( elem )
-                    generated << elem
+            node.nodes_by_name( 'select' ).each do |select_node|
+                next if select_node['name'] != input
+
+                select_node.children.each do |child|
+                    next if !child.is_a?( Arachni::Parser::Nodes::Element ) ||
+                        child.name != :option
+
+                    try_input do
+                        elem = self.dup
+                        elem.mutation_with_original_values
+                        elem.affected_input_name  = input
+                        elem.affected_input_value = child['value'] || child.text.strip
+                        yield elem if !generated.include?( elem )
+                        generated << elem
+                    end
                 end
             end
         end
@@ -125,4 +131,3 @@ end
 end
 end
 end
-

data/lib/arachni/element/form/capabilities/submittable.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/form/capabilities/with_dom.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/form/dom.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/form.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -32,6 +32,9 @@ class Form < Base
     include Capabilities::Submittable
     include Capabilities::Mutable
 
+    include Arachni::Element::Capabilities::Auditable::Buffered
+    include Arachni::Element::Capabilities::Auditable::LineBuffered
+
     # {Form} error namespace.
     #
     # All {Form} errors inherit from and live under it.
@@ -196,8 +199,6 @@ class Form < Base
     #    </form>
     #    HTML
     #
-    #    f = Form.from_document( 'http://stuff.com', html_form ).first
-    #
     #    p f.field_type_for 'text-input'
     #    #=> :text
     #
@@ -219,17 +220,11 @@ class Form < Base
         field_type_for( name ) == :fake
     end
 
-    # @param   (see .encode)
-    # @return  (see .encode)
-    #
     # @see .encode
     def encode( str )
         self.class.encode( str )
     end
 
-    # @param   (see .decode)
-    # @return  (see .decode)
-    #
     # @see .decode
     def decode( str )
         self.class.decode( str )
@@ -254,34 +249,25 @@ class Form < Base
         #
         # @return   [Array<Form>]
         def from_response( response, ignore_scope = false )
-            from_document( response.url, response.body, ignore_scope )
+            from_parser( Arachni::Parser.new( response ), ignore_scope )
         end
 
         # Extracts forms from an HTML document.
         #
-        # @param    [String]    url
-        #   URL of the document -- used for path normalization purposes.
-        # @param    [String, Nokogiri::HTML::Document]    document
+        # @param    [Arachni::Parser]    parser
         #
         # @return   [Array<Form>]
-        def from_document( url, document, ignore_scope = false )
-            if !document.is_a?( Nokogiri::HTML::Document )
-                document = document.to_s
+        def from_parser( parser, ignore_scope = false )
+            return [] if parser.body && !in_html?( parser.body )
 
-                return [] if !in_html?( document )
+            base_url = to_absolute( parser.base, parser.url )
 
-                document = Arachni::Parser.parse( document )
-            end
-
-            base_url = (document.search( '//base[@href]' )[0]['href'] rescue url)
-            base_url = to_absolute( base_url, url )
-
-            document.search( '//form' ).map do |node|
+            parser.document.nodes_by_name( :form ).map do |node|
                 next if !(forms = from_node( base_url, node, ignore_scope ))
                 next if forms.empty?
 
                 forms.each do |form|
-                    form.url = url.freeze
+                    form.url = parser.url
                     form
                 end
             end.flatten.compact
@@ -308,9 +294,10 @@ class Form < Base
             # we keep track of this here and create a new form for each choice.
             multiple_choice_submits = {}
 
-            %w(textarea input select button).each do |attr|
-                options[attr] ||= []
-                node.css( attr ).each do |elem|
+            %w(textarea input select button).each do |tag|
+                options[tag] ||= []
+
+                node.nodes_by_name( tag ).each do |elem|
                     elem_attrs = attributes_to_hash( elem.attributes )
                     elem_attrs[:type] = elem_attrs[:type].to_sym if elem_attrs[:type]
 
@@ -318,7 +305,7 @@ class Form < Base
                     next if !name
 
                     # Handle the easy stuff first...
-                    if elem.name != 'select'
+                    if elem.name != :select
                         options[:inputs][name] = elem_attrs
 
                         if elem_attrs[:type] == :submit
@@ -336,12 +323,14 @@ class Form < Base
                         next
                     end
 
+                    children = elem.nodes_by_name( 'option' )
+
                     # If the select has options figure out which to use.
-                    if elem.children.css('option').any?
-                        elem.children.css('option').each do |child|
+                    if children.any?
+                        children.each do |child|
                             h = attributes_to_hash( child.attributes )
                             h[:type]    = :select
-                            h[:value] ||= child.text
+                            h[:value] ||= child.text.strip
 
                             if too_big?( h[:value] )
                                 h[:value] = ''

data/lib/arachni/element/generic_dom.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/header/capabilities/inputtable.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/header/capabilities/mutable.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/header.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -21,6 +21,8 @@ class Header < Base
 
     # Generic element capabilities.
     include Arachni::Element::Capabilities::Auditable
+    include Arachni::Element::Capabilities::Auditable::Buffered
+    include Arachni::Element::Capabilities::Auditable::LineBuffered
     include Arachni::Element::Capabilities::Submittable
     include Arachni::Element::Capabilities::Inputtable
     include Arachni::Element::Capabilities::Analyzable

data/lib/arachni/element/json/capabilities/inputtable.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/json/capabilities/mutable.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/json.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -22,6 +22,8 @@ class JSON < Base
 
     # Generic element capabilities.
     include Arachni::Element::Capabilities::Auditable
+    include Arachni::Element::Capabilities::Auditable::Buffered
+    include Arachni::Element::Capabilities::Auditable::LineBuffered
     include Arachni::Element::Capabilities::Submittable
     include Arachni::Element::Capabilities::Analyzable
     include Arachni::Element::Capabilities::WithSource
@@ -67,17 +69,11 @@ class JSON < Base
         { self.action => self.inputs }
     end
 
-    # @param   (see .encode)
-    # @return  (see .encode)
-    #
     # @see .encode
     def encode( *args )
         self.class.encode( *args )
     end
 
-    # @param   (see .decode)
-    # @return  (see .decode)
-    #
     # @see .decode
     def decode( *args )
         self.class.decode( *args )
@@ -113,7 +109,7 @@ class JSON < Base
             rescue ::JSON::ParserError
             end
 
-            return if !data || data.empty?
+            return if !data.is_a?( Hash ) || data.empty?
 
             new(
                 url:    url,