arachni 1.4 → 1.6.0

data/lib/arachni/element/capabilities/analyzable/signature.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -20,8 +20,11 @@ module Signature
     }
 
     SIGNATURE_OPTIONS = {
-        # The signatures to look for the response body, if `Regexp` it will be
-        # matched against it, if `String` it'll be used as a needle.
+        # The signatures to look for in each line of the response body,
+        # if `Regexp` it will be matched against it, if `String` it'll be used
+        # as a needle.
+        #
+        # Multi-line Regexp is not supported.
         signatures: [],
 
         # Array of signatures to ignore.
@@ -34,20 +37,19 @@ module Signature
     FILE_SIGNATURES = {
         'environ'  => proc do |response|
             next if !response.body.include?( 'DOCUMENT_ROOT=' )
-            /DOCUMENT_ROOT=.*HTTP_USER_AGENT=/m
+            /DOCUMENT_ROOT=.*HTTP_USER_AGENT=/
         end,
         'passwd'   => proc do |response|
-            next if !response.body.include?( 'bin/' )
-            /:.+:\d+:\d+:.+:[0-9a-zA-Z\/]+/
-        end,
-        'boot.ini' => proc do |response|
-            next if !response.body.include?( '[boot loader]' )
-            /\[boot loader\].*\[operating systems\]/m
-        end,
-        'win.ini'  => proc do |response|
-            next if !response.body.include?( '[fonts]' )
-            /\[fonts\].*\[extensions\]/m
+            if response.body.include?( 'bin/' )
+                /:.+:\d+:\d+:.+:[0-9a-zA-Z\/]+/
+
+            # Response may have encoded chars as HTML entities.
+            elsif response.body.include?( 'bin&#x2f;' ) && response.body.include?( '&#x3a;' )
+                /&#x3a;.+&#x3a;\d+&#x3a;\d+&#x3a;.+&#x3a;[0-9a-zA-Z&#;]+/
+            end
         end,
+        'boot.ini' => '[boot loader]',
+        'win.ini'  => '[extensions]',
         'web.xml'  => '<web-app'
     }
 
@@ -77,15 +79,17 @@ module Signature
         java: [
             /<%|<%=|<%@\s+page|<%@\s+include|<%--|import\s+javax.servlet|
                 import\s+java.io|import=['"]java.io|request\.getParameterValues\(|
-                response\.setHeader|response\.setIntHeader\(/m
+                response\.setHeader|response\.setIntHeader\(/
         ],
         asp:  [
             /<%|Response\.Write|Request\.Form|Request\.QueryString|
                 Response\.Flush|Session\.SessionID|Session\.Timeout|
-                Server\.CreateObject|Server\.MapPath/m
+                Server\.CreateObject|Server\.MapPath/
         ]
     }
 
+    LINE_BUFFER_SIZE = 1_000
+
     # Performs signatures analysis and logs an issue, should there be one.
     #
     # It logs an issue when:
@@ -105,7 +109,7 @@ module Signature
     #       {Element::Capabilities::Submittable#platforms applicable platforms}
     #       for the {Element::Capabilities::Submittable#action resource} to be audited.
     # @param  [Hash]    opts
-    #   Options as described in {Arachni::Check::Auditor::OPTIONS} and
+    #   Options as described in {Arachni::Element::Auditable::OPTIONS} and
     #   {SIGNATURE_OPTIONS}.
     #
     # @return   [Bool]
@@ -124,8 +128,10 @@ module Signature
         # we've evaluated our control response.
         @candidate_issues = []
 
-        # Perform the analysis.
         opts = self.class::OPTIONS.merge( SIGNATURE_OPTIONS.merge( opts ) )
+
+        fail_if_signatures_invalid( opts[:signatures] )
+
         audit( payloads, opts ) { |response| get_matches( response ) }
     end
 
@@ -150,6 +156,22 @@ module Signature
     end
     public :get_matches
 
+    def fail_if_signatures_invalid( signatures )
+        case signatures
+            when Regexp
+                if (signatures.options & Regexp::MULTILINE) == Regexp::MULTILINE
+                    fail ArgumentError,
+                         'Multi-line regular expressions are not supported.'
+                end
+
+            when Array
+                signatures.each { |s| fail_if_signatures_invalid s }
+
+            when Hash
+                fail_if_signatures_invalid signatures.values
+        end
+    end
+
     def find_signatures( signatures, response, opts )
         k = [signatures, response.body]
         return if SIGNATURE_CACHE[:match][k] == false

data/lib/arachni/element/capabilities/analyzable/timeout.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/capabilities/analyzable.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/capabilities/auditable/buffered.rb

@@ -0,0 +1,92 @@
+=begin
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
+
+    This file is part of the Arachni Framework project and is subject to
+    redistribution and commercial restrictions. Please see the Arachni Framework
+    web site for more information on licensing and terms of use.
+=end
+
+module Arachni
+module Element::Capabilities
+module Auditable
+
+# @author Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>
+module Buffered
+
+    DEFAULT_BUFFER_SIZE = 15_000
+
+    def buffered_audit( payloads, options = {}, &block )
+        fail ArgumentError, 'Missing block.' if !block_given?
+
+        options     = options.dup
+        buffer_size = options[:buffer_size] || DEFAULT_BUFFER_SIZE
+
+        print_debug_level_2 "About to audit at least #{buffer_size} bytes at a time: #{audit_id}"
+
+        buffers = {}
+
+        options[:submit] ||= {}
+        options[:submit][:on_body] = proc do |chunk, response|
+            # In case of redirection or runtime scope changes.
+            if !response.parsed_url.seed_in_host? && response.scope.out?
+                print_debug_level_3 "Response out of scope for #{audit_id}: #{response.url}"
+                print_debug_level_3 'Aborting...'
+                next :abort
+            end
+
+            print_debug_level_3 "Got data for: #{audit_id}"
+            if debug?( 4 )
+                print_debug_level_4 chunk
+            end
+
+            request = response.request
+
+            buffers[request.id] ||= ''
+            buffer = buffers[request.id]
+
+            buffer << chunk
+
+            print_debug_level_3 "Buffer is at: #{buffer.size}/#{buffer_size}"
+            next if buffer.size < buffer_size
+
+            print_debug_level_3 'Buffer full, setting response body.'
+            print_debug_level_4 buffer
+            response.body = buffer
+
+            print_debug_level_3 "Calling: #{block}"
+            r = block.call( response, request.performer, false )
+
+            print_debug_level_3 "Block returned: #{r}"
+            print_debug_level_3 'Emptying buffer.'
+
+            # Create a new object, we don't want to mess with reference issues.
+            buffers[request.id] = ''
+
+            r
+        end
+
+        audit( payloads, options ) do |response|
+            request = response.request
+            buffer  = buffers[request.id]
+
+            if !buffer.to_s.empty?
+                print_debug_level_3 "There's more data in the buffer, setting response body."
+                print_debug_level_3 buffer
+
+                response.body = buffer
+            else
+                print_debug_level_3 "There's no buffer, leaving response body as is."
+                print_debug_level_3 response.body
+            end
+
+            block.call response, request.performer, true
+
+            buffers.delete( request.id )
+        end
+    end
+
+end
+
+end
+end
+end

data/lib/arachni/element/capabilities/auditable/line_buffered.rb

@@ -0,0 +1,103 @@
+=begin
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
+
+    This file is part of the Arachni Framework project and is subject to
+    redistribution and commercial restrictions. Please see the Arachni Framework
+    web site for more information on licensing and terms of use.
+=end
+
+module Arachni
+module Element::Capabilities
+module Auditable
+
+# @author Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>
+module LineBuffered
+
+    DEFAULT_LINE_BUFFER_SIZE = 1_000
+
+    def line_buffered_audit( payloads, options = {}, &block )
+        fail ArgumentError, 'Missing block.' if !block_given?
+
+        options     = options.dup
+        buffer_size = options[:buffer_size] || DEFAULT_LINE_BUFFER_SIZE
+
+        print_debug_level_2 "About to audit #{buffer_size} lines at a time: #{audit_id}"
+
+        buffers = {}
+
+        options[:submit] ||= {}
+        options[:submit][:on_body_lines] = proc do |lines, response|
+            # In case of redirection or runtime scope changes.
+            if !response.parsed_url.seed_in_host? && response.scope.out?
+                print_debug_level_3 "Response out of scope for #{audit_id}: #{response.url}"
+                print_debug_level_3 'Aborting...'
+                next :abort
+            end
+
+            print_debug_level_3 "Got lines for: #{audit_id}"
+            print_debug_level_4 lines
+
+            request = response.request
+
+            buffers[request.id] ||= {
+                data:    '',
+                counter: 0
+            }
+            buffer = buffers[request.id]
+
+            buffer[:data]    << lines
+            buffer[:counter] += lines.count( "\n" )
+
+            print_debug_level_3 "Buffer is at: #{buffer[:counter]}/#{buffer_size}"
+            next if buffer[:counter] < buffer_size
+
+            print_debug_level_3 'Buffer full, setting response body.'
+            print_debug_level_4 buffer[:data]
+
+            response.body = buffer[:data]
+
+            print_debug_level_3 "Calling: #{block}"
+
+            # `false` means we're still buffering.
+            r = block.call( response, request.performer, false )
+
+            print_debug_level_3 "Block returned: #{r}"
+            print_debug_level_3 'Emptying buffer.'
+
+            # Create a new object, we don't want to mess with reference issues.
+            buffer[:data]    = ''
+            buffer[:counter] = 0
+
+            r
+        end
+
+        audit( payloads, options ) do |response|
+            print_debug_level_3 "Line buffering completed for: #{audit_id}"
+
+            request = response.request
+            buffer  = buffers[request.id]
+
+            # The response body can include remnants from the HTTP line buffer
+            # and our own buffer could have lines that didn't exceed the flush
+            # threshold, hence we combine them
+            if buffer && !buffer[:data].empty?
+                b = response.body
+                response.body = buffer[:data]
+                response.body << b
+            end
+
+            print_debug_level_3 "Calling: #{block}"
+
+            # `true` means we've read the entire response.
+            block.call response, request.performer, true
+
+            print_debug_level_3 'Deleted buffer.'
+            buffers.delete( request.id )
+        end
+    end
+
+end
+
+end
+end
+end

data/lib/arachni/element/capabilities/auditable.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -105,12 +105,10 @@ module Auditable
     #       * There are no `payloads` applicable to the element's platforms.
     #
     # @raise    ArgumentError
-    #   On missing `block` or unsupported `payloads` type.
+    #   On unsupported `payloads` type.
     #
     # @see #submit
     def audit( payloads, opts = {}, &block )
-        fail ArgumentError, 'Missing block.' if !block_given?
-
         return false if self.inputs.empty?
 
         if scope.out?
@@ -282,12 +280,8 @@ module Auditable
     #       is `false` -- the default.
     #    * The element matches a {.skip_like} block.
     #
-    # @raise    [ArgumentError]
-    #   On missing `block`.
-    #
     # @see #submit
     def audit_single( payload, opts = { }, &block )
-        fail ArgumentError, 'Missing block.' if !block_given?
 
         if !valid_input_data?( payload )
             print_debug_level_2 "Payload not supported by #{self}: #{payload.inspect}"

data/lib/arachni/element/capabilities/dom_only.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/capabilities/inputtable.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -38,7 +38,7 @@ module Capabilities::Inputtable
     end
 
     INPUTTABLE_CACHE = {
-        inputtable_id: Support::Cache::LeastRecentlyPushed.new( 100_000 )
+        inputtable_id: Support::Cache::LeastRecentlyPushed.new( 1_000 )
     }
 
     # Frozen version of {#inputs}, has all the original names and values.
@@ -134,6 +134,10 @@ module Capabilities::Inputtable
         end
     end
 
+    def updated?
+        @default_inputs != self.inputs
+    end
+
     # Resets the inputs to their original format/values.
     def reset
         super if defined?( super )

data/lib/arachni/element/capabilities/mutable.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/capabilities/refreshable.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/capabilities/submittable.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/capabilities/with_auditor/output.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -21,8 +21,9 @@ module Output
     [ :debug?, :print_error, :print_status, :print_verbose, :print_info,
       :print_line, :print_ok, :print_bad, :print_debug, :print_debug_backtrace,
       :print_error_backtrace, :print_debug_level_1, :print_debug_level_2,
-      :print_debug_level_3, :debug_level_1?, :debug_level_2?, :debug_level_3?,
-      :print_exception, :print_debug_exception ].each do |method|
+      :print_debug_level_3, :print_debug_level_4, :print_debug_level_5,
+      :debug_level_1?, :debug_level_2?, :debug_level_3?, :debug_level_4?,
+      :debug_level_5?, :print_exception, :print_debug_exception ].each do |method|
         define_method method do |*args|
             (orphan? ? UI::Output : auditor).send( method, *args )
         end

data/lib/arachni/element/capabilities/with_auditor.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/capabilities/with_dom.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/capabilities/with_node.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -15,10 +15,10 @@ module Element::Capabilities
 module WithNode
     include WithSource
 
-    # @return [Nokogiri::XML::Element]
+    # @return [Ox::Element]
     def node
         return if !@source
-        Arachni::Parser.parse_fragment( @source.dup )
+        Arachni::Parser.parse_fragment( @source )
     end
 
 end

data/lib/arachni/element/capabilities/with_scope/scope.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/capabilities/with_scope.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/capabilities/with_source.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -27,7 +27,7 @@ module WithSource
     end
 
     def source=( s )
-        @source = (s ? s : s.freeze )
+        @source = (s ? s.strip : s.freeze )
     end
 
     def to_h

data/lib/arachni/element/cookie/capabilities/inputtable.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/cookie/capabilities/mutable.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/cookie/capabilities/with_dom.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/cookie/dom.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework