arachni 1.4 → 1.6.0

data/lib/arachni/parser.rb

@@ -1,15 +1,30 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
     web site for more information on licensing and terms of use.
 =end
 
+require 'ox'
+
+Ox.default_options = {
+    indent:          4,
+    mode:            :generic,
+    effort:          :tolerant,
+    smart:           true,
+    invalid_replace: nil
+}
+
 module Arachni
 
 lib = Options.paths.lib
 
+require lib + 'parser/extractors/base'
+require lib + 'parser/document'
+require lib + 'parser/sax'
+require lib + 'parser/with_children'
+
 # Load all available element types.
 Dir.glob( lib + 'element/*.rb' ).each { |f| require f }
 
@@ -24,42 +39,11 @@ class Parser
     include UI::Output
     include Utilities
 
-    # @author Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>
-    module Extractors
-
-        # @author Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>
-        # @abstract
-        class Base
-
-            attr_reader :html
-            attr_reader :document
-            attr_reader :downcased_html
-
-            def initialize( options = {} )
-                @html           = options[:html]
-                @downcased_html = @html.downcase
-                @document       = options[:document]
-            end
-
-            # This method must be implemented by all checks and must return an
-            # array of paths as plain strings
-            #
-            # @return   [Array<String>]  paths
-            # @abstract
-            def run
-            end
-
-            def includes?( string_or_regexp )
-                !!@downcased_html[string_or_regexp]
-            end
-
-        end
-    end
-
     CACHE_SIZES = {
         parse:          50,
         parse_xml:      50,
-        parse_fragment: 100
+        parse_fragment: 100,
+        html?:          100_000
     }
 
     CACHE = {}
@@ -67,27 +51,115 @@ class Parser
         CACHE[name] = Support::Cache::LeastRecentlyPushed.new( size )
     end
 
+    WHITELIST = %w(
+        title base a form frame iframe meta input select option script link area
+        textarea input select button comment !--
+    )
+
+    IGNORE_REQUEST_HEADERS = [
+        HTTP::Client::SEED_HEADER_NAME,
+        'Content-Length'
+    ]
+
     class <<self
 
-        def parse( html )
-            CACHE[__method__].fetch html do
-                Nokogiri::HTML( html ).freeze
+        def parse( html, options = {} )
+            CACHE[__method__].fetch [html, options] do
+                handler, sax_options = prepare_ox_options( options )
+
+                begin
+                    Ox.sax_html( handler, StringIO.new( html ), sax_options )
+                rescue SAX::Stop
+                end
+
+                handler.document
             end
         end
 
+        def push_parse( options = {} )
+            buffer, buffer_in = IO.pipe
+
+            document, sax_options = prepare_ox_options( options )
+
+            push_parse_pool.post do
+                begin
+                    Ox.sax_html( document, buffer, sax_options )
+                rescue SAX::Stop
+                end
+            end
+
+            [buffer_in, document]
+        end
+
         def parse_fragment( html )
             CACHE[__method__].fetch html do
-                Nokogiri::HTML.fragment( html ).children.first.freeze
+                parse( html ).children.first.tap do |o|
+                    o.parent   = nil
+                    o.document = nil
+                end
             end
         end
 
         def parse_xml( xml )
             CACHE[__method__].fetch xml do
-                Nokogiri::XML( xml ).freeze
+                Nokogiri::XML( xml )
+            end
+        end
+
+        def markup?( string )
+            begin
+                Ox.parse( string ).is_a?( Ox::Element )
+            rescue => e
+                false
+            end
+        end
+
+        def html?( string )
+            CACHE[__method__].fetch string do
+                begin
+                    _html? string
+                rescue => e
+                    false
+                end
+            end
+        end
+
+        private
+
+        def _html?( string )
+            parse( string ).traverse do |n|
+                return true if n.is_a? Nodes::Element
+            end
+
+            false
+        end
+
+        def push_parse_pool
+            @push_parse_pool ||= Concurrent::CachedThreadPool.new
+        end
+
+        def prepare_ox_options( options )
+            handler = options[:handler] || SAX.new( options )
+
+            sax_options = {}
+            if options[:whitelist] && options[:whitelist].any?
+                overlay = Ox.sax_html_overlay.dup
+                overlay.each do |k, v|
+                    overlay[k] = :off
+                end
+
+                options[:whitelist].each do |e|
+                    overlay[e] = :active
+                end
+
+                sax_options[:overlay] = overlay
             end
+
+            [handler, sax_options]
         end
 
     end
+    push_parse_pool
 
     alias :skip? :skip_path?
 
@@ -95,28 +167,40 @@ class Parser
     attr_reader :url
 
     # @return   [HTTP::Response]
-    attr_reader :response
+    attr_accessor :response
 
-    # @param  [HTTP::Response, Array<HTTP::Response>] response
+    # @param  [Document, HTTP::Response, Array<HTTP::Response>] resource
     #   Response(s) to analyze and parse. By providing multiple responses the
     #   parser will be able to perform some preliminary differential analysis
     #   and identify nonce tokens in inputs.
-    #
-    # @param  [Options] options
-    def initialize( response, options = Options )
-        @options = options
-
-        if response.is_a? Array
-            @secondary_responses = response[1..-1]
-            @secondary_responses.compact! if @secondary_responses
-            response = response.shift
-        end
+    def initialize( resource )
+        case resource
+
+            when Document
+                @resource = :document
+                @document = resource
+
+            when HTTP::Response
+                @resource = :response
+
+                @response = resource
+                self.url = @response.url
+
+            when Array
+                @secondary_responses = resource[1..-1]
+                @secondary_responses.compact! if @secondary_responses
+                response = resource.shift
 
-        @response = response
-        self.url  = response.url
+                @resource = :response
+
+                @response = response
+                self.url = response.url
+        end
     end
 
     def url=( str )
+        return @url = nil if !str
+
         @url = normalize_url( uri_decode( str ) )
         @url = normalize_url( str ) if !@url
         @url.freeze
@@ -147,7 +231,15 @@ class Parser
     # @return   [Boolean]
     #   `true` if the given HTTP response data are text based, `false` otherwise.
     def text?
-        !@body.to_s.empty? || @response.text?
+        from_response? ? @response.text? : true
+    end
+
+    def from_response?
+        @resource == :response
+    end
+
+    def from_document?
+        @resource == :document
     end
 
     # @return    [String]
@@ -158,20 +250,18 @@ class Parser
     end
 
     def body
-        @body || @response.body
+        @body || (@response.body if from_response?)
     end
 
-    # @return   [Nokogiri::HTML, nil]
+    # @return   [Arachni::Parser::Document, nil]
     #   Returns a parsed HTML document from the body of the HTTP response or
     #   `nil` if the response data wasn't {#text? text-based} or the response
     #   couldn't be parsed.
     def document
+        return @document if @document
         return if !text?
-        return @document.freeze if @document
 
-        @document = self.class.parse( body )
-    rescue
-        nil
+        @document = self.class.parse( body, filter: true )
     end
 
     # @note It will include common request headers as well headers from the HTTP
@@ -185,34 +275,37 @@ class Parser
                 '/xml;q=0.9,*/*;q=0.8',
             'Accept-Charset'  => 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
             'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
-            'From'            => @options.authorized_by  || '',
-            'User-Agent'      => @options.http.user_agent || '',
+            'From'            => Options.authorized_by  || '',
+            'User-Agent'      => Options.http.user_agent || '',
             'Referer'         => @url,
             'Pragma'          => 'no-cache'
-        }.merge( response.request.headers ).
-            map { |k, v| Header.new( url: @url, inputs: { k => v } ) }.freeze
+        }.merge(
+            response.request.headers.dup.tap do |h|
+                IGNORE_REQUEST_HEADERS.each { |k| h.delete k }
+            end
+        ).map { |k, v| Header.new( url: @url, inputs: { k => v } ) }.freeze
     end
 
     # @return [Array<Element::Form>]
     #   Forms from {#document}.
     def forms
         return @forms.freeze if @forms
-        return [] if !text? || !Form.in_html?( body )
+        return [] if !text? || (body && !Form.in_html?( body ))
 
-        f = Form.from_document( @url, document )
+        f = Form.from_parser( self )
         return f if !@secondary_responses
 
         @secondary_responses.each do |response|
             next if response.body.to_s.empty?
 
-            Form.from_document( @url, response.body ).each do |form2|
+            Form.from_parser( Parser.new( response ) ).each do |form2|
                 f.each do |form|
                     next if "#{form.coverage_id}:#{form.name_or_id}" !=
                         "#{form2.coverage_id}:#{form2.name_or_id}"
 
                     form.inputs.each do |k, v|
-                        next if !(v != form2.inputs[k] &&
-                            form.field_type_for( k ) == :hidden)
+                        next if v == form2.inputs[k] ||
+                            form.field_type_for( k ) != :hidden
 
                         form.nonce_name = k
                     end
@@ -226,7 +319,7 @@ class Parser
     # @return [Element::Link]
     #   Link to the page.
     def link
-        return if link_vars.empty? && !@response.redirection?
+        return if link_vars.empty? && (@response && !@response.redirection?)
         Link.new( url: @url, inputs: link_vars )
     end
 
@@ -248,9 +341,9 @@ class Parser
     #   Links in {#document}.
     def links
         return @links.freeze if @links
-        return @links = [link].compact if !text? || !Link.in_html?( body )
+        return @links = [link].compact if !text? || (body && !Link.in_html?( body ))
 
-        @links = [link].compact | Link.from_document( @url, document )
+        @links = [link].compact | Link.from_parser( self )
     end
 
     # @return [Array<Element::LinkTemplate>]
@@ -260,7 +353,7 @@ class Parser
         return @link_templates = [link_template].compact if !text?
 
         @link_templates =
-            [link_template].compact | LinkTemplate.from_document( @url, document )
+            [link_template].compact | LinkTemplate.from_parser( self )
     end
 
     # @return [Array<Element::JSON>]
@@ -299,7 +392,15 @@ class Parser
         @cookies = Cookie.from_headers( @url, @response.headers )
         return @cookies if !text? || !Cookie.in_html?( body )
 
-        @cookies |= Cookie.from_document( @url, document )
+        @cookies |= Cookie.from_parser( self )
+    end
+
+    # @return   [Array<Element::NestedCookie>]
+    #   Nested cookies from {#cookies_to_be_audited}.
+    def nested_cookies
+        return @nested_cookies.freeze if @nested_cookies
+
+        @nested_cookies = NestedCookie.from_cookies( cookies_to_be_audited )
     end
 
     # @return   [Array<Element::Cookie>]
@@ -356,7 +457,7 @@ class Parser
     # @return   [String]
     #   Base `href`, if there is one.
     def base
-        @base ||= document.search( '//base[@href]' ).first['href'] rescue nil
+        @base ||= document.nodes_by_name( :base ).map { |b| b['href'] }.first || @url
     end
 
     private
@@ -371,14 +472,16 @@ class Parser
             self.class.extractors.available.each do |name|
                 exception_jail false do
                     unsanitized_paths.merge self.class.extractors[name].new(
-                        document: document,
-                        html:     body
-                    ).run
+                        parser: self,
+                        html:   body
+                    ).run.flatten
                 end
             end
 
             sanitized_paths = Set.new
             unsanitized_paths.map do |path|
+                next if !path || path =~ /^mailto:/i
+
                 abs = to_absolute( path )
                 next if !abs || skip?( abs )
 

data/lib/arachni/platform/fingerprinter.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/platform/list.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/platform/manager.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -175,7 +175,7 @@ class Manager
         cherrypy: 'CherryPy'
     }
 
-    PLATFORM_CACHE_SIZE = 1000
+    PLATFORM_CACHE_SIZE = 500
 
     def self.synchronize( &block )
         @mutex.synchronize( &block )

data/lib/arachni/platform.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/plugin/base.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -92,7 +92,7 @@ class Base < Component::Base
 
     # Pauses the {#framework}.
     def framework_pause
-        @pause_id ||= framework.pause
+        @pause_id ||= framework.pause( false )
     end
 
     # Aborts the {#framework}.

data/lib/arachni/plugin/formatter.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/plugin/manager.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -64,17 +64,20 @@ class Manager < Arachni::Component::Manager
 
         schedule.each do |name, options|
             instance = create( name, options )
-            instance.prepare
+
+            exception_jail do
+                instance.prepare
+            end rescue next
 
             @jobs[name] = Thread.new do
                 exception_jail( false ) do
                     Thread.current[:instance] = instance
                     Thread.current[:instance].run
                     Thread.current[:instance].clean_up
+                end
 
-                    synchronize do
-                        @jobs.delete name
-                    end
+                synchronize do
+                    @jobs.delete name
                 end
             end
         end

data/lib/arachni/plugin.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/processes/dispatchers.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/processes/executables/base.rb

@@ -6,7 +6,8 @@ if !$options[:without_arachni]
     require 'arachni'
 
     include Arachni
-    Options.update $options.delete(:options)
+
+    Options.update Marshal.load( Base64.strict_decode64( ENV['arachni_options'] ) )
 else
     if Gem.win_platform?
         require 'Win32API'

data/lib/arachni/processes/executables/browser.rb

@@ -73,8 +73,6 @@ puts_stderr "#{Process.pid}: Started"
 $stdout.puts "PID: #{process.pid}"
 
 while !exit?
-    puts_stderr "#{Process.pid}: Working"
-
     begin
         break if !process.alive?
 

data/lib/arachni/processes/helpers/dispatchers.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/processes/helpers/instances.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/processes/helpers/processes.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/processes/helpers.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/processes/instances.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/processes/manager.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -38,16 +38,21 @@ class Manager
         pid
     end
 
-    # @param    [Integer]   pid PID of the process to kill.
+    # @param    [Integer]   pid
+    #   PID of the process to kill.
     def kill( pid )
         Timeout.timeout 10 do
             while sleep 0.1 do
                 begin
                     Process.kill( Arachni.windows? ? 'KILL' : 'TERM', pid )
-                # Either kill was succesful or we don't have enough perms or
+
+                # Either kill was successful or we don't have enough perms or
                 # we hit a reused PID for someone else's process, either way,
                 # consider the process gone.
-                rescue Errno::ESRCH, Errno::EPERM
+                rescue Errno::ESRCH, Errno::EPERM,
+                    # Don't kill ourselves.
+                    SignalException
+
                     @pids.delete pid
                     return
                 end
@@ -127,7 +132,7 @@ class Manager
     #   PID of the process.
     def spawn( executable, options = {} )
         fork = options.delete(:fork)
-        fork = true if fork.nil?
+        fork = false if fork.nil?
 
         stdin      = options.delete(:stdin)
         stdout     = options.delete(:stdout)
@@ -150,13 +155,12 @@ class Manager
 
         options[:ppid]  = Process.pid
 
-        options[:options] ||= {}
-        options[:options]   = Options.to_h.merge( options[:options] )
-
+        arachni_options = Options.to_h.merge( options.delete(:options) || {} )
         # Paths are not included in RPC nor Hash representations as they're
         # considered local, in this case though they're necessary to provide
         # the same environment the processes.
-        options[:options][:paths] = Options.paths.to_h
+        arachni_options[:paths] = Options.paths.to_h
+        encoded_arachni_options = Base64.strict_encode64( Marshal.dump( arachni_options ) )
 
         executable      = "#{Options.paths.executables}/#{executable}.rb"
         encoded_options = Base64.strict_encode64( Marshal.dump( options ) )
@@ -194,6 +198,8 @@ class Manager
                 Framework.reset
                 Reactor.stop
 
+                ENV['arachni_options'] = encoded_arachni_options
+
                 ARGV.replace( argv )
                 load RUNNER
             end
@@ -202,6 +208,9 @@ class Manager
             # it bypasses the OS shell and we can thus count on a 1-to-1 process
             # creation and that the PID we get will be for the actual process.
             pid = Process.spawn(
+                {
+                    'arachni_options' => encoded_arachni_options
+                },
                 RbConfig.ruby,
                 RUNNER,
                 *(argv + [spawn_options])

data/lib/arachni/processes.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework