arachni 1.3.2 → 1.4

data/lib/arachni/page/dom/transition.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -235,7 +235,7 @@ class Transition
     end
 
     def dup
-        deep_clone
+        rpc_clone
     end
 
     # @return   [Hash]

data/lib/arachni/page/scope.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/parser.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -56,6 +56,39 @@ class Parser
         end
     end
 
+    CACHE_SIZES = {
+        parse:          50,
+        parse_xml:      50,
+        parse_fragment: 100
+    }
+
+    CACHE = {}
+    CACHE_SIZES.each do |name, size|
+        CACHE[name] = Support::Cache::LeastRecentlyPushed.new( size )
+    end
+
+    class <<self
+
+        def parse( html )
+            CACHE[__method__].fetch html do
+                Nokogiri::HTML( html ).freeze
+            end
+        end
+
+        def parse_fragment( html )
+            CACHE[__method__].fetch html do
+                Nokogiri::HTML.fragment( html ).children.first.freeze
+            end
+        end
+
+        def parse_xml( xml )
+            CACHE[__method__].fetch xml do
+                Nokogiri::XML( xml ).freeze
+            end
+        end
+
+    end
+
     alias :skip? :skip_path?
 
     # @return    [String]
@@ -133,8 +166,12 @@ class Parser
     #   `nil` if the response data wasn't {#text? text-based} or the response
     #   couldn't be parsed.
     def document
+        return if !text?
         return @document.freeze if @document
-        @document = Nokogiri::HTML( body ) if text? rescue nil
+
+        @document = self.class.parse( body )
+    rescue
+        nil
     end
 
     # @note It will include common request headers as well headers from the HTTP
@@ -160,7 +197,7 @@ class Parser
     #   Forms from {#document}.
     def forms
         return @forms.freeze if @forms
-        return [] if !text? || !(body =~ /<\s*form/i)
+        return [] if !text? || !Form.in_html?( body )
 
         f = Form.from_document( @url, document )
         return f if !@secondary_responses
@@ -211,7 +248,7 @@ class Parser
     #   Links in {#document}.
     def links
         return @links.freeze if @links
-        return @links = [link].compact if !text? || !(body =~ /\?.*=/)
+        return @links = [link].compact if !text? || !Link.in_html?( body )
 
         @links = [link].compact | Link.from_document( @url, document )
     end
@@ -260,7 +297,7 @@ class Parser
         return @cookies.freeze if @cookies
 
         @cookies = Cookie.from_headers( @url, @response.headers )
-        return @cookies if !text?
+        return @cookies if !text? || !Cookie.in_html?( body )
 
         @cookies |= Cookie.from_document( @url, document )
     end

data/lib/arachni/platform.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/platform/fingerprinter.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/platform/list.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/platform/manager.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -290,7 +290,7 @@ class Manager
         end
 
         return new_from_options if !(key = make_key( uri ))
-        synchronize { @platforms[key] ||= new_from_options }
+        synchronize { @platforms.fetch(key) { new_from_options } }
     end
 
     # Sets platform manager for the given `uri`.

data/lib/arachni/plugin.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/plugin/base.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/plugin/formatter.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/plugin/manager.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -60,18 +60,15 @@ class Manager < Arachni::Component::Manager
     # @raise [Error::UnsatisfiedDependency]
     #   If the environment is {#sane_env? not sane}.
     def run
-        prepared_signals = Queue.new
+        print_status 'Preparing plugins...'
 
         schedule.each do |name, options|
+            instance = create( name, options )
+            instance.prepare
+
             @jobs[name] = Thread.new do
                 exception_jail( false ) do
-                    begin
-                        Thread.current[:instance] = create( name, options )
-                        Thread.current[:instance].prepare
-                    ensure
-                        prepared_signals << nil
-                    end
-
+                    Thread.current[:instance] = instance
                     Thread.current[:instance].run
                     Thread.current[:instance].clean_up
 
@@ -82,10 +79,6 @@ class Manager < Arachni::Component::Manager
             end
         end
 
-        return if @jobs.empty?
-
-        print_status 'Preparing plugins...'
-        self.size.times { prepared_signals.pop }
         print_status '... done.'
     end
 
@@ -111,6 +104,7 @@ class Manager < Arachni::Component::Manager
 
         ordered << unordered
         ordered.flatten!
+        ordered.compact!
 
         ordered.inject({}) do |h, ph|
             name   = ph.keys.first

data/lib/arachni/processes.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/processes/dispatchers.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -75,7 +75,7 @@ class Dispatchers
                 pool_size:        options[:pool_size]
             },
             rpc:        {
-                server_port:    options[:port]    || available_port,
+                server_port:    options[:port]    || Utilities.available_port,
                 server_address: options[:address] || '127.0.0.1'
             }
         }

data/lib/arachni/processes/executables/base.rb

@@ -1,9 +1,50 @@
-require 'arachni'
-
-include Arachni
+require 'base64'
 
 $options = Marshal.load( Base64.strict_decode64( ARGV.pop ) )
 
-Options.update $options.delete(:options)
+if !$options[:without_arachni]
+    require 'arachni'
+
+    include Arachni
+    Options.update $options.delete(:options)
+else
+    if Gem.win_platform?
+        require 'Win32API'
+        require 'win32ole'
+    end
+end
+
+def ppid
+    $options[:ppid]
+end
+
+def parent_alive?
+    # Windows is not big on POSIX so try it its own way if possible.
+    if Gem.win_platform?
+        begin
+            alive = false
+            wmi = WIN32OLE.connect( 'winmgmts://' )
+            processes = wmi.ExecQuery( "select ProcessId from win32_process where ProcessID='#{ppid}'")
+            processes.each do |proc|
+                proc.ole_free
+                alive = true
+            end
+            processes.ole_free
+            wmi.ole_free
+
+            return alive
+        rescue WIN32OLERuntimeError
+        end
+    end
+
+    !!(Process.kill( 0, ppid ) rescue false)
+end
+
+def puts_stderr( str )
+    return if $stderr.closed?
+
+    $stderr.puts str
+rescue
+end
 
 load ARGV.pop

data/lib/arachni/processes/executables/browser.rb

@@ -0,0 +1,91 @@
+require 'childprocess'
+
+def print_exception( e )
+    puts_stderr "#{Process.pid}: [#{e.class}] #{e}"
+    e.backtrace.each do |line|
+        puts_stderr "#{Process.pid}: #{line}"
+    end
+rescue
+end
+
+def exit?
+    $stdin.read_nonblock( 1 )
+    false
+rescue Errno::EWOULDBLOCK
+    false
+# Parent dead or willfully closed STDIN as a signal.
+rescue EOFError, Errno::EPIPE => e
+    print_exception( e )
+    true
+end
+
+executable = $options[:executable]
+port       = $options[:port]
+proxy_url  = $options[:proxy_url]
+
+process = ChildProcess.build(
+    executable,
+    "--webdriver=#{port}",
+    "--proxy=#{proxy_url}",
+
+    # As lax as possible to allow for easy SSL interception.
+    # The actual request to the origin server will obey the system-side SSL options.
+    '--ignore-ssl-errors=true',
+    '--ssl-protocol=any',
+
+    # Uncomment to show better error messages.
+    # '--web-security=false',
+
+    '--disk-cache=true'
+)
+
+handle_exit = proc do
+    next if @called
+    @called = true
+
+    puts_stderr "#{Process.pid}: Exiting"
+
+    begin
+        process.stop
+    rescue => e
+        print_exception( e )
+    end
+end
+
+at_exit( &handle_exit )
+
+# Try our best to terminate cleanly if some external entity tries to kill us.
+%w(EXIT TERM QUIT INT KILL).each do |signal|
+    next if !Signal.list.include?( signal )
+    trap( signal, &handle_exit ) rescue Errno::EINVAL
+end
+
+# Break out of the process group in order to ignore signals sent to the parent.
+process.leader = true
+
+# Forward output.
+process.io.stdout = $stdout
+process.io.stdout.sync = true
+
+process.start
+puts_stderr "#{Process.pid}: Started"
+
+$stdout.puts "PID: #{process.pid}"
+
+while !exit?
+    puts_stderr "#{Process.pid}: Working"
+
+    begin
+        break if !process.alive?
+
+    # If for whatever reason we can't get a status on the browser consider it
+    # dead.
+    rescue => e
+        print_exception( e )
+        break
+    end
+
+    sleep 0.03
+end
+
+puts_stderr "#{Process.pid}: EOF"

data/lib/arachni/processes/executables/rest_service.rb

@@ -0,0 +1,14 @@
+require Options.paths.root + 'ui/cli/output'
+require Options.paths.lib  + 'rest/server'
+
+Rest::Server.run!(
+    port:     Options.rpc.server_port,
+    bind:     Options.rpc.server_address,
+
+    username: Options.datastore['username'],
+    password: Options.datastore['password'],
+
+    ssl_ca:          Options.rpc.ssl_ca,
+    ssl_key:         Options.rpc.server_ssl_private_key,
+    ssl_certificate: Options.rpc.server_ssl_certificate
+)

data/lib/arachni/processes/helpers.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/processes/helpers/dispatchers.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/processes/helpers/instances.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/processes/helpers/processes.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/processes/instances.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -69,14 +69,14 @@ class Instances
     #
     # @return   [RPC::Client::Instance]
     def spawn( options = {}, &block )
-        token = options.delete(:token) || generate_token
+        token = options.delete(:token) || Utilities.generate_token
         fork  = options.delete(:fork)
 
         options = {
             spawns: options[:spawns],
             rpc:    {
                 server_socket:  options[:socket],
-                server_port:    options[:port]    || available_port,
+                server_port:    options[:port]    || Utilities.available_port,
                 server_address: options[:address] || '127.0.0.1'
             }
         }
@@ -126,7 +126,7 @@ class Instances
         options[:grid_size].times do |i|
             last_member = Dispatchers.spawn(
                 neighbour: last_member ? last_member.url : last_member,
-                pipe_id:   available_port.to_s + available_port.to_s
+                pipe_id:   Utilities.available_port.to_s + Utilities.available_port.to_s
             )
         end
 
@@ -151,7 +151,7 @@ class Instances
         options[:grid_size].times do |i|
             last_member = Dispatchers.light_spawn(
                 neighbour: last_member ? last_member.url : last_member,
-                pipe_id:   available_port.to_s + available_port.to_s
+                pipe_id:   Utilities.available_port.to_s + Utilities.available_port.to_s
             )
         end