arachni 1.1 → 1.2

data/lib/arachni/support/cache/preference.rb

@@ -28,7 +28,7 @@ class Preference < Base
     #   `v`
     def store( k, v )
         prune if capped? && (size > max_size - 1)
-        cache[k.hash] = v
+        cache[make_key( k )] = v
     end
 
     def prefer( &block )

data/lib/arachni/support/cache/random_replacement.rb

@@ -16,34 +16,10 @@ module Support::Cache
 # @author Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>
 class RandomReplacement < Base
 
-    # @see Arachni::Cache::Base#initialize
-    def initialize( * )
-        super
-        @keys = []
-    end
-
-    # @see Arachni::Cache::Base#store
-    def store( k, v )
-        already_in = include?( k )
-
-        super( k, v )
-    ensure
-        @keys << k if !already_in
-    end
-
-    def clear
-        super
-        @keys.clear
-    end
-
     private
 
-    def prune_candidate
-        @keys.delete_at( rand( size ) )
-    end
-
     def prune
-        delete( prune_candidate )
+        @cache.delete( @cache.keys.sample )
     end
 
 end

data/lib/arachni/support/database/queue.rb

@@ -80,12 +80,13 @@ class Queue < Base
     def pop( non_block = false )
         synchronize do
             loop do
-                if empty?
+                if internal_empty?
                     raise ThreadError, 'queue empty' if non_block
                     @waiting.push Thread.current
                     @mutex.sleep
                 else
-                    return @buffer.shift || load_and_delete_file( @disk.shift )
+                    return @buffer.shift if !@buffer.empty?
+                    return load_and_delete_file( @disk.shift )
                 end
             end
         end
@@ -100,6 +101,10 @@ class Queue < Base
     end
     alias :length :size
 
+    def free_buffer_size
+        max_buffer_size - buffer_size
+    end
+
     def buffer_size
         @buffer.size
     end
@@ -111,17 +116,21 @@ class Queue < Base
     # @return   [Bool]
     #   `true` if the queue if empty, `false` otherwise.
     def empty?
-        @buffer.empty? && @disk.empty?
+        synchronize do
+            internal_empty?
+        end
     end
 
     # Removes all objects from the queue.
     def clear
-        @buffer.clear
+        synchronize do
+            @buffer.clear
 
-        while !@disk.empty?
-            path = @disk.pop
-            next if !path
-            delete_file path
+            while !@disk.empty?
+                path = @disk.pop
+                next if !path
+                delete_file path
+            end
         end
     end
 
@@ -131,6 +140,10 @@ class Queue < Base
 
     private
 
+    def internal_empty?
+        @buffer.empty? && @disk.empty?
+    end
+
     def synchronize( &block )
         @mutex.synchronize( &block )
     end

data/lib/arachni/support/lookup/base.rb

@@ -83,7 +83,13 @@ class Base
     end
 
     def dup
-        deep_clone
+        self.class.new( @options.dup ).tap { |c| c.collection = @collection.dup }
+    end
+
+    protected
+
+    def collection=( c )
+        @collection = c
     end
 
     private

data/lib/arachni/support/mixins/observable.rb

@@ -24,7 +24,9 @@ module Mixins
 #
 # @author Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>
 module Observable
+    include UI::Output
     include Utilities
+
     include MonitorMixin
 
     def self.included( base )
@@ -73,7 +75,7 @@ module Observable
     def notify_observers( event, *args )
         synchronize do
             observers_for( event ).each do |block|
-                exception_jail { block.call( *args ) }
+                exception_jail( false ) { block.call( *args ) }
             end
         end
 

data/lib/arachni/support/profiler.rb

@@ -6,6 +6,7 @@
     web site for more information on licensing and terms of use.
 =end
 
+require 'objspace'
 require 'sys/proctable'
 require 'ruby-mass'
 require 'stackprof'
@@ -79,30 +80,70 @@ class Profiler
 
     def object_space( options = {} )
         klass       = options[:class]
-        namespaces  = options[:namespaces] || [Arachni]
+        namespaces  = options[:namespaces] || [
+            Arachni,
+            Ethon,
+            Typhoeus,
+            Watir,
+            Selenium,
+            Addressable,
+            Nokogiri,
+            String,
+            Hash,
+            Array,
+            Set,
+            Thread
+        ]
+
         max_entries = options[:max_entries] || 50
 
-        object_space    = Hash.new(0)
-        @object_space ||= Hash.new(0)
+        object_space    = {}
+        @object_space ||= {}
 
         ObjectSpace.each_object do |o|
             next if o.class != klass && !object_within_namespace?( o, namespaces )
-            object_space[o.class] += 1
+
+            # if o.class == Thread
+            #     ap ObjectSpace.allocation_class_path( o ).to_s
+            #     ap "#{ObjectSpace.allocation_sourcefile( o )}:#{ObjectSpace.allocation_sourceline( o )}"
+            #     ap Utilities.bytes_to_megabytes( ObjectSpace.memsize_of( o ) )
+            #     ap '-' * 120
+            # end
+
+            object_space[o.class] ||= {
+                memsize: 0,
+                count:   0
+            }
+
+            object_space[o.class][:memsize] += ObjectSpace.memsize_of(o)
+            object_space[o.class][:count]   += 1
         end
 
-        object_space = Hash[object_space.sort_by { |_, v| v }.reverse[0..max_entries]]
+        object_space = Hash[object_space.sort_by { |_, v| v[:memsize] }.reverse[0..max_entries]]
+
+        with_deltas = {}
+        object_space.each do |k, v|
+            @object_space[k] ||= {
+                memsize: 0,
+                count:   0
+            }
 
-        with_deltas = object_space.dup
-        with_deltas.each do |k, v|
-            if v.is_a? Numeric
-                with_deltas[k] = "#{v} (#{v - @object_space[k].to_i})"
+            if v[:count].is_a? Numeric
+                with_deltas[k] = "#{v[:count]} (#{v[:count] - @object_space[k][:count]})"
+                with_deltas[k] << " -- #{Utilities.bytes_to_megabytes v[:memsize]}"
+                with_deltas[k] << " (#{Utilities.bytes_to_megabytes v[:memsize] - @object_space[k][:memsize]})"
             else
-                with_deltas[k] = v
+                with_deltas[k] = "#{v[:count]} -- #{Utilities.bytes_to_megabytes v[:memsize]}"
             end
         end
 
         @object_space = object_space.dup
         with_deltas
+
+    rescue => e
+        ap e
+        ap e.backtrace
+        {}
     end
 
     def write_object_space( file, options = {} )

data/lib/arachni/support/signature.rb

@@ -14,6 +14,10 @@ module Arachni::Support
 # @author Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>
 class Signature
 
+    CACHE = {
+        tokens: Cache::LeastRecentlyPushed.new( 100 )
+    }
+
     attr_reader :tokens
 
     # @note The string will be tokenized based on whitespace.
@@ -109,14 +113,19 @@ class Signature
     #   hashes, depending on which is smaller in size.
     def tokenize( data )
         return data.tokens if data.is_a? self.class
-        compress data.split( /(?![\w])/ )
+
+        if CACHE[:tokens][data]
+            CACHE[:tokens][data].dup
+        else
+            CACHE[:tokens][data] = compress( data.split( /(?![\w])/ ) )
+        end
     end
 
     # Compresses the tokens by only storing unique #hash values.
     # Seems kinda silly but this can actually save us GB of RAM when comparing
     # large signatures, not to mention CPU cycles.
     def compress( tokens )
-        tokens.uniq.map(&:hash)
+        Set.new( tokens.map(&:hash) )
     end
 
 end

data/lib/arachni/trainer.rb

@@ -120,6 +120,7 @@ class Trainer
         # no new cookies have appeared there's no reason to analyze the page
         if incoming_page.body == @page.body && !has_new_elements &&
             @page.url == incoming_page.url
+            incoming_page.clear_cache
             print_debug 'Page hasn\'t changed.'
             return
         end
@@ -140,12 +141,17 @@ class Trainer
             end
         end
 
+        incoming_page.clear_cache
+
         print_debug 'Training complete.'
     end
 
     def has_new?( incoming_page, element_type )
-        count = ElementFilter.send( "update_#{element_type}".to_sym, incoming_page.send( element_type ) )
-        incoming_page.clear_cache
+        count = ElementFilter.send(
+            "update_#{element_type}".to_sym,
+            incoming_page.send( element_type )
+        )
+
         return if count == 0
 
         print_info "Found #{count} new #{element_type}."

data/lib/arachni/uri.rb

@@ -50,21 +50,21 @@ class URI
     end
 
     CACHE_SIZES = {
-        parse:       600,
-        ruby_parse:  600,
-        fast_parse:  600,
+        parse:       1000,
+        ruby_parse:  1000,
+        fast_parse:  1000,
+        encode:      1000,
+        decode:      1000,
         normalize:   1000,
         to_absolute: 1000
     }
 
     CACHE = {
-        parser:      ::URI::Parser.new,
-        ruby_parse:  Support::Cache::RandomReplacement.new( CACHE_SIZES[:ruby_parse] ),
-        parse:       Support::Cache::RandomReplacement.new( CACHE_SIZES[:parse] ),
-        fast_parse:  Support::Cache::RandomReplacement.new( CACHE_SIZES[:fast_parse] ),
-        normalize:   Support::Cache::RandomReplacement.new( CACHE_SIZES[:normalize] ),
-        to_absolute: Support::Cache::RandomReplacement.new( CACHE_SIZES[:to_absolute] )
+        parser: ::URI::Parser.new
     }
+    CACHE_SIZES.each do |name, size|
+        CACHE[name] = Support::Cache::LeastRecentlyPushed.new( size )
+    end
 
     class <<self
 
@@ -76,14 +76,15 @@ class URI
         # URL encodes a string.
         #
         # @param [String] string
-        # @param [String, Regexp] bad_characters
-        #   Class of characters to encode -- if {String} is passed, it should
+        # @param [String, Regexp] good_characters
+        #   Class of characters to allow -- if {String} is passed, it should
         #   formatted as a regexp (for `Regexp.new`).
         #
         # @return   [String]
         #   Encoded string.
-        def encode( string, bad_characters = nil )
-            Addressable::URI.encode_component( *[string, bad_characters].compact )
+        def encode( string, good_characters = nil )
+            CACHE[__method__][[string, good_characters]] ||=
+                Addressable::URI.encode_component( *[string, good_characters].compact )
         end
 
         # URL decodes a string.
@@ -92,7 +93,7 @@ class URI
         #
         # @return   [String]
         def decode( string )
-            Addressable::URI.unencode( string )
+            CACHE[__method__][string] ||= Addressable::URI.unencode( string )
         end
 
         # @note This method's results are cached for performance reasons.
@@ -176,8 +177,11 @@ class URI
             url = url.to_s.dup
 
             # Remove the fragment if there is one.
-            url   = url.split( '#', 2 )[0...-1].join if url.include?( '#' )
-            c_url = url.to_s.dup
+            if url.include?( '#' )
+                url = url.split( '#', 2 )[0...-1].join
+            end
+
+            c_url = url.dup
 
             components = {
                 scheme:   nil,
@@ -204,7 +208,7 @@ class URI
                     return cache[c_url] = addressable_parse( c_url ).freeze
                 end
 
-                url = url.recode
+                url = url.recode!
                 url = html_decode( url )
 
                 dupped_url = url.dup
@@ -282,9 +286,8 @@ class URI
                         !(query = dupped_url.split( '?', 2 ).last).empty?
 
                         components[:query] = (query.split( '&', -1 ).map do |pair|
-                            Addressable::URI.normalize_component( pair,
-                                Addressable::URI::CharacterClasses::QUERY.sub( '\\&', '' )
-                            )
+                            encode( decode( pair ),
+                                    Addressable::URI::CharacterClasses::QUERY.sub( '\\&', '' ) )
                         end).join( '&' )
                     end
                 end
@@ -300,7 +303,7 @@ class URI
                     print_debug "Error: #{e}"
                     print_debug_backtrace( e )
 
-                    cache[c_url] = addressable_parse( c_url.recode ).freeze
+                    cache[c_url] = addressable_parse( c_url.recode! ).freeze
                 rescue => ex
                     print_debug "Failed to parse '#{c_url}'."
                     print_debug "Error: #{ex}"
@@ -405,8 +408,8 @@ class URI
 
             cache = CACHE[__method__]
 
-            url   = url.to_s.strip.dup
-            c_url = url.to_s.strip.dup
+            url   = url.to_s.strip
+            c_url = url.dup
 
             begin
                 if (v = cache[url]) && v == :err
@@ -494,13 +497,13 @@ class URI
                               self.class.ruby_parse( url )
 
                           when ::URI
-                              url.dup
+                              url
 
                           when Hash
                               ::URI::Generic.build( url )
 
                           when Arachni::URI
-                              self.parsed_url = url.parsed_url.dup
+                              self.parsed_url = url.parsed_url
 
                           else
                               to_string = url.to_s rescue ''

data/lib/arachni/uri/scope.rb

@@ -80,7 +80,7 @@ class Scope < Arachni::Scope
 
         check_scheme = @url.scheme.to_s
 
-        return false if !%(http https).include?( check_scheme )
+        return false if check_scheme != 'http' && check_scheme != 'https'
 
         parsed_ref = Arachni::URI( Options.url )
         return false if !parsed_ref

data/lib/arachni/utilities.rb

@@ -391,6 +391,14 @@ module Utilities
         0
     end
 
+    def bytes_to_megabytes( bytes )
+        (bytes / 1024.0 / 1024.0).round( 3 )
+    end
+
+    def bytes_to_kilobytes( bytes )
+        (bytes / 1024.0 ).round( 3 )
+    end
+
     # Wraps the `block` in exception handling code and runs it.
     #
     # @param    [Bool]  raise_exception