arachni 0.4.2 → 0.4.3

data/lib/arachni/rpc/server/spider.rb

@@ -20,18 +20,18 @@ module RPC
 class Server
 
 #
-#
-# Extends the regular {Arachni::Spider} with high-performance distributed capabilities.
+# Extends the regular {Arachni::Spider} with high-performance distributed
+# capabilities.
 #
 # @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
 #
 class Spider < Arachni::Spider
 
     # Amount of URLs to buffer before distributing.
-    BUFFER_SIZE     = 200
+    BUFFER_SIZE     = 1000
 
     # How many times to try and fill the buffer before distributing what's in it.
-    FILLUP_ATTEMPTS = 30
+    FILLUP_ATTEMPTS = 200
 
     private :push, :done?, :sitemap, :running?
     public  :push, :done?, :sitemap, :running?
@@ -43,23 +43,26 @@ class Spider < Arachni::Spider
         @peers        = {}
         @done_signals = Hash.new( true )
 
-        @distribution_filter   = BloomFilter.new
+        @distribution_filter   = Support::LookUp::Moolb.new
 
         @after_each_run_blocks = []
         @on_first_run_blocks   = []
     end
 
-    # @param    [Block] block   Block to be called after each URL batch has been exhausted.
+    # @param    [Block] block
+    #   Block to be called after each URL batch has been consumed.
     def after_each_run( &block )
         @after_each_run_blocks << block
     end
 
+    # @param    [Block] block
+    #   Block to be called just before the crawl starts.
     def on_first_run( &block )
         @on_first_run_blocks << block
     end
 
     # @see Arachgni::Spider#run
-    def run( *args, &block )
+    def run( *args )
         @first_run_blocks ||= call_on_first_run
 
         if !solo?
@@ -67,7 +70,7 @@ class Spider < Arachni::Spider
             @on_complete_blocks.clear
         end
 
-        super( *args, &block )
+        super( *args )
 
         flush_url_distribution_buffer
         master_done_handler
@@ -83,6 +86,16 @@ class Spider < Arachni::Spider
         sitemap
     end
 
+    #
+    # Updates the list of Instances to assist in the crawl.
+    #
+    # @param    [Array<Hash>]  peers
+    #   Array containing Instance info hashes -- with `:url` and `:token`
+    #   at least.
+    #
+    # @param    [Block] block
+    #   Block to be called once the update operation has completed.
+    #
     def update_peers( peers, &block )
         @peers_array = peers
         sorted_peers = @peers_array.inject( {} ) do |h, p|
@@ -92,10 +105,13 @@ class Spider < Arachni::Spider
 
         @peers = Hash[sorted_peers]
 
-        @peers[framework.self_url] = framework
+        @peers[framework.multi_self_url] = framework
 
         @peers = Hash[@peers.sort]
 
+        @peer_urls    = @peers.keys
+        @peer_clients = @peers.values
+
         if !master?
             block.call if block_given?
             return true
@@ -112,10 +128,67 @@ class Spider < Arachni::Spider
         true
     end
 
+    # @return   [Hash<String, Integer>]
+    #   URLs crawled by this Instance, along with their HTTP status codes.
+    def local_sitemap
+        @sitemap
+    end
+
+    # @return   [Array<String>] Crawled URLs.
     def sitemap
         @distributed_sitemap || super
     end
 
+    #
+    # Sets a peer crawler's state to finished. Exposed so that peers can signal
+    # the master once they're done.
+    #
+    # @param    [String]    url URL of the finished peer.
+    #
+    def peer_done( url )
+        @done_signals[url] = true
+        master_done_handler
+        true
+    end
+
+    #
+    # Signals the `master` Instance that this crawler has finished.
+    #
+    # @param    [Arachni::RPC::Client::Instance]    master
+    #
+    def signal_if_done( master )
+        return if !done?
+        master.spider.peer_done( framework.multi_self_url ){}
+    end
+
+    private
+
+    # @return   [RPC::Client::Instance] Peer to handle the given item.
+    def route_item_to_client( item )
+        @peers[route_item_to_url( item )]
+    end
+
+    # @return   [RPC::Client::Instance]
+    #   URL of the peer to handle the given item.
+    def route_item_to_url( item )
+        return peer_urls.first if @peers.size == 1
+
+        peer_urls[item.to_s.persistent_hash.modulo( peer_urls.size )]
+    end
+
+    # @return   [Array<String>] Sorted peer URLs.
+    def peer_urls
+        @peer_urls
+    end
+
+    # @return   [Array<Arachni::RPC::Client::Instance>] Sorted peer clients.
+    def peer_clients
+        @peer_clients
+    end
+
+    # Collects sitemaps from all peers.
+    #
+    # @param    [Block] block Block to be passed the merged sitemap.
     def collect_sitemaps( &block )
         local_sitemap = sitemap
 
@@ -132,18 +205,6 @@ class Spider < Arachni::Spider
         map_peers( foreach, after )
     end
 
-    def peer_done( url )
-        @done_signals[url] = true
-        master_done_handler
-        true
-    end
-
-    def signal_if_done( master )
-        master.spider.peer_done( framework.self_url ){} if done?
-    end
-
-    private
-
     def call_on_first_run
         @on_first_run_blocks.each( &:call )
     end
@@ -152,29 +213,54 @@ class Spider < Arachni::Spider
         @after_each_run_blocks.each( &:call )
     end
 
+    # @param    [String]    url    URL of the peer to set as not done.
     def peer_not_done( url )
         @done_signals[url] = false
         true
     end
 
+    #
+    # Checks whether or not the master and its slaves have finished.
+    # If so, it collects the sitemaps and calls `on_complete` callbacks.
+    #
+    # @return   [Boolean]
+    #   `true` if all peers were done and it proceeded to processing the
+    #   results, `false` if the crawl is still in progress.
+    #
     def master_done_handler
-        return if !master? || !done? || !slaves_done?
+        # Once we realize that we're done for the first time then that's it.
+        # Ignore residual signals from slaves to avoid calling the #on_complete
+        # callbacks more than once.
+        @all_done ||= false
+        return if @all_done || !master? || !done? || !slaves_done?
 
-        # really make sure all slaves are done
+        # Really make sure all slaves are done.
         if_slaves_done do
+            @all_done = true
+
             collect_sitemaps do |aggregate_sitemap|
                 @distributed_sitemap = aggregate_sitemap
                 call_on_complete_blocks
             end
         end
+
+        true
     end
 
+    # @param  [Block]   block
+    #   Block to call if all slaves have finished crawling.
     def if_slaves_done( &block )
         each  = proc { |peer, iter| peer.spider.running? { |b| iter.return !!b } }
         after = proc { |results| block.call if !results.include?( true )}
         map_peers( each, after )
     end
 
+    # @note When checking for slave status also use {#if_slaves_done} to
+    #   make sure that there weren't any new paths pushed to the in the interim.
+    #
+    # @return   [Boolean]
+    #   `true` if all slaves have signaled that they've finished, `false`
+    #   otherwise.
     def slaves_done?
         !@peers.reject{ |url, _| url == self_instance_info[:url] }.keys.
             map { |peer_url| @done_signals[peer_url] }.include?( false )
@@ -194,21 +280,24 @@ class Spider < Arachni::Spider
 
     def self_instance_info
         {
-            url:   framework.self_url,
+            url:   framework.multi_self_url,
             token: framework.token
         }
     end
 
     #
-    # Distributes the paths to the peers
+    # @note Paths are buffered in order to be pushed in batches for better
+    #   bandwidth utilization and to keep RPC calls to a minimum
     #
-    # @param    [Array<String>]  urls    to distribute
+    # Distributes the paths to the peers.
+    #
+    # @param    [Array<String>]  urls    URLs to distribute.
     #
     def distribute( urls )
         urls = dedup( urls )
         return false if urls.empty?
 
-        @first_run ||= Arachni::BloomFilter.new
+        @first_run ||= Support::LookUp::HashSet.new
 
         @routed          ||= {}
         @buffer_size     ||= 0
@@ -223,19 +312,20 @@ class Spider < Arachni::Spider
 
         return if @buffer_size == 0
 
-        # remove and push our URLs right way
+        # Remove and push our URLs right way.
         push( @routed.delete( framework ) )
 
         @fillup_attempts += 1
 
         return if @buffer_size < BUFFER_SIZE && @fillup_attempts < FILLUP_ATTEMPTS
 
-        # distribute the buffered outgoing URLs
+        # Distribute the buffered outgoing URLs.
         flush_url_distribution_buffer
 
         true
     end
 
+    # Sends the buffered paths to their assigned peer and empties the buffer.
     def flush_url_distribution_buffer
         @routed ||= {}
         @routed.dup.each do |peer, r_urls|
@@ -250,20 +340,36 @@ class Spider < Arachni::Spider
             end
         end
 
-        # clear the counters and the buffer
+        # Clear the counters and the buffer.
         @fillup_attempts = 0
         @buffer_size     = 0
         @routed.clear
     end
 
+    # @param    [String]    url
+    #
+    # @return   [Boolean]
+    #   `true` if the `url` has already been distributed, `false` otherwise.
+    #
+    # @see #distributed
     def distributed?( url )
         @distribution_filter.include? url
     end
 
+    # @param    [String]    url URL to set a having been distributed.
+    # @see #distributed?
     def distributed( url )
         @distribution_filter << url
     end
 
+    # @param    [String]    url URL to route.
+    # @return   [RPC::Client::Instance] Peer to handle the `url`.
+    def route( url )
+        return if !url || url.empty?
+        return framework if @peers.empty?
+        route_item_to_client( url )
+    end
+
     def map_peers( foreach, after )
         wrap = proc do |instance, iterator|
             foreach.call( instance, iterator )
@@ -285,14 +391,6 @@ class Spider < Arachni::Spider
         )
     end
 
-    def route( url )
-        return if !url || url.empty?
-        return framework if @peers.empty?
-        return @peers.values.first if @peers.size == 1
-
-        @peers.values[url.bytes.inject( :+ ).modulo( @peers.size )]
-    end
-
     def framework
         @framework
     end

data/lib/arachni/ruby.rb

@@ -1,6 +1,25 @@
+=begin
+    Copyright 2010-2013 Tasos Laskos <tasos.laskos@gmail.com>
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+=end
+
 lib = Arachni::Options.dir['lib']
 require lib + 'ruby/object'
+require lib + 'ruby/hash'
+require lib + 'ruby/set'
 require lib + 'ruby/enumerable'
 require lib + 'ruby/array'
 require lib + 'ruby/string'
+require lib + 'ruby/io'
 require lib + 'ruby/webrick'

data/lib/arachni/ruby/hash.rb

@@ -0,0 +1,62 @@
+=begin
+    Copyright 2010-2013 Tasos Laskos <tasos.laskos@gmail.com>
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+=end
+
+class Hash
+
+    #
+    # Converts the hash keys to strings.
+    #
+    # @param    [Boolean]    recursively    Go through the Hash recursively?
+    #
+    # @return [Hash]
+    #   Hash with +self+'s keys recursively converted to strings.
+    #
+    def stringify_keys( recursively = true )
+        stringified = {}
+        each do |k, v|
+            stringified[k.to_s] = (recursively && v.is_a?( Hash ) ? v.stringify_keys : v)
+        end
+        stringified
+    end
+
+    #
+    # Converts the hash keys to symbols.
+    #
+    # @param    [Boolean]    recursively    Go through the Hash recursively?
+    #
+    # @return [Hash]
+    #   Hash with +self+'s keys recursively converted to symbols.
+    #
+    def symbolize_keys( recursively = true )
+        symbolize = {}
+        each do |k, v|
+            symbolize[k.to_s.to_sym] = (recursively && v.is_a?( Hash ) ? v.symbolize_keys : v)
+        end
+        symbolize
+    end
+
+    # @return   [Hash]
+    #   Self with the keys and values converted to lower-case strings.
+    def downcase
+        stringify_keys.inject({}) do |h, (k, v)|
+            k = k.downcase if k.is_a?( String )
+            v = v.downcase if v.is_a?( String )
+            h[k] = v
+            h
+        end
+    end
+
+end

data/lib/arachni/ruby/io.rb

@@ -0,0 +1,49 @@
+=begin
+    Copyright 2010-2013 Tasos Laskos <tasos.laskos@gmail.com>
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+=end
+
+class IO
+    TAIL_BUF_LENGTH = 1 << 16
+
+    #
+    # @param    [Integer]   n
+    #   Amount of lines to return from the bottom of the file.
+    #
+    # @return   [Array<String>]
+    #   `n` amount of lines from the bottom of the file.
+    #
+    # @see http://stackoverflow.com/questions/3024372/how-to-read-a-file-from-bottom-to-top-in-ruby/3024704#3024704
+    #
+    def tail( n )
+        return [] if n < 1
+
+        seek_to = TAIL_BUF_LENGTH
+        seek_to = size if seek_to > size
+
+        seek -seek_to, IO::SEEK_END
+
+        buf = ''
+        while buf.count( "\n" ) <= n
+            buf = read( seek_to ) + buf
+
+            seek_to *= 2
+            seek_to = size if seek_to > size
+
+            seek -seek_to, IO::SEEK_CUR
+        end
+
+        buf.split( "\n" )[-n..-1]
+    end
+end