arachnid 0.2.0 → 0.2.1

data/lib/arachnid.rb

@@ -24,7 +24,7 @@ class Arachnid
 		#defaults to 1 thread so people don't do a stupid amount of crawling on unsuspecting domains
 		threads = options[:threads] ? options[:threads] : 1
 		#defaults to -1 so it will always keep running until it runs out of urls
-		max_urls = options[:max_urls] ? options[:max_urls] : -1
+		max_urls = options[:max_urls] ? options[:max_urls] : nil
 
 		@hydra = Typhoeus::Hydra.new(:max_concurrency => threads)
 		@global_visited = BloomFilter::Native.new(:size => 1000000, :hashes => 5, :seed => 1, :bucket => 8, :raise => false)
@@ -32,13 +32,13 @@ class Arachnid
 
 		@global_queue << @start_url
 		
-		while(@global_queue.size > 0 && @global_visited.size != max_urls)
+		while(@global_queue.size > 0 && (max_urls && @global_visited.size.to_i < max_urls))
 			temp_queue = @global_queue
 
 			temp_queue.each do |q|
 
 				begin
-					request = Typhoeus::Request.new(q, :timeout => 10000)
+					request = Typhoeus::Request.new(q, :timeout => 10000, :follow_location => true)
 
 					request.on_complete do |response|
 

metadata

@@ -2,7 +2,7 @@
 name: arachnid
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors: 
 - dchuk