arachnid 0.3.0 → 0.3.1

data/lib/arachnid.rb

@@ -15,6 +15,7 @@ class Arachnid
 		@split_url_at_hash = options[:split_url_at_hash] ? options[:split_url_at_hash] : false
 		@exclude_urls_with_hash = options[:exclude_urls_with_hash] ? options[:exclude_urls_with_hash] : false
 		@exclude_urls_with_images = options[:exclude_urls_with_images] ? options[:exclude_urls_with_images] : false
+		@proxy_list = options[:proxy_list] ? options[:proxy_list] : nil
 		
 		@debug = options[:debug] ? options[:debug] : false
 	end
@@ -26,6 +27,8 @@ class Arachnid
 		#defaults to -1 so it will always keep running until it runs out of urls
 		max_urls = options[:max_urls] ? options[:max_urls] : nil
 
+
+
 		@hydra = Typhoeus::Hydra.new(:max_concurrency => threads)
 		@global_visited = BloomFilter::Native.new(:size => 1000000, :hashes => 5, :seed => 1, :bucket => 8, :raise => false)
 		@global_queue = []
@@ -38,7 +41,11 @@ class Arachnid
 			temp_queue.each do |q|
 
 				begin
-					request = Typhoeus::Request.new(q, :timeout => 10000, :follow_location => true)
+					ip,port,user,pass = grab_proxy
+
+					request = Typhoeus::Request.new(q, :timeout => 10000, :follow_location => true) if ip == nil
+					request = Typhoeus::Request.new(q, :timeout => 10000, :follow_location => true, :proxy => "#{ip}:#{port}") if ip != nil && user == nil
+					request = Typhoeus::Request.new(q, :timeout => 10000, :follow_location => true, :proxy => "#{ip}:#{port}", :proxy_username => user, :proxy_password => pass) if user != nil
 
 					request.on_complete do |response|
 
@@ -95,6 +102,20 @@ class Arachnid
 		end
 	end
 
+	def internal_link?(url, effective_url)
+
+		absolute_url = make_absolute(url, effective_url)
+
+		parsed_url = parse_domain(absolute_url)
+
+	def grab_proxy
+
+		return nil unless @proxy_list
+
+		return @proxy_list.sample.split(':')
+
+	end
+
 	def internal_link?(url, effective_url)
 
 		absolute_url = make_absolute(url, effective_url)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: arachnid
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
   prerelease: 
 platform: ruby
 authors: