pantopoda 0.0.7 → 0.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d13ad12df8d445df6bccd998ccbf8404944919c0
-  data.tar.gz: 7a7f1a1d9f98ae5f5213979f3246e2caa4c9071c
+  metadata.gz: a0b37fa28e3130b06c5458f8e5e37695d340b5c7
+  data.tar.gz: c251c2f9d6235c29c913917f7bbf4366bfce7e5f
 SHA512:
-  metadata.gz: 40a2d0be2bacd7a562666fb1393dfa685e396790294172c56c837756df50ad97e13db1e1bd57b2b5a007d846be7ea642368a91ee54cad2fda6b0d6c53300c015
-  data.tar.gz: 6f4d4dbb16c2bb670730a2e315a6abf8d76282c87edc493b22c01658d3839894d36f69d3473b0a7c7fd8bdd55de8e925394e65a7fb7ba763495767a1d2f3ee6d
+  metadata.gz: d8a2675abf7fafe9c6e4f73bf0935d86d6f9469463d3a07b48afe34e25ea8a017a7dc239683917dae44a6f8c0c0cfa65b6b60fcec7409864096ae15450842efc
+  data.tar.gz: 20c6c6da2a3b323c94bf84e44d0584756efcc56efe8836fa7ace1939a7ee0ea26afad7eabf6c4a834e01730d5eea6b4c47fb41e441cd36c6af8427bc235cf569

data/lib/pantopoda/version.rb

@@ -1,3 +1,3 @@
 module Pantopoda
-  VERSION = "0.0.7"
+  VERSION = "0.0.8"
 end

data/lib/pantopoda.rb

@@ -14,6 +14,7 @@ module Pantopoda
 			@split_url_at_hash = options[:split_url_at_hash] ? options[:split_url_at_hash] : false
 			@exclude_urls_with_hash = options[:exclude_urls_with_hash] ? options[:exclude_urls_with_hash] : false
 			@exclude_urls_with_extensions = options[:exclude_urls_with_extensions] ? options[:exclude_urls_with_extensions] : false
+			@debug = options[:debug] ? options[:debug] : false
 		end
 
 		def crawl(options = {})
@@ -35,7 +36,7 @@ module Pantopoda
 					begin
 						ip,port,user,pass = nil
 
-						request = Typhoeus::Request.new(q, :timeout => 10000, :follow_location => true) if ip == nil
+						request = Typhoeus::Request.new(q, :timeout => 100, :follow_location => true) if ip == nil
 						request.on_complete do |response|
 							yield response
 							links = Nokogiri::HTML.parse(response.body).xpath('.//a/@href')
@@ -67,7 +68,7 @@ module Pantopoda
 		end
 
 		def parse_domain(url)
-			puts "Parsing URL: #{url}"
+			puts "Parsing URL: #{url}" if @debug
 
 			begin
 				parsed_domain = Domainatrix.parse(url)
@@ -78,7 +79,7 @@ module Pantopoda
 				end
 
 			rescue NoMethodError, Addressable::URI::InvalidURIError => e
-				puts "URL Parsing Exception (#{url}) : #{e}"
+				puts "URL Parsing Exception (#{url}) : #{e}" if @debug
 				return nil
 			end
 		end
@@ -117,7 +118,7 @@ module Pantopoda
 			@exclude_urls_with_extensions.each do |e|
 				if(url.to_s.length > e.size && url.to_s[-e.size .. -1].downcase == e.to_s.downcase)
 					not_found = false
-					puts "#{e} Found At URL: #{url}"
+					puts "#{e} Found At URL: #{url}" if @debug
 				end
 			end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pantopoda
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.8
 platform: ruby
 authors:
 - Gabriel Lim