rubyretriever 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cdb3cd79480f748af792189d712aa1b6cb94bb83
-  data.tar.gz: f605cdf36b00ca8e1ad337922e8fa56417e5aa2c
+  metadata.gz: f27793b1294bd489c3338aa5bc739dc6058d479a
+  data.tar.gz: e48491765a741087ba7bf708d62a8bbf8aa3cf80
 SHA512:
-  metadata.gz: 69effd60463fb7a9069761102f007e944630844f3367f89f17f54804da8f31f0d2143119fd3819b6fcb95dfe829c602c9ff42c7687d2f410b285d78eae980f9d
-  data.tar.gz: bda8c8f896720495fda0ac34deaeaae7c74df72ae384a8de939fb601cc0958fa03296cbf21cc7b0b8df7ed056c5fae4055cc203bf0edb15773fbf88d9f3e9d61
+  metadata.gz: 60c017cf5dda8c659b12c3146f565df743dc4f6cf0e8436889bebb86cdccd5b128cca233d76b1413f526c21d1d99478c8149e6c43772e5dd0db1067f8dea5263
+  data.tar.gz: ec8142f3a8cbd75861c74322dd0358da51ee45388376ca0118d0998b81cf302b182eef93c39d88ef17dd35ef80a6253bf7a058e2e2052ea60ea3266d401892ab

data/lib/retriever/fetch.rb

@@ -155,7 +155,7 @@ module Retriever
 				@linkStack.concat(new_links_arr)
 				@sitemap.concat(new_links_arr) if @s
 			end
-			@progressbar.finish
+			@progressbar.finish if @prgrss
 		end
 		def asyncGetWave() #send a new wave of GET requests, using current @linkStack
 			new_stuff = []
@@ -167,11 +167,10 @@ module Retriever
 			    	if @already_crawled.include?(url)
 			    		@linkStack.delete(url)
 			    		next
-			    	else
-			    		@already_crawled.insert(url)
 			    	end
 			    	resp = EventMachine::HttpRequest.new(url).get
 					lg("URL Crawled: #{url}")
+			    	@already_crawled.insert(url)
 					if @prgrss
 						@progressbar.increment if @already_crawled.size < @maxPages
 					end

data/lib/retriever/version.rb

@@ -1,3 +1,3 @@
 module Retriever
-  VERSION = '0.1.2'
+  VERSION = '0.1.3'
 end

data/readme.md

@@ -45,7 +45,7 @@ command-line arguments
 Usage: rr [MODE FLAG] [OPTIONS] Target_URL  
 
 Where MODE FLAG is required, and is either:  
-	-s, --sitemap FORMAT  
+	-s, --sitemap FORMAT  (only accepts CSV or XML atm)  
 	-f, --files FILETYPE  
   
 and OPTIONS is the applicable:  

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rubyretriever
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Joe Norton