link_scrapper 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6117f7af82850c7d9d9072ab59837b266a4c82ae
-  data.tar.gz: c4e01ea6b7d55f4bc7e51a15f131b306bab0246d
+  metadata.gz: 98b6d54cc75862b1e063a04e355a6f638deff14b
+  data.tar.gz: fbe3a8ecfbf27d81458862ec96543700e994a049
 SHA512:
-  metadata.gz: 8de711206d0270bfb23ac104684822355069ce0609b402157454e32c4dad4bf5c833b93054cc6546c9b9d42d3295d08a37c54c9fd90d2a0283bdfdbd85e4c7da
-  data.tar.gz: 17bf223f967f0eb4573a36f5013c825181c82861382cf1f7f5caa28fdc129af9d93267d1928ffc4fd79e1a8f36f8e259f128393dabffeb381e2cc1e0661bd3d4
+  metadata.gz: 34296971fcebf4d437aba6703c36ca93a8599ee51d5ee73032e2e58217e8c25029feabb9c83c51f298fdbe5a27155196faaec2bf860e5751a228da843dcd499b
+  data.tar.gz: f3d82c1973ec70e37815ad88f35523b5edab699d0c473e7c8ce61e9387e83b1ffbf3ec5b456eb05001cc7e653282b88954790bc4136c79e24b818c8cabf05b28

data/lib/link_scrapper.rb

@@ -7,9 +7,15 @@ SEARCH_DOMAIN = 'http://virginiabeachwebdevelopment.com/'
 # class for grabbing and parsing domain links
 class LinkScrapper
 
-	def initialize(search_domain = SEARCH_DOMAIN)
+	def initialize(settings)
+
+		# available default settings
+		# domain: domain to be searched
+		# verbose: prints output as the script goes along
+		# results: hash or csv
 
 		# init link store hashes
+		@settings = settings;
 		@search_index = 0
 		@search_iteration = 0
 		@links = Array.new
@@ -20,11 +26,11 @@ class LinkScrapper
 		# gather search domain
 		if ARGV[0]
 			@search_domain = ARGV[0].dup
-		elsif search_domain == 'ue'
+		elsif @settings[:domain] == 'ue'
 			puts "Please enter a domain to search: (Default: #{SEARCH_DOMAIN})"
 			@search_domain = gets.chomp
-		elsif search_domain
-			@search_domain = search_domain
+		elsif @settings[:domain]
+			@search_domain = @settings[:domain]
 		end
 
 		# override with default domain if entry is left empty
@@ -59,12 +65,17 @@ class LinkScrapper
 				@search_uri = @links[@search_index][0].chomp
 			else
 				# save results and exit
-				save_results
+				if @settings[:results] == 'csv'
+					save_results
+				else
+					return { checked_links: @checked_links, error_links: @error_links, external_links: @external_links}
+				end
 				exit
 			end
 
 			# check for direct link
 			if @search_uri =~ /^htt(p|ps):/
+
 				# if external link go to next link
 				if @search_uri.index(@local_domain[0]) == nil
 					if !@external_links[@search_uri.to_sym]
@@ -73,11 +84,11 @@ class LinkScrapper
 							response = Net::HTTP.get_response(URI.parse(URI.encode(@search_uri)))
 							t2 = Time.now
 							delta = t2 - t1
-							rescode = response.code
+							code = response.code
 						rescue => ex
-							rescode = 408
+							code = 408
 						end
-						@external_links[@search_uri.to_sym] = {res: rescode, time: delta}
+						@external_links[@search_uri.to_sym] = {res: code, time: delta}
 					end
 					@skip = 1
 				end
@@ -93,8 +104,17 @@ class LinkScrapper
 					@skip = 1
 				else
 					# check for protocol agnostic and indirect links
-					if @search_uri[0,2] == '//' || @search_uri[0,2] == './' || @search_uri[0,3] == '../'
-						@search_uri[0,2] = ""
+					case @search_uri[0,1]
+					when '.'
+						@search_uri[0,1] = ''
+					end
+					case @search_uri[0,2]
+					when '//', './', '..'
+						@search_uri[0,2] = ''
+					end
+					case @search_uri[0,3]
+					when '../'
+						@search_uri[0,3] = ''
 					end
 					# check for relative link
 					if @search_uri[0] == '/'
@@ -104,7 +124,8 @@ class LinkScrapper
 					if @search_uri !~ /^([\w]|%|#|\?)/
 						@search_index += 1
 						@skip = 1
-						puts "invalid uri #{@search_uri}"
+						@error_links[@search_uri] = ''
+						puts "invalid uri #{@search_uri}" if @settings[:verbose]
 						return
 					end
 					# define uri string
@@ -143,7 +164,7 @@ class LinkScrapper
 		if @skip == 0
 
 			# let user know which uri is currently active
-			puts @search_uri
+			puts @search_uri if @settings[:verbose]
 
 			# gather page request response
 			begin
@@ -193,15 +214,20 @@ class LinkScrapper
 	def save_results
 		# save search results
 		CSV.open('results.csv', 'wb') {|csv|
-			@checked_links.each {|key|
-				csv << [key[0], key[1][:res], key[1][:time]]
+			@checked_links.each {|link|
+				csv << [link[0], link[1][:res], link[1][:time]]
 			}
 		}
-
 		# save list of external links
 		CSV.open('external-links.csv', 'wb') {|csv|
-			@external_links.each do |key|
-			   csv << [key[0], key[1][:res], key[1][:time]]
+			@external_links.each do |link|
+			   csv << [link[0], link[1][:res], link[1][:time]]
+			end
+		}
+		# save list of invalid links
+		CSV.open('invalid.csv', 'wb') {|csv|
+			@error_links.each do |link|
+			   csv << link
 			end
 		}
 	end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: link_scrapper
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Robert McDowell
@@ -10,7 +10,9 @@ bindir: bin
 cert_chain: []
 date: 2016-04-02 00:00:00.000000000 Z
 dependencies: []
-description: A simple gem for scrapping links within an assigned website.
+description: A simple gem for scrapping links within an assigned website. Results
+  for domain, external links, and invalid URLs can be saved as CSVs or returned as
+  a hash.
 email: doodersrage@gmail.com
 executables: []
 extensions: []