link_scrapper 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 98b6d54cc75862b1e063a04e355a6f638deff14b
-  data.tar.gz: fbe3a8ecfbf27d81458862ec96543700e994a049
+  metadata.gz: 213970214a5f8e494141c4172373f5ae21190714
+  data.tar.gz: 0ba291b9a887f6fc244ebd25939b342f77c61b46
 SHA512:
-  metadata.gz: 34296971fcebf4d437aba6703c36ca93a8599ee51d5ee73032e2e58217e8c25029feabb9c83c51f298fdbe5a27155196faaec2bf860e5751a228da843dcd499b
-  data.tar.gz: f3d82c1973ec70e37815ad88f35523b5edab699d0c473e7c8ce61e9387e83b1ffbf3ec5b456eb05001cc7e653282b88954790bc4136c79e24b818c8cabf05b28
+  metadata.gz: 579ab33563beb450bf54e78c88b2b19b4d034a87acb722224b91ddb15e075a527ef05fd2a6dfd7b5a69dac464f616bba161a4dedaaff14285dadfbb8a04e0717
+  data.tar.gz: a7180512da166efd1f9322199e69d454128b159610306289a6575220852a678991fee7922b29fe9fb7ce26833e1ce7178121475d786cd3fdd0a442dec2a87469

data/lib/link_scrapper.rb

@@ -19,6 +19,7 @@ class LinkScrapper
 		@search_index = 0
 		@search_iteration = 0
 		@links = Array.new
+		@link_parents = Hash.new
 		@checked_links = Hash.new
 		@error_links = Hash.new
 		@external_links = Hash.new
@@ -184,9 +185,19 @@ class LinkScrapper
 
 				# update anchors and indirect links to use direct links
 				links_array.each { |val|
-					if val[0] != '/' || val !~ /^htt(p|ps):/ || val[0,2] != '//'
-						val = "#{@search_uri}#{val}"
+					if (val[0][0,2] == "//" || val[0][0] == "/" || val[0][0,3] == "../") && val[0] !~ /^htt(p|ps):/
+						if val[0][0,3] == "../"
+							val[0][0,3] = ""
+						end
+						if val[0][0,2] == "//"
+							val[0][0,2] = ""
+						end
+						if val[0][0] == "/"
+							val[0][0] = ""
+						end
+						val[0] = "#{@search_domain}#{val[0]}"
 					end
+					@link_parents[val[0].chomp.to_sym] = @search_uri.strip
 				}
 
 				# combine found links with links array
@@ -200,7 +211,7 @@ class LinkScrapper
 			end
 
 			# store results in checked hash
-			@checked_links[@search_uri.to_sym] = {res: code, time: delta}
+			@checked_links[@search_uri.to_sym] = {res: code, time: delta, parent: @link_parents[@search_uri.to_sym]}
 
 		end
 
@@ -215,13 +226,13 @@ class LinkScrapper
 		# save search results
 		CSV.open('results.csv', 'wb') {|csv|
 			@checked_links.each {|link|
-				csv << [link[0], link[1][:res], link[1][:time]]
+				csv << [link[0], link[1][:res], link[1][:time], link[1][:parent]]
 			}
 		}
 		# save list of external links
 		CSV.open('external-links.csv', 'wb') {|csv|
 			@external_links.each do |link|
-			   csv << [link[0], link[1][:res], link[1][:time]]
+			   csv << [link[0], link[1][:res], link[1][:time], link[1][:parent]]
 			end
 		}
 		# save list of invalid links

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: link_scrapper
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Robert McDowell
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-02 00:00:00.000000000 Z
+date: 2017-02-24 00:00:00.000000000 Z
 dependencies: []
 description: A simple gem for scrapping links within an assigned website. Results
   for domain, external links, and invalid URLs can be saved as CSVs or returned as
@@ -19,7 +19,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - lib/link_scrapper.rb
-homepage: http://virginiabeachwebdevelopment.com
+homepage: https://github.com/Studio-Center/Ruby-Link-Extractor
 licenses:
 - MIT
 metadata: {}