link_scrapper 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 213970214a5f8e494141c4172373f5ae21190714
-  data.tar.gz: 0ba291b9a887f6fc244ebd25939b342f77c61b46
+  metadata.gz: b6cb0bc3b1eacf9d8fdff2f6ee057bc411c6da03
+  data.tar.gz: 53c1f9da27e968db305e5fbaae3d3e4480905e90
 SHA512:
-  metadata.gz: 579ab33563beb450bf54e78c88b2b19b4d034a87acb722224b91ddb15e075a527ef05fd2a6dfd7b5a69dac464f616bba161a4dedaaff14285dadfbb8a04e0717
-  data.tar.gz: a7180512da166efd1f9322199e69d454128b159610306289a6575220852a678991fee7922b29fe9fb7ce26833e1ce7178121475d786cd3fdd0a442dec2a87469
+  metadata.gz: 5aa5eac8873d6275b298beaf2663441a530f57389a42794250e137084d1ad2d98705f5ab83db11c0fc71c66e14899e27f066c39985ade3d50486a39a290c5f66
+  data.tar.gz: 821fe2bfba46ed516be52a7803e15a78a2e00e4a656d07e787b1830d4d2b0089b85901a51945505fd719649094a5d06a12aeefd53fb9224b8570fec8931d7055

data/lib/link_scrapper.rb

@@ -82,7 +82,7 @@ class LinkScrapper
 					if !@external_links[@search_uri.to_sym]
 						begin
 							t1 = Time.now
-							response = Net::HTTP.get_response(URI.parse(URI.encode(@search_uri)))
+							response = Net::HTTP.get_response(URI.parse(@search_uri))
 							t2 = Time.now
 							delta = t2 - t1
 							code = response.code
@@ -170,7 +170,7 @@ class LinkScrapper
 			# gather page request response
 			begin
 				t1 = Time.now
-				response = Net::HTTP.get_response(URI.parse(URI.encode(@search_uri.strip)))
+				response = Net::HTTP.get_response(URI.parse(@search_uri.strip))
 				t2 = Time.now
 				delta = t2 - t1
 
@@ -184,7 +184,8 @@ class LinkScrapper
 				links_array = body.scan(/<a[^>]+href\s*=\s*["']([^"']+)["'][^>]*>(.*?)<\/a>/mi)
 
 				# update anchors and indirect links to use direct links
-				links_array.each { |val|
+				links_array.each_with_index { |val, index|
+					skip = 0
 					if (val[0][0,2] == "//" || val[0][0] == "/" || val[0][0,3] == "../") && val[0] !~ /^htt(p|ps):/
 						if val[0][0,3] == "../"
 							val[0][0,3] = ""
@@ -197,7 +198,16 @@ class LinkScrapper
 						end
 						val[0] = "#{@search_domain}#{val[0]}"
 					end
-					@link_parents[val[0].chomp.to_sym] = @search_uri.strip
+					@links.each { |lnk|
+						if val[0] == lnk[0]
+							skip = 1
+						end
+					}
+					if skip == 0
+						@link_parents[val[0].chomp.to_sym] = @search_uri.strip
+					else
+						val.delete_at(index)
+					end
 				}
 
 				# combine found links with links array

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: link_scrapper
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Robert McDowell