find_dead_link 0.0.1

data/lib/find_dead_link.rb

@@ -0,0 +1,16 @@
+require 'semantic_logger' unless defined?(logger)
+require 'nokogiri'
+require 'open-uri'
+require 'pry'
+
+SemanticLogger.default_level = :info
+SemanticLogger.add_appender('find_dead_link.log')
+
+module FindDeadLink
+  GEM_ROOT = File.join(File.dirname(__FILE__), '..')
+end
+
+require_relative './find_dead_link/crawler'
+require_relative './find_dead_link/html_doc'
+require_relative './find_dead_link/url_opener'
+require_relative './find_dead_link/version'

data/lib/find_dead_link/bad_url_exception.rb

@@ -0,0 +1,3 @@
+module FindDeadLink
+ class BadUrlException < StandardError;end
+end

data/lib/find_dead_link/crawler.rb

@@ -0,0 +1,41 @@
+module FindDeadLink
+	class Crawler
+		include SemanticLogger::Loggable
+		def initialize(url)
+			raise BadUrlException unless (URI(url) rescue false)
+			@base_url = url
+			@host = URI(url).host
+			@deadlinks = []
+			@visited_links = []
+		end
+
+		def crawl
+			visit(@base_url)
+			logger.info("you have Deadlinks url #{@deadlinks}")
+		end
+
+	private
+		def visit(url)
+			logger.info("Visiting url #{url}")
+			return if visited_url?(url) or external_url?(url)
+			@visited_links << url
+			url_opener = UrlOpener.new(url)
+			return (@deadlinks << url) if url_opener.dead_link?
+			links = HtmlDoc.new(url_opener.get_content).get_links
+			links.each{|url| visit(get_url(url)) }
+		end
+
+		def visited_url?(url)
+			@visited_links.include?(url)
+		end
+
+		def get_url(url)
+			return url if url.start_with?('http') or url.start_with?('https')
+			URI.join(@base_url, url).to_s
+		end
+
+		def external_url?(url)
+			!(@host == URI(url).host rescue false)
+		end
+	end
+end

data/lib/find_dead_link/html_doc.rb

@@ -0,0 +1,21 @@
+module FindDeadLink
+	class HtmlDoc
+		IGNORE_URLS_START_WITH = ['#', 'javascript', 'mailto']
+
+		def initialize(html_content)
+			@html_content = html_content
+		end
+
+		def get_links
+			@html_content.css('a').collect do |link| 
+				link["href"] unless ignore?(link["href"])
+			end.compact
+		end
+
+		def ignore?(url)
+			return true if url.nil?
+			IGNORE_URLS_START_WITH.each{|element|	return true if url.start_with?(element) }
+			false
+		end
+	end
+end

data/lib/find_dead_link/url_opener.rb

@@ -0,0 +1,28 @@
+require 'net/http'
+module FindDeadLink
+  class UrlOpener
+    include SemanticLogger::Loggable
+
+    def initialize(url)
+      @url = url
+    end
+
+    def dead_link?
+      begin
+        (Net::HTTP.get_response(URI(@url)).code) == "404"
+      rescue StandardError => e
+        logger.error "DEADLINK :: Error in opening URL:: #{@url}", error: e.inspect
+        return true
+      end
+    end
+
+    def get_content
+      begin
+        Nokogiri::HTML(open(@url))
+      rescue StandardError => e
+        logger.error "Error in parsing content for URL:: #{@url}", error: e.inspect
+        Nokogiri::HTML("")
+      end
+    end
+  end
+end

data/lib/find_dead_link/version.rb

@@ -0,0 +1,3 @@
+module FindDeadLink
+	VERSION = "0.0.1"
+end

metadata

@@ -0,0 +1,98 @@
+--- !ruby/object:Gem::Specification
+name: find_dead_link
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Ankur Maheshwari
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-04-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.1
+- !ruby/object:Gem::Dependency
+  name: semantic_logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.7.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.7.0
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: The library takes a url and search for deadlinks within given website
+email:
+- amaheshwari@systango.com
+executables:
+- find_dead_link.rb
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- Gemfile.lock
+- README.md
+- bin/find_dead_link.rb
+- find_dead_link.gemspec
+- find_dead_link.log
+- lib/find_dead_link.rb
+- lib/find_dead_link/bad_url_exception.rb
+- lib/find_dead_link/crawler.rb
+- lib/find_dead_link/html_doc.rb
+- lib/find_dead_link/url_opener.rb
+- lib/find_dead_link/version.rb
+homepage: http://systango.com/
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Looks for deadlinks exists in provided url
+test_files: []