ruby-web-crawler 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 16e50a5b1d4eb91ce797f9ce98467c445a35d1ae
+  data.tar.gz: 0f8e52dafc5fce1e23294bf5cb6cbd64bef8c143
+SHA512:
+  metadata.gz: 345c699c94597ff0da7cbc1225f3513d19fc0c2ca675ce3afc957d67b7f39e4872a9940816ec85152323d81c2773dde499e281dc42a0d40e0a210b443774621b
+  data.tar.gz: dd4bbd7d6f18e4fbbb83c2c2d8691067609c9d7e9bf7389f63c085f2b408e18e77dd157fa2a1decd5293cd8f6a580669f3cdc5d2506b8a208c5c9f41b469ea88

data/lib/ruby-web-crawler.rb

@@ -0,0 +1,70 @@
+require 'net/http'
+require 'uri'
+
+class RubyWebCrawler
+	
+	attr_accessor :urls, :root_url, :url_limit, :time_limit
+
+	def initialize url, url_limit = 50, time_limit = 60
+		self.urls = []
+		self.root_url = url 			# Root URL from where the Crawling Starts
+		self.url_limit = url_limit 		# Default URL Limit for the Crawler
+		self.time_limit = time_limit	# Timeout limit in seconds
+	end
+
+	def start_crawl
+		begin
+			is_running = Timeout::timeout(self.time_limit) {
+				self.get_urls_for_page self.root_url	
+			}	
+		rescue Exception => e
+			# Do Nothing just don't let it error out
+		end
+		return self.urls
+	end
+
+	# Get all URLs on a Page
+	def get_urls_for_page url
+		page_content = self.get_page_content url
+		
+		# Regex to get all "links" in the page
+		urls = page_content.scan(/\<a href\=(\"(http|https)\:.*?\")/)			
+		urls.each { |u| 
+			sanitized_url = u.first.gsub(/\"/, '').strip
+			unless self.urls.include? sanitized_url
+				self.urls.push(sanitized_url)
+
+				# If Unexpected Error happens when trying to fetch URLs move on to the next URL
+				begin
+					break if self.urls.count >= self.url_limit
+					self.get_urls_for_page(sanitized_url)	
+				rescue Exception => e
+					next
+				end
+			end
+		}
+		return self.urls
+	end
+
+	# Get HTML/Content of the Page to be parsed
+	def get_page_content url
+		uri = URI(url)
+		request = Net::HTTP::Get.new(uri)
+
+		http = Net::HTTP.new(uri.host, uri.port)
+		
+		# Neet to enable use of SSL if the URL protocol is HTTPS
+		http.use_ssl = (uri.scheme == "https")
+
+		response = http.request(request)
+
+		# Check if URL needs to be forwarded because of redirect
+		case response
+		when Net::HTTPSuccess
+			return response.body
+		when Net::HTTPMovedPermanently || Net::HTTPRedirection
+			self.get_page_content response['location']
+		end
+	end
+	
+end # END RubyWebCrawler

metadata

@@ -0,0 +1,73 @@
+--- !ruby/object:Gem::Specification
+name: ruby-web-crawler
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Dwijen Purohit
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-06-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: net/http
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: uri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A simple ruby gem to recursively traverse all URLs on a Root URL. It
+  returns all the URLs it encountered
+email: dwijen4190@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/ruby-web-crawler.rb
+homepage: http://rubygems.org/gems/ruby-web-crawler
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5.1
+signing_key: 
+specification_version: 4
+summary: Gem to crawl from a root URL to all child URLs
+test_files: []