apollo-crawler 0.1.4 → 0.1.5

data/lib/apollo_crawler.rb

@@ -1,20 +1,20 @@
-# Main
-require 'apollo_crawler/cache'
-require 'apollo_crawler/crawler'
-require 'apollo_crawler/formatter'
-
-# Caches
-require 'apollo_crawler/caches/factory'
-require 'apollo_crawler/caches/memory_cache'
-require 'apollo_crawler/caches/null_cache'
-
-# Crawlers
-require 'apollo_crawler/crawlers/alexa_com/alexa'
-require 'apollo_crawler/crawlers/firmy_cz/firmy'
-require 'apollo_crawler/crawlers/slashdot_org/slashdot'
-require 'apollo_crawler/crawlers/ycombinator_com/hacker_news'
-
-# Formatters
-require 'apollo_crawler/formatters/formatter_json'
-require 'apollo_crawler/formatters/formatter_plain'
-require 'apollo_crawler/formatters/formatter_table'
+# Main
+require 'apollo_crawler/cache'
+require 'apollo_crawler/crawler'
+require 'apollo_crawler/formatter'
+
+# Caches
+require 'apollo_crawler/caches/factory'
+require 'apollo_crawler/caches/memory_cache'
+require 'apollo_crawler/caches/null_cache'
+
+# Crawlers
+require 'apollo_crawler/crawlers/alexa_com/alexa'
+require 'apollo_crawler/crawlers/firmy_cz/firmy'
+require 'apollo_crawler/crawlers/slashdot_org/slashdot'
+require 'apollo_crawler/crawlers/ycombinator_com/hacker_news'
+
+# Formatters
+require 'apollo_crawler/formatters/formatter_json'
+require 'apollo_crawler/formatters/formatter_plain'
+require 'apollo_crawler/formatters/formatter_table'

data/lib/apollo_crawler/cache.rb

@@ -1,34 +1,34 @@
-module Apollo
-	module Caches
-		class Cache
-			# Get value associated with key from cache
-			def get(key, *args)
-
-				# Not found, Create, cache and return
-				res = yield args
-				return res
-			end
-
-			# Set value associated with key 
-			# Return cached value
-			def set(key, value)
-				return value
-			end
-
-			# Check if cache contains specified key
-			def contains(key)
-				return false
-			end
-
-			# Invalidate key/value pair
-			def invalidate(key)
-				return true
-			end
-
-			# Clear cache
-			def clear
-				return
-			end
-		end # Cache
-	end # Caches
-end # Apollo
+module Apollo
+	module Caches
+		class Cache
+			# Get value associated with key from cache
+			def get(key, *args)
+
+				# Not found, Create, cache and return
+				res = yield args
+				return res
+			end
+
+			# Set value associated with key 
+			# Return cached value
+			def set(key, value)
+				return value
+			end
+
+			# Check if cache contains specified key
+			def contains(key)
+				return false
+			end
+
+			# Invalidate key/value pair
+			def invalidate(key)
+				return true
+			end
+
+			# Clear cache
+			def clear
+				return
+			end
+		end # Cache
+	end # Caches
+end # Apollo

data/lib/apollo_crawler/caches/factory.rb

@@ -1,30 +1,30 @@
-require File.join(File.dirname(__FILE__), '..', 'cache') 
-require 'singleton'
-
-module Apollo
-	module Caches
-		class Factory
-			include Singleton
-
-			def self.construct()
-				self.singleton.construct()
-			end
-
-			def construct()
-				# Basic implementation
-				# res = Cache.new()
-
-				# Filesystem implementation
-				res = Filesystem.new()
-
-				# In-memory implementation
-				# res = Memory.new()
-
-				# Null (Dummy) implementation
-				# res = Null.new()
-
-				return res
-			end
-		end # Factory
-	end # Caches
-end # Apollo
+require File.join(File.dirname(__FILE__), '..', 'cache') 
+require 'singleton'
+
+module Apollo
+	module Caches
+		class Factory
+			include Singleton
+
+			def self.construct()
+				self.singleton.construct()
+			end
+
+			def construct()
+				# Basic implementation
+				# res = Cache.new()
+
+				# Filesystem implementation
+				res = Filesystem.new()
+
+				# In-memory implementation
+				# res = Memory.new()
+
+				# Null (Dummy) implementation
+				# res = Null.new()
+
+				return res
+			end
+		end # Factory
+	end # Caches
+end # Apollo

data/lib/apollo_crawler/caches/filesystem_cache.rb

@@ -1,34 +1,34 @@
-require File.join(File.dirname(__FILE__), '..', 'cache') 
-
-module Apollo
-	module Caches
-		class Filesystem < Cache
-			def initialize
-				# puts "This if Filesystem cache"
-			end
-
-			# Get value associated with key from cache
-			def get(key, *args)
-				# Not found, Create, cache and return
-				res = yield args
-				return res
-			end
-
-			# Set value associated with key 
-			# Return cached value
-			def set(key, value)
-				return value
-			end
-
-			# Check if cache contains specified key
-			def contains(key)
-				return false
-			end
-
-			# Invalidate key/value pair
-			def invalidate(key)
-				return true
-			end
-		end # Filesystem
-	end # Caches
-end # Apollo
+require File.join(File.dirname(__FILE__), '..', 'cache') 
+
+module Apollo
+	module Caches
+		class Filesystem < Cache
+			def initialize
+				# puts "This if Filesystem cache"
+			end
+
+			# Get value associated with key from cache
+			def get(key, *args)
+				# Not found, Create, cache and return
+				res = yield args
+				return res
+			end
+
+			# Set value associated with key 
+			# Return cached value
+			def set(key, value)
+				return value
+			end
+
+			# Check if cache contains specified key
+			def contains(key)
+				return false
+			end
+
+			# Invalidate key/value pair
+			def invalidate(key)
+				return true
+			end
+		end # Filesystem
+	end # Caches
+end # Apollo

data/lib/apollo_crawler/caches/memory_cache.rb

@@ -1,43 +1,43 @@
-require File.join(File.dirname(__FILE__), '..', 'cache') 
-
-module Apollo
-	module Caches
-		class Memory < Cache
-			@storage = nil
-
-			def initialize
-				@storage = {}
-			end
-
-			# Get value associated with key from cache
-			def get(key, *args)
-				@storage[key]
-
-				# Not found, Create, cache and return
-				res = yield args
-				return res
-			end
-
-			# Set value associated with key 
-			# Return cached value
-			def set(key, value)
-				@storage[key] = value
-			end
-
-			# Check if cache contains specified key
-			def contains(key)
-				@storage.has_key?(key)
-			end
-
-			# Invalidate key/value pair
-			def invalidate(key)
-				@storage.delete(key)
-			end
-
-			# Clear cache
-			def clear
-				@storage.clear
-			end
-		end # Null
-	end # Caches
-end # Apollo
+require File.join(File.dirname(__FILE__), '..', 'cache') 
+
+module Apollo
+	module Caches
+		class Memory < Cache
+			@storage = nil
+
+			def initialize
+				@storage = {}
+			end
+
+			# Get value associated with key from cache
+			def get(key, *args)
+				@storage[key]
+
+				# Not found, Create, cache and return
+				res = yield args
+				return res
+			end
+
+			# Set value associated with key 
+			# Return cached value
+			def set(key, value)
+				@storage[key] = value
+			end
+
+			# Check if cache contains specified key
+			def contains(key)
+				@storage.has_key?(key)
+			end
+
+			# Invalidate key/value pair
+			def invalidate(key)
+				@storage.delete(key)
+			end
+
+			# Clear cache
+			def clear
+				@storage.clear
+			end
+		end # Null
+	end # Caches
+end # Apollo

data/lib/apollo_crawler/caches/null_cache.rb

@@ -1,30 +1,30 @@
-require File.join(File.dirname(__FILE__), '..', 'cache') 
-
-module Apollo
-	module Caches
-		class Null < Cache
-			# Get value associated with key from cache
-			def get(key, *args)
-				# Not found, Create, cache and return
-				res = yield args
-				return res
-			end
-
-			# Set value associated with key 
-			# Return cached value
-			def set(key, value)
-				return value
-			end
-
-			# Check if cache contains specified key
-			def contains(key)
-				return false
-			end
-
-			# Invalidate key/value pair
-			def invalidate(key)
-				return true
-			end
-		end # Null
-	end # Caches
-end # Apollo
+require File.join(File.dirname(__FILE__), '..', 'cache') 
+
+module Apollo
+	module Caches
+		class Null < Cache
+			# Get value associated with key from cache
+			def get(key, *args)
+				# Not found, Create, cache and return
+				res = yield args
+				return res
+			end
+
+			# Set value associated with key 
+			# Return cached value
+			def set(key, value)
+				return value
+			end
+
+			# Check if cache contains specified key
+			def contains(key)
+				return false
+			end
+
+			# Invalidate key/value pair
+			def invalidate(key)
+				return true
+			end
+		end # Null
+	end # Caches
+end # Apollo

data/lib/apollo_crawler/crawler.rb

@@ -1,128 +1,155 @@
-require "open-uri"
-require "nokogiri"
-
-module Apollo
-	module Crawlers
-		class Crawler
-			@backlog = nil
-
-			def initialize
-				@backlog = []
-			end
-
-			# Name of the crawler
-			def name
-				return "Crawler Base" 
-			end
-
-			def url
-				return nil
-			end
-
-			# - (0) Figure out URL
-			# - (1) Extract Data
-			# - (2) Extract Links
-			# - (3) Go to (0) eventually
-			def etl(url=nil, &block)
-				# Look for passed URL use default instead and fail if it is not valid
-				if(url.nil? || url.empty?)
-					url = self.url
-				end
-
-				if(url.nil?)
-					return nil
-				end
-
-				if(url.kind_of?(Array))
-					@backlog.concat(url)
-				else
-					@backlog << url
-				end
-
-				res = []
-				# TODO: Respect limit of documents/urls processed
-				while(@backlog.empty? == false)
-					url = @backlog.shift
-
-					# puts "Processing '#{url}'"
-					doc = self.process_url(url)
-					res << doc
-					
-					# TODO: Use log4r and log it only on info level
-					# TODO: Add some async/callback signal for document processed
-					yield res
-
-					if(!doc.nil? && !doc.empty?)
-						doc[:links].each do |link|
-							url = link[:link].to_s
-							# TODO: Use log4r and log it only on info level
-							#puts url
-
-							# TODO: Check if it is unique
-							@backlog << url
-						end
-					end
-				end
-				return res
-			end
-
-			def process_url(url)
-				# Try fetch document
-				doc = self.fetch_document(url)
-				if(doc.nil?)
-					return nil
-				end
-
-				# Try extract data from document
-				data = self.extract_data(doc)
-
-				# Try extract links for another documents 
-				links = self.extract_links(doc)
-
-				# Format ETL result
-				res = { 
-					:crawler => self.class.name,
-					:title => doc.title,
-					:data => data,
-					:links => links
-				}
-
-				return res
-			end
-
-			# Fetch document
-			def fetch_document(url)
-				# TODO: Refactor following idiom
-				if(url == nil)
-					url = self.url
-				end
-
-				if(url.nil?)
-					return nil
-				end
-
-				# TODO: Use some (custom-made) low-level HTTTP Protocol cache - just for sure
-				cache = Apollo::Caches::Factory.instance.construct
-				raw = cache.get(url) do
-					open(url).read
-				end
-
-				# TODO: Encapsulate and make more robust => invalid hostname, timeouts and so
-				doc = Nokogiri::HTML(raw)
-				return doc
-			end
-
-			# Extracts data from document
-			def extract_data(doc)
-				res = []
-				return res
-			end
-
-			# Extract links to another documents from this document
-			def extract_links(doc)
-				res = []
-				return res
-			end
-		end
-	end
+require "open-uri"
+require "nokogiri"
+
+module Apollo
+	module Crawlers
+		class Crawler
+			@backlog = nil
+
+			def initialize
+				@backlog = []
+			end
+
+			# Name of the crawler
+			def name
+				return "Crawler Base" 
+			end
+
+			def url
+				return nil
+			end
+
+			def self.try_get_url(root, url)
+				begin
+					return URI.join(root, url)
+				rescue
+					return nil
+				end
+			end
+
+			# - (0) Figure out URL
+			# - (1) Extract Data
+			# - (2) Extract Links
+			# - (3) Go to (0) eventually
+			def etl(url=nil, &block)
+				# Look for passed URL use default instead and fail if it is not valid
+				if(url.nil? || url.empty?)
+					url = self.url
+				end
+
+				if(url.nil?)
+					return nil
+				end
+
+				if(url.kind_of?(Array))
+					@backlog.concat(url)
+				else
+					@backlog << url
+				end
+
+				res = []
+				# TODO: Respect limit of documents/urls processed
+				while(@backlog.empty? == false)
+					url = @backlog.shift
+
+					# puts "Processing '#{url}'"
+					doc = self.process_url(url)
+					res << doc
+					
+					# TODO: Use log4r and log it only on info level
+					# TODO: Add some async/callback signal for document processed
+					yield res
+
+					if(!doc.nil? && !doc.empty?)
+						doc[:links].each do |link|
+							url = link[:link].to_s
+							# TODO: Use log4r and log it only on info level
+							#puts url
+
+							# TODO: Check if it is unique
+							@backlog << url
+						end
+					end
+				end
+				return res
+			end
+
+			def process_url(url)
+				# Try fetch document
+				doc = self.fetch_document(url)
+				if(doc.nil?)
+					return nil
+				end
+
+				# Try extract data from document
+				data = self.extract_data(doc)
+
+				# Try extract links for another documents 
+				links = self.extract_links(doc)
+				puts links.inspect
+
+				# Format ETL result
+				res = { 
+					:crawler => self.class.name,
+					:title => doc.title,
+					:data => data,
+					:links => links
+				}
+
+				return res
+			end
+
+			# Fetch document
+			def fetch_document(url)
+				# TODO: Refactor following idiom
+				if(url == nil)
+					url = self.url
+				end
+
+				if(url.nil?)
+					return nil
+				end
+
+				# TODO: Use some (custom-made) low-level HTTTP Protocol cache - just for sure
+				cache = Apollo::Caches::Factory.instance.construct
+				raw = cache.get(url) do
+					max_attempts = 3
+					attempt_no = 0
+					success = false
+					
+					res = nil
+					while(attempt_no < max_attempts && success == false) do
+						begin
+							res = open(url).read
+							success = true
+						rescue Exception => e
+							puts "EXCEPTION: Unable to fetch '#{url}', reason: '#{e.to_s}'"
+							sleep 1
+
+							attempt_no = attempt_no + 1
+							success = false
+						end
+					end
+
+					res
+				end
+
+				# TODO: Encapsulate and make more robust => invalid hostname, timeouts and so
+				doc = Nokogiri::HTML(raw)
+				return doc
+			end
+
+			# Extracts data from document
+			def extract_data(doc)
+				res = []
+				return res
+			end
+
+			# Extract links to another documents from this document
+			def extract_links(doc)
+				res = []
+				return res
+			end
+		end
+	end
 end