apollo-crawler 0.0.48 → 0.0.49

data/bin/apollo-crawler

@@ -24,6 +24,7 @@ require File.join(File.dirname(__FILE__), '..', 'lib', 'apollo_crawler', 'versio
 
 module Apollo
 	class CrawlerProgram
+		@@CACHES_DIR = File.join(File.dirname(__FILE__), "..", "lib", "apollo_crawler", "caches")
 		@@CRAWLERS_DIR = File.join(File.dirname(__FILE__), "..", "lib", "apollo_crawler", "crawlers")
 		@@FORMATTERS_DIR = File.join(File.dirname(__FILE__), "..", "lib", "apollo_crawler", "formatters")
 		@@CRAWLER_TEMPLATE_NAME = "crawler_template.rb"
@@ -33,12 +34,14 @@ module Apollo
 		# OptionParser.
 		@options = nil
 		@optparser = nil
+		@caches = nil
 		@crawlers = nil
 		@formatters = nil
 		@formatter = nil
 
 		# Initializer - Constructor
 		def initialize
+			@caches = {}
 			@crawlers = {}
 			@formatters = {}
 		end
@@ -48,6 +51,9 @@ module Apollo
 			@options = {}
 			@options[:verbose] = false
 			@options[:version] = false
+			@options[:cache_dirs] = [
+				@@CACHES_DIR
+			]
 			@options[:crawler_dirs] = [
 				@@CRAWLERS_DIR
 			]
@@ -130,10 +136,10 @@ module Apollo
 			end
 		end
 
-		# Register formatters
-		def register_formatters(dir)			
+		# Register caches
+		def register_cache(dir)			
 			if(@options[:verbose]) 
-				puts "Registering formatters - '#{dir}'"
+				puts "Registering caches - '#{dir}'"
 			end
 
 			files = File.join(dir, "**", "*.rb")
@@ -141,24 +147,25 @@ module Apollo
 				require file
 			end
 
-			tmp = Apollo::Formatters.constants.select { |c| 
-				Class === Apollo::Formatters.const_get(c)
+			tmp = Apollo::Caches.constants.select { |c| 
+				Class === Apollo::Caches.const_get(c)
 			}
 
 			tmp.each do |x| 
-				klass = Object.const_get('Apollo').const_get('Formatters').const_get(x)
-				@formatters.merge!({ x.downcase.to_s => klass})
+				klass = Object.const_get('Apollo').const_get('Caches').const_get(x)
+				@caches.merge!({ x.downcase.to_s => klass})
 			end
 
 			if(@options[:verbose])
-				@formatters.each do |formatter, klass|
-					name = klass.new.class.name
+				@caches.each do |cache, klass|
+					name = klass
 
-					if  name == "Apollo::Formatters::Formatter"
+					# klass.ancestors.include?(Apollo::Caches::Cache)
+					if  name == "Apollo::Caches::Cache"
 						next
 					end
 
-					puts "Registered formatter '#{formatter}' -> '#{name}'"
+					puts "Registered cache '#{cache}' -> '#{name}'"
 				end
 			end
 		end
@@ -196,6 +203,39 @@ module Apollo
 			end
 		end
 
+		# Register formatters
+		def register_formatters(dir)			
+			if(@options[:verbose]) 
+				puts "Registering formatters - '#{dir}'"
+			end
+
+			files = File.join(dir, "**", "*.rb")
+			Dir.glob(files).each do |file|
+				require file
+			end
+
+			tmp = Apollo::Formatters.constants.select { |c| 
+				Class === Apollo::Formatters.const_get(c)
+			}
+
+			tmp.each do |x| 
+				klass = Object.const_get('Apollo').const_get('Formatters').const_get(x)
+				@formatters.merge!({ x.downcase.to_s => klass})
+			end
+
+			if(@options[:verbose])
+				@formatters.each do |formatter, klass|
+					name = klass.new.class.name
+
+					if  name == "Apollo::Formatters::Formatter"
+						next
+					end
+
+					puts "Registered formatter '#{formatter}' -> '#{name}'"
+				end
+			end
+		end
+
 		def generate_crawler(name, url = nil, matcher = nil)
 			name = name.titleize.gsub(" ", "")
 
@@ -262,6 +302,11 @@ module Apollo
 				exit
 			end
 
+			# Register caches which can be used
+			@options[:cache_dirs].each do |dir|
+				register_cache(dir)
+			end
+
 			# Register sites which can be crawled
 			@options[:crawler_dirs].each do |dir|
 				register_crawlers(dir)
@@ -278,7 +323,6 @@ module Apollo
 				formatter_name = @options[:formatter]
 			end
 
-			
 			# Look for specified formatter
 			f = @formatters.select { |k, v|
 				k.downcase == formatter_name.downcase
@@ -308,6 +352,8 @@ module Apollo
 				return
 			end
 
+
+
 			crawlers = []
 			if(ARGV.length > 0)
 				crawlers << ARGV.shift

data/lib/apollo_crawler.rb

@@ -1,6 +1,13 @@
+# Main
+require 'apollo_crawler/cache'
 require 'apollo_crawler/crawler'
 require 'apollo_crawler/formatter'
 
+# Caches
+require 'apollo_crawler/caches/factory'
+require 'apollo_crawler/caches/memory_cache'
+require 'apollo_crawler/caches/null_cache'
+
 # Crawlers
 require 'apollo_crawler/crawlers/alexa_com/alexa'
 require 'apollo_crawler/crawlers/firmy_cz/firmy'

data/lib/apollo_crawler/cache.rb

@@ -0,0 +1,34 @@
+module Apollo
+	module Caches
+		class Cache
+			# Get value associated with key from cache
+			def get(key, *args)
+
+				# Not found, Create, cache and return
+				res = yield args
+				return res
+			end
+
+			# Set value associated with key 
+			# Return cached value
+			def set(key, value)
+				return value
+			end
+
+			# Check if cache contains specified key
+			def contains(key)
+				return false
+			end
+
+			# Invalidate key/value pair
+			def invalidate(key)
+				return true
+			end
+
+			# Clear cache
+			def clear
+				return
+			end
+		end # Cache
+	end # Caches
+end # Apollo

data/lib/apollo_crawler/caches/factory.rb

@@ -0,0 +1,18 @@
+require File.join(File.dirname(__FILE__), '..', 'cache') 
+require 'singleton'
+
+module Apollo
+	module Caches
+		class Factory
+			include Singleton
+
+			def self.construct()
+				self.singleton.construct()
+			end
+
+			def construct()
+				Memory.new()
+			end
+		end # Factory
+	end # Caches
+end # Apollo

data/lib/apollo_crawler/caches/filesystem_cache.rb

@@ -0,0 +1,30 @@
+require File.join(File.dirname(__FILE__), '..', 'cache') 
+
+module Apollo
+	module Caches
+		class Filesystem < Cache
+			# Get value associated with key from cache
+			def get(key, *args)
+				# Not found, Create, cache and return
+				res = yield args
+				return res
+			end
+
+			# Set value associated with key 
+			# Return cached value
+			def set(key, value)
+				return value
+			end
+
+			# Check if cache contains specified key
+			def contains(key)
+				return false
+			end
+
+			# Invalidate key/value pair
+			def invalidate(key)
+				return true
+			end
+		end # Filesystem
+	end # Caches
+end # Apollo

data/lib/apollo_crawler/caches/memory_cache.rb

@@ -0,0 +1,43 @@
+require File.join(File.dirname(__FILE__), '..', 'cache') 
+
+module Apollo
+	module Caches
+		class Memory < Cache
+			@storage = nil
+
+			def initialize
+				@storage = {}
+			end
+
+			# Get value associated with key from cache
+			def get(key, *args)
+				@storage[key]
+
+				# Not found, Create, cache and return
+				res = yield args
+				return res
+			end
+
+			# Set value associated with key 
+			# Return cached value
+			def set(key, value)
+				@storage[key] = value
+			end
+
+			# Check if cache contains specified key
+			def contains(key)
+				@storage.has_key?(key)
+			end
+
+			# Invalidate key/value pair
+			def invalidate(key)
+				@storage.delete(key)
+			end
+
+			# Clear cache
+			def clear
+				@storage.clear
+			end
+		end # Null
+	end # Caches
+end # Apollo

data/lib/apollo_crawler/caches/null_cache.rb

@@ -0,0 +1,30 @@
+require File.join(File.dirname(__FILE__), '..', 'cache') 
+
+module Apollo
+	module Caches
+		class Null < Cache
+			# Get value associated with key from cache
+			def get(key, *args)
+				# Not found, Create, cache and return
+				res = yield args
+				return res
+			end
+
+			# Set value associated with key 
+			# Return cached value
+			def set(key, value)
+				return value
+			end
+
+			# Check if cache contains specified key
+			def contains(key)
+				return false
+			end
+
+			# Invalidate key/value pair
+			def invalidate(key)
+				return true
+			end
+		end # Null
+	end # Caches
+end # Apollo

data/lib/apollo_crawler/crawler.rb

@@ -40,11 +40,27 @@ module Apollo
 				end
 
 				res = []
+				# TODO: Respect limit of documents/urls processed
 				while(@backlog.empty? == false)
 					url = @backlog.shift
 
 					# puts "Processing '#{url}'"
-					res << self.process_url(url)
+					doc = self.process_url(url)
+					res << doc
+					
+					# TODO: Use log4r and log it only on info level
+					puts doc.inspect
+
+					if(!doc.nil? && !doc.empty?)
+						doc[:links].each do |link|
+							url = link[:link].to_s
+							# TODO: Use log4r and log it only on info level
+							#puts url
+
+							# TODO: Check if it is unique
+							@backlog << url
+						end
+					end
 				end
 				return res
 			end
@@ -62,17 +78,22 @@ module Apollo
 				# Try extract links for another documents 
 				links = self.extract_links(doc)
 
-				# Return ETL result
-				return { 
+				# Format ETL result
+				res = { 
 					:crawler => self.class.name,
 					:title => doc.title,
 					:data => data,
 					:links => links
 				}
+
+				# TODO: Add some async/callback signal for document processed
+
+				return res
 			end
 
 			# Fetch document
 			def fetch_document(url)
+				# TODO: Refactor following idiom
 				if(url == nil)
 					url = self.url
 				end
@@ -81,7 +102,11 @@ module Apollo
 					return nil
 				end
 
-				raw = open(url).read
+				# TODO: Use some (custom-made) low-level HTTTP Protocol cache - just for sure
+				cache = Apollo::Caches::Factory.instance.construct
+				raw = cache.get(url) do
+					open(url).read
+				end
 
 				# TODO: Encapsulate and make more robust => invalid hostname, timeouts and so
 				doc = Nokogiri::HTML(raw)

data/lib/apollo_crawler/crawler_template.rb

@@ -19,6 +19,6 @@ module Apollo
 					}
 				}
 			end
-		end
+		end # CRAWLER_CLASS_NAME
 	end # Crawlers
 end # Apollo

data/lib/apollo_crawler/crawlers/slashdot_org/slashdot.rb

@@ -5,7 +5,7 @@ module Apollo
 		class Slashdot < Crawler
 			@@MATCHER_ITEM = "//article/header/h2/span/a"
 
-			def name
+			def name()
 				return "Slashdot"
 			end
 

data/lib/apollo_crawler/crawlers/stackoverflow_com/stackoverflow.rb

@@ -5,8 +5,8 @@ module Apollo
 		class StackOverflow < Crawler
 			@@MATCHER_ITEM = "//div[@class = 'summary']/h3/a"
 
-			def name
-				return "StackOverflow"
+			def name()
+				return "Stackoverflow"
 			end
 
 			def url()

data/lib/apollo_crawler/crawlers/xkcd_com/xkcd.rb

@@ -17,10 +17,19 @@ module Apollo
 				res = doc.xpath(@@MATCHER_ITEM).map { |node|
 					{ 
 						:text => node['title'],
-						:link => URI.join(self.url, node['src'])
+						:link => URI.join(self.url, node['src']),
 					}
 				}
 			end
+
+			def extract_links(doc)
+				res = doc.xpath("//ul[@class = 'comicNav']/li/a[@accesskey = 'p']").map { |node|
+					{ 
+						:link => URI.join(self.url, node['href']),
+					}
+				}
+				res.uniq
+			end
 		end
 	end # Crawlers
 end # Apollo

data/lib/apollo_crawler/crawlers/ycombinator_com/hacker_news.rb

@@ -5,7 +5,7 @@ module Apollo
 		class HackerNews < Crawler
 			@@MATCHER_ITEM = "//td[@class = 'title']/a"
 
-			def name
+			def name()
 				return "Hacker News"
 			end
 

data/lib/apollo_crawler/formatter.rb

@@ -1,6 +1,6 @@
 module Apollo
 	module Formatters
 		class Formatter
-		end
+		end # Formatter
 	end # Formatters
 end # Apollo

data/lib/apollo_crawler/formatters/formatter_table.rb

@@ -24,8 +24,7 @@ module Apollo
 
 					rows << data
 				end
-
-
+				
 				table = Terminal::Table.new :headings => headings, :rows => rows
 				return table
 			end

data/lib/apollo_crawler/version.rb

@@ -1,5 +1,5 @@
 module Apollo
 	module Crawler
-		VERSION = '0.0.48'
+		VERSION = '0.0.49'
 	end # Crawler
 end # Apollo

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: apollo-crawler
 version: !ruby/object:Gem::Version
-  version: 0.0.48
+  version: 0.0.49
   prerelease: 
 platform: ruby
 authors:
@@ -230,6 +230,10 @@ files:
 - ./lib/apollo_crawler/formatters/formatter_json.rb
 - ./lib/apollo_crawler/formatters/formatter_table.rb
 - ./lib/apollo_crawler/version.rb
+- ./lib/apollo_crawler/caches/factory.rb
+- ./lib/apollo_crawler/caches/null_cache.rb
+- ./lib/apollo_crawler/caches/memory_cache.rb
+- ./lib/apollo_crawler/caches/filesystem_cache.rb
 - ./lib/apollo_crawler/crawler_template.rb
 - ./lib/apollo_crawler/crawler.rb
 - ./lib/apollo_crawler/crawlers/stackoverflow_com/stackoverflow.rb
@@ -240,6 +244,7 @@ files:
 - ./lib/apollo_crawler/crawlers/alexa_com/alexa.rb
 - ./lib/apollo_crawler/crawlers/ycombinator_com/hacker_news.rb
 - ./lib/apollo_crawler/formatter.rb
+- ./lib/apollo_crawler/cache.rb
 - ./lib/apollo_crawler.rb
 - bin/apollo-crawler
 homepage: https://github.com/korczis/apollo-crawler