apollo-crawler 0.1.8 → 0.1.9

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    YjEzODFmZTRlMzYyYzc1MjhkMzc4MzQ5ZDk4MWNmYWQ4NjhkNWZjNw==
+    Y2I3MDM1OWQ1NmU2ZDMzMTg0OGRhNTYzODc5Mzg4MDhhZTkxOWJlMQ==
   data.tar.gz: !binary |-
-    YTc2ZTFhYmU3NjE1OGJjMTU1OTllN2JkY2ZjODBmOTdjNDgwMWVhMQ==
+    ZWY0N2M3ZjU5ZGNmZjgwMTdkNWI0Y2JhYmZjZmUwNDFjYTA5ZjAwOA==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    OGI1YzJjZWU0MGY0MGVjN2MxYzczMGY4ODc4OWVhMTc0MTNlNTgwYjkwNzI4
-    ZDIxYTBlMDJlMWEwMjMzYzBmOWUyYmQ5OWMwNGEwMWNlOGI5M2U1YjIxNjVi
-    NDg2ZGM1NDBlNzI2ZTVlOTBjOGQ5NjljYTcwNmQ2NWZhY2E1OWI=
+    MjM1ZWUzMGUyYzBiYjdmNmFjMmE4OTU5MjA0NDE3YWJlZTE4OWQ0YTE5MWQx
+    ZDBmN2UzZjk3N2Y3NDYwOGNiMWFiY2JkY2I2ODJmYzFkYWU3YTYzYjI5YTI0
+    NDQxZDk4YzBlODBmNDg3MzRkMDU2OWY1ZmViNmUzYWFhZjZlNGU=
   data.tar.gz: !binary |-
-    Mjk0MWJkNDc0ZmI1MWM0ZmRkZTA4Nzg5MmQzNjNjNGZiNTE2NTgwMzU4YzI5
-    Nzc5YWUzZWVlODUwNjJjMTc3MzBlNWUxNDQ4YWEzYzZjYmM1MDdkYWQ0ZGIz
-    NTRmMjQwMzEwZGM0NGJhODUxZjY3ZGQwODFhY2Y4M2I3ZDEzYTk=
+    MDIzY2RhMmFkNmI4MjBlYzkzNmZhOTA5NjM2NWUyM2YzODQwM2M0NGIzN2U1
+    OTgxNGYwYTdhMTIxZGVkYzNlYjE1M2U1N2NjMDdiN2I4MWNkMTMwMzRmMDMy
+    MWFiOWRhNGIyOTZhY2NmZDE2YTRjMzUxYjQyODU1NTAzNjQwM2M=

data/lib/apollo_crawler.rb

@@ -5,27 +5,34 @@
 require File.join(File.dirname(__FILE__), 'apollo_crawler/program')
 
 # Caches
-require File.join(File.dirname(__FILE__), 'apollo_crawler/cache/cache_base')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/cache/base_cache')
 require File.join(File.dirname(__FILE__), 'apollo_crawler/cache/factory')
-require File.join(File.dirname(__FILE__), 'apollo_crawler/cache/filesystem_cache')
 require File.join(File.dirname(__FILE__), 'apollo_crawler/cache/memcached_cache')
 require File.join(File.dirname(__FILE__), 'apollo_crawler/cache/memory_cache')
 require File.join(File.dirname(__FILE__), 'apollo_crawler/cache/null_cache')
 
 # Crawlers
-require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/crawler_base')
-require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/google_com/google')
-require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/slashdot_org/slashdot')
-require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/stackoverflow_com/stackoverflow')
-require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/xkcd_com/xkcd')
-require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/ycombinator_com/hacker_news')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/base_crawler')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/google_crawler')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/hacker_news_crawler')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/slashdot_crawler')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/stackoverflow_crawler')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/xkcd_crawler')
 
 # Fetchers
-require File.join(File.dirname(__FILE__), 'apollo_crawler/fetcher/fetcher_base')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/fetcher/base_fetcher')
 require File.join(File.dirname(__FILE__), 'apollo_crawler/fetcher/simple_fetcher')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/fetcher/smart_fetcher')
 
 # Formatters
-require File.join(File.dirname(__FILE__), 'apollo_crawler/formatter/formatter_base')
-require File.join(File.dirname(__FILE__), 'apollo_crawler/formatter/formatter_json')
-require File.join(File.dirname(__FILE__), 'apollo_crawler/formatter/formatter_plain')
-require File.join(File.dirname(__FILE__), 'apollo_crawler/formatter/formatter_table')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/formatter/base_formatter')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/formatter/json_formatter')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/formatter/plain_formatter')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/formatter/table_formatter')
+
+# Loggers
+require File.join(File.dirname(__FILE__), 'apollo_crawler/logger/base_logger')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/logger/console_logger')
+
+# Stores
+require File.join(File.dirname(__FILE__), 'apollo_crawler/store/base_store')

data/lib/apollo_crawler/cache/{cache_base.rb → base_cache.rb}

@@ -1,6 +1,6 @@
 module Apollo
 	module Cache
-		class CacheBase
+		class BaseCache
 			# Get value associated with key from cache
 			def get(key, *args)
 
@@ -32,6 +32,6 @@ module Apollo
 			def clear
 				return
 			end
-		end # CacheBase
-	end # Cache
-end # Apollo
+		end # class BaseCache
+	end # module Cache
+end # module Apollo

data/lib/apollo_crawler/cache/factory.rb

@@ -2,7 +2,7 @@
 require File.join(File.dirname(__FILE__), '..', 'config')
 
 # Cache instance base class
-require File.join(File.dirname(__FILE__), 'cache_base') 
+require File.join(File.dirname(__FILE__), 'base_cache') 
 
 # Factory uses singleton pattern
 require 'singleton'

data/lib/apollo_crawler/cache/memcached_cache.rb

@@ -1,10 +1,10 @@
-require File.join(File.dirname(__FILE__), 'cache_base') 
+require File.join(File.dirname(__FILE__), 'base_cache') 
 
 require 'dalli'
 
 module Apollo
 	module Cache
-		class Memcached < CacheBase
+		class MemcachedCache < BaseCache
 			@cache = nil
 
 			def initialize
@@ -46,6 +46,6 @@ module Apollo
 			def clear
 				# TODO: Implement
 			end
-		end # Null
-	end # Cache
-end # Apollo
+		end # class MemcachedCache
+	end # module Cache
+end # module Apollo

data/lib/apollo_crawler/cache/memory_cache.rb

@@ -1,8 +1,8 @@
-require File.join(File.dirname(__FILE__), 'cache_base') 
+require File.join(File.dirname(__FILE__), 'base_cache') 
 
 module Apollo
 	module Cache
-		class Memory < CacheBase
+		class MemoryCache < BaseCache
 			@cache = nil
 
 			def initialize
@@ -41,6 +41,6 @@ module Apollo
 			def clear
 				@cache.clear
 			end
-		end # Null
-	end # Cache
-end # Apollo
+		end # class MemoryCache
+	end # module Cache
+end # module Apollo

data/lib/apollo_crawler/cache/null_cache.rb

@@ -1,8 +1,8 @@
-require File.join(File.dirname(__FILE__), 'cache_base') 
+require File.join(File.dirname(__FILE__), 'base_cache') 
 
 module Apollo
 	module Cache
-		class Null < CacheBase
+		class NullCache < BaseCache
 			# Get value associated with key from cache
 			def get(key, *args)
 				# Not found, Create, cache and return
@@ -28,6 +28,6 @@ module Apollo
 			def invalidate(key)
 				return true
 			end
-		end # Null
+		end # NullCache
 	end # Cache
 end # Apollo

data/lib/apollo_crawler/config.rb

@@ -14,40 +14,55 @@ module RbConfig
 	############################################################
 	#
 	# Filesystem backend
-	# CACHE_CLASS = Apollo::Cache::Filesystem
+	# CACHE_CLASS = Apollo::Cache::FilesystemCache
 	#
 	# Memcached - expects localhost:11211
-	# CACHE_CLASS = Apollo::Cache::Memcached
+	# CACHE_CLASS = Apollo::Cache::MemcachedCache
 	#
 	# Pure naive ruby in-memory implementation
-	# CACHE_CLASS = Apollo::Cache::Memory
+	# CACHE_CLASS = Apollo::Cache::MemoryCache
 	#
 	# Null caching - no caching at all
-	# CACHE_CLASS = Apollo::Cache::Null
+	# CACHE_CLASS = Apollo::Cache::NullCache
 
 	# Used caching mechanism by default
-	CACHE_CLASS = Apollo::Cache::Memcached
+	CACHE_CLASS = Apollo::Cache::MemcachedCache
 
 
 
 	############################################################
 	# Crawlers - Built-in out-of box working crawlers
 	############################################################
-	CRAWLERS_DIR = File.join(File.dirname(__FILE__), "crawlers")
+	CRAWLERS_DIR = File.join(File.dirname(__FILE__), "crawler")
 	
 	# Template used for generated crawlers
-	CRAWLER_TEMPLATE_NAME = "crawler_template.rb"
+	CRAWLER_TEMPLATE_NAME = "crawler_template.trb"
 
 	# Path of template
-	CRAWLER_TEMPLATE_PATH = File.join(File.dirname(__FILE__), "crawler_template.rb")
+	CRAWLER_TEMPLATE_PATH = File.join(CRAWLERS_DIR, CRAWLER_TEMPLATE_NAME)
 
 
 
+	############################################################
+	# Fetchers - used for fetching documents
+	############################################################
+	FETCHERS_DIR = File.join(File.dirname(__FILE__), "fetcher")
+	
+	DEFAULT_FETCHER = Apollo::Fetcher::SmartFetcher
+
+
 	############################################################
 	# Formatters - used for formatting crawled documents results
 	############################################################
-	FORMATTERS_DIR = File.join(File.dirname(__FILE__), "formatters")
+	FORMATTERS_DIR = File.join(File.dirname(__FILE__), "formatter")
 
 	# Default formatter if no other specified
-	DEFAULT_FORMATTER = Apollo::Formatter::Json
+	DEFAULT_FORMATTER = Apollo::Formatter::JsonFormatter
+
+
+
+	############################################################
+	# Loggers - used for formatting output messages
+	############################################################
+	DEFAULT_LOGGER = Apollo::Logger::ConsoleLogger
 end # Config

data/lib/apollo_crawler/crawler/{crawler_base.rb → base_crawler.rb}

@@ -3,13 +3,19 @@ require "nokogiri"
 
 module Apollo
 	module Crawler
-		class CrawlerBase
+		class BaseCrawler
+			
+
 			@backlog = nil
 
 			def initialize
 				@backlog = []
 			end
 
+			def self.name_re()
+				return /crawler$/
+			end
+
 			# Name of the crawler
 			def name
 				return "Crawler Base" 
@@ -19,6 +25,10 @@ module Apollo
 				return nil
 			end
 
+			def self.fetch(url)
+				RbConfig::DEFAULT_FETCHER.fetch(url)
+			end
+
 			def self.try_get_url(root, url)
 				begin
 					return URI.join(root, url)
@@ -28,7 +38,7 @@ module Apollo
 			end
 
 			def self.try_get_doc(root, url)
-				doc = CrawlerBase.try_get_url(root, url)
+				doc = BaseCrawler.try_get_url(root, url)
 				
 				# TODO: Set experition header
 				return {
@@ -47,16 +57,19 @@ module Apollo
 					url = self.url
 				end
 
+				# TODO: Be more agressive, use assert, it is clients responsibility!
 				if(url.nil?)
 					return nil
 				end
 
+				# We support both - list of urls or single url
 				if(url.kind_of?(Array))
 					@backlog.concat(url)
 				else
 					@backlog << url
 				end
 
+				# Counter of processed documents (pages)
 				docs_processed = 0
 
 				res = []
@@ -66,34 +79,40 @@ module Apollo
 
 					# puts "Processing '#{url}'"
 					doc = self.process_url(url)
-					res << doc
 					
-					# TODO: Use log4r and log it only on info level
-					# TODO: Add some async/callback signal for document processed
-					if block_given?
-						yield res
-					end
+					# Increase counter of processed documents
+					docs_processed = docs_processed + 1
+
+					# Process document if was successfuly retreived
+					if(!doc.nil?)
+						# TODO: Use log4r and log it only on info level
+						if block_given?
+							yield doc
+						end
 
-					if(!doc.nil? && !doc.empty?)
-						doc[:links].each do |link|
-							url = link[:link].to_s
-							# TODO: Use log4r and log it only on info level
-							#puts url
+						# Add document to queue of results
+						res << doc
 
-							# TODO: Check if it is unique
-							@backlog << url
+						# If
+						if(doc[:links].nil? == false)
+							doc[:links].each do |link|
+								url = link[:link].to_s
+								# TODO: Use log4r and log it only on info level
+								#puts url
+
+								# TODO: Check if it is unique
+								@backlog << url
+							end
 						end
 					end
 
-					# Increase counter of processed documents
-					docs_processed = docs_processed + 1
+					# Break if limit of documents to processed was reached
 					break if opts[:doc_limit] && docs_processed >= opts[:doc_limit]
 				end
 				return res
 			end
 
 			def process_url(url)
-				# Try fetch document
 				doc = self.fetch_document(url)
 				if(doc.nil?)
 					return nil
@@ -140,7 +159,7 @@ module Apollo
 					res = nil
 					while(attempt_no < max_attempts && success == false) do
 						begin
-							res = open(url).read
+							res = BaseCrawler.fetch(url)
 							success = true
 						rescue Exception => e
 							puts "EXCEPTION: Unable to fetch '#{url}', reason: '#{e.to_s}'"
@@ -170,6 +189,6 @@ module Apollo
 				res = []
 				return res
 			end
-		end # CrawlerBase
-	end # Crawler
-end # Apollo
+		end # class BaseCrawler
+	end # module Crawler
+end # module Apollo

data/lib/apollo_crawler/crawler/{google_com/google.rb → google_crawler.rb}

@@ -1,8 +1,8 @@
-require File.join(File.dirname(__FILE__), '..', 'crawler_base') 
+require File.join(File.dirname(__FILE__), 'base_crawler') 
 
 module Apollo
 	module Crawler
-		class Google < CrawlerBase
+		class GoogleCrawler < BaseCrawler
 			@@MATCHER_ITEM = "//h3/a"
 
 			def name()
@@ -15,7 +15,7 @@ module Apollo
 
 			def extract_data(doc)
 				res = doc.xpath(@@MATCHER_ITEM).map { | node |
-					url = CrawlerBase.try_get_url(self.url, node['href'])
+					url = BaseCrawler.try_get_url(self.url, node['href'])
 					next if url.nil?
 
 					{ 
@@ -27,7 +27,7 @@ module Apollo
 
 			def extract_links(doc)
 				res = doc.xpath("(//td[@class = 'b']/a)[last()]").map { | node |
-					res_doc = CrawlerBase.try_get_url(self.url, node['href'])
+					res_doc = BaseCrawler.try_get_url(self.url, node['href'])
 					next if url.nil?
 
 					{ 
@@ -35,6 +35,6 @@ module Apollo
 					}
 				}
 			end
-		end
-	end # Crawler
-end # Apollo
+		end # class GoogleCrawler
+	end # module Crawler
+end # module Apollo

data/lib/apollo_crawler/crawler/{ycombinator_com/hacker_news.rb → hacker_news_crawler.rb}

@@ -1,8 +1,8 @@
-require File.join(File.dirname(__FILE__), '..', 'crawler_base') 
+require File.join(File.dirname(__FILE__), 'base_crawler') 
 
 module Apollo
 	module Crawler
-		class HackerNews < CrawlerBase
+		class HackerNewsCrawler < BaseCrawler
 			@@MATCHER_ITEM = "(//td[@class = 'title']/a)[not(position() > last() -1)]"
 
 			def name()
@@ -15,7 +15,7 @@ module Apollo
 
 			def extract_data(doc)
 				res = doc.xpath(@@MATCHER_ITEM).map { |node|
-					url = CrawlerBase.try_get_url(self.url, node['href'])
+					url = BaseCrawler.try_get_url(self.url, node['href'])
 					next if url.nil?
 
 					{ 
@@ -29,7 +29,7 @@ module Apollo
 
 			def extract_links(doc)
 				res = doc.xpath("(//a[@class = 'prevnextbutact'])").map { |node|
-					url = CrawlerBase.try_get_url(self.url, node['href'])
+					url = BaseCrawler.try_get_url(self.url, node['href'])
 					next if url.nil?
 
 					{ 
@@ -39,6 +39,6 @@ module Apollo
 				
 				return res.uniq
 			end
-		end
-	end # Crawler
-end # Apollo
+		end # class HackerNewsCrawler
+	end # module Crawler
+end # module Apollo

data/lib/apollo_crawler/crawler/{slashdot_org/slashdot.rb → slashdot_crawler.rb}

@@ -1,8 +1,8 @@
-require File.join(File.dirname(__FILE__), '..', 'crawler_base') 
+require File.join(File.dirname(__FILE__), 'base_crawler') 
 
 module Apollo
 	module Crawler
-		class Slashdot < CrawlerBase
+		class SlashdotCrawler < BaseCrawler
 			@@MATCHER_ITEM = "//article/header/h2/span/a"
 
 			def name()
@@ -15,7 +15,7 @@ module Apollo
 
 			def extract_data(doc)
 				res = doc.xpath(@@MATCHER_ITEM).map { | node |
-					url = CrawlerBase.try_get_url(self.url, node['href'])
+					url = BaseCrawler.try_get_url(self.url, node['href'])
 					next if url.nil?
 
 					{ 
@@ -27,7 +27,7 @@ module Apollo
 
 			def extract_links(doc)
 				res = doc.xpath(@@MATCHER_ITEM).map { | node |
-					url = CrawlerBase.try_get_url(self.url, node['href'])
+					url = BaseCrawler.try_get_url(self.url, node['href'])
 					next if url.nil?
 
 					{ 
@@ -35,6 +35,6 @@ module Apollo
 					}
 				}
 			end
-		end
-	end # Crawler
-end # Apollo
+		end # class SlashdotCrawler
+	end # module Crawler
+end # module Apollo

data/lib/apollo_crawler/crawler/{stackoverflow_com/stackoverflow.rb → stackoverflow_crawler.rb}

@@ -1,8 +1,8 @@
-require File.join(File.dirname(__FILE__), '..', 'crawler_base') 
+require File.join(File.dirname(__FILE__), 'base_crawler') 
 
 module Apollo
 	module Crawler
-		class StackOverflow < CrawlerBase
+		class StackoverflowCrawler < BaseCrawler
 			@@MATCHER_ITEM = "//div[@class = 'summary']/h3/a"
 
 			def name()
@@ -15,7 +15,7 @@ module Apollo
 
 			def extract_data(doc)
 				res = doc.xpath(@@MATCHER_ITEM).map { |node|
-					url = CrawlerBase.try_get_url(self.url, node['href'])
+					url = BaseCrawler.try_get_url(self.url, node['href'])
 					next if url.nil?
 
 					{ 
@@ -29,7 +29,7 @@ module Apollo
 
 			def extract_links(doc)
 				res = doc.xpath("(//div[@class = 'pager fl']/a)[last()]").map { |node|
-					url = CrawlerBase.try_get_url(self.url, node['href'])
+					url = BaseCrawler.try_get_url(self.url, node['href'])
 					next if url.nil?
 
 					{ 
@@ -39,6 +39,6 @@ module Apollo
 				
 				return res.uniq
 			end
-		end
-	end # Crawler
-end # Apollo
+		end # class StackoverflowCrawler
+	end # module Crawler
+end # module Apollo

data/lib/apollo_crawler/crawler/{xkcd_com/xkcd.rb → xkcd_crawler.rb}

@@ -1,8 +1,8 @@
-require File.join(File.dirname(__FILE__), '..', 'crawler_base') 
+require File.join(File.dirname(__FILE__), 'base_crawler') 
 
 module Apollo
 	module Crawler
-		class Xkcd < CrawlerBase
+		class XkcdCrawler < BaseCrawler
 			@@MATCHER_ITEM = "//div[@id = 'comic']/img"
 
 			def name()
@@ -30,6 +30,6 @@ module Apollo
 				}
 				res.uniq
 			end
-		end
-	end # Crawler
-end # Apollo
+		end # class XkcdCrawler
+	end # module Crawler
+end # module Apollo

data/lib/apollo_crawler/fetcher/base_fetcher.rb

@@ -0,0 +1,11 @@
+module Apollo
+	module Fetcher
+		class BaseFetcher
+			def self.fetch(url)
+				# TODO: Throw exception ???
+				return nil
+			end
+			
+		end # class BaseFetcher
+	end # module Fetcher
+end # module Apollo

data/lib/apollo_crawler/fetcher/simple_fetcher.rb

@@ -1,8 +1,15 @@
-require File.join(File.dirname(__FILE__), 'fetcher_base') 
+require "open-uri"
+require "nokogiri"
+
+require File.join(File.dirname(__FILE__), 'base_fetcher') 
 
 module Apollo
 	module Fetcher
-		class SimpleFetcher < FetcherBase
-		end # Crawler
-	end # Fetcher
-end # Apollo
+		class SimpleFetcher < BaseFetcher
+			def self.fetch(url)
+				# TODO: Throw exception ???
+				return open(url).read
+			end
+		end # class SimpleFetcher
+	end # module Fetcher
+end # module Apollo

data/lib/apollo_crawler/fetcher/smart_fetcher.rb

@@ -0,0 +1,15 @@
+require "open-uri"
+require "nokogiri"
+
+require File.join(File.dirname(__FILE__), 'base_fetcher') 
+
+module Apollo
+	module Fetcher
+		class SmartFetcher < BaseFetcher
+			def self.fetch(url)
+				# TODO: Throw exception ???
+				return open(url).read
+			end
+		end # class SimpleFetcher
+	end # module SmartFetcher
+end # module Apollo

data/lib/apollo_crawler/formatter/base_formatter.rb

@@ -0,0 +1,9 @@
+module Apollo
+	module Formatter
+		class BaseFormatter
+			def self.name_re()
+				return /formatter$/
+			end
+		end # class BaseFormatter
+	end # module Formatter
+end # module Apollo

data/lib/apollo_crawler/formatter/{formatter_json.rb → json_formatter.rb}

@@ -1,10 +1,10 @@
 require 'json'
 
-require File.join(File.dirname(__FILE__), 'formatter_base') 
+require File.join(File.dirname(__FILE__), 'base_formatter') 
 
 module Apollo
 	module Formatter
-		class Json < FormatterBase
+		class JsonFormatter < BaseFormatter
 			def format(obj)
 				return Json.format(obj)
 			end
@@ -12,6 +12,6 @@ module Apollo
 			def self.format(obj)
 				 return JSON.pretty_generate(obj)
 			end
-		end
-	end # Formatter
-end # Apollo
+		end # class JsonFormatter
+	end # module Formatter
+end # module Apollo

data/lib/apollo_crawler/formatter/plain_formatter.rb

@@ -0,0 +1,17 @@
+require 'awesome_print'
+
+require File.join(File.dirname(__FILE__), 'base_formatter') 
+
+module Apollo
+	module Formatter
+		class PlainFormatter < BaseFormatter
+			def format(obj)
+				return Plain.format(obj)
+			end
+
+			def self.format(obj)
+				 return obj.inspect
+			end
+		end # class PlainFormatter
+	end # module Formatter
+end # module Apollo

data/lib/apollo_crawler/formatter/{formatter_table.rb → table_formatter.rb}

@@ -1,10 +1,10 @@
 require 'terminal-table'
 
-require File.join(File.dirname(__FILE__), 'formatter_base') 
+require File.join(File.dirname(__FILE__), 'base_formatter') 
 
 module Apollo
 	module Formatter
-		class Table < FormatterBase
+		class TableFormatter < BaseFormatter
 			def format(obj)
 				return Table.format(obj)
 			end
@@ -30,6 +30,6 @@ module Apollo
 				table = Terminal::Table.new :headings => headings, :rows => rows
 				return table
 			end
-		end
-	end # Formatter
-end # Apollo
+		end # class TableFormatter
+	end # module Formatter
+end # module Apollo

data/lib/apollo_crawler/lib.rb

@@ -2,27 +2,34 @@
 require File.join(File.dirname(__FILE__), 'program')
 
 # Caches
-require File.join(File.dirname(__FILE__), 'cache/cache_base')
+require File.join(File.dirname(__FILE__), 'cache/base_cache')
 require File.join(File.dirname(__FILE__), 'cache/factory')
-require File.join(File.dirname(__FILE__), 'cache/filesystem_cache')
 require File.join(File.dirname(__FILE__), 'cache/memcached_cache')
 require File.join(File.dirname(__FILE__), 'cache/memory_cache')
 require File.join(File.dirname(__FILE__), 'cache/null_cache')
 
 # Crawlers
-require File.join(File.dirname(__FILE__), 'crawler/crawler_base')
-require File.join(File.dirname(__FILE__), 'crawler/google_com/google')
-require File.join(File.dirname(__FILE__), 'crawler/slashdot_org/slashdot')
-require File.join(File.dirname(__FILE__), 'crawler/stackoverflow_com/stackoverflow')
-require File.join(File.dirname(__FILE__), 'crawler/xkcd_com/xkcd')
-require File.join(File.dirname(__FILE__), 'crawler/ycombinator_com/hacker_news')
+require File.join(File.dirname(__FILE__), 'crawler/base_crawler')
+require File.join(File.dirname(__FILE__), 'crawler/google_crawler')
+require File.join(File.dirname(__FILE__), 'crawler/hacker_news_crawler')
+require File.join(File.dirname(__FILE__), 'crawler/slashdot_crawler')
+require File.join(File.dirname(__FILE__), 'crawler/stackoverflow_crawler')
+require File.join(File.dirname(__FILE__), 'crawler/xkcd_crawler')
 
 # Fetchers
-require File.join(File.dirname(__FILE__), 'fetcher/fetcher_base')
+require File.join(File.dirname(__FILE__), 'fetcher/base_fetcher')
 require File.join(File.dirname(__FILE__), 'fetcher/simple_fetcher')
+require File.join(File.dirname(__FILE__), 'fetcher/smart_fetcher')
 
 # Formatters
-require File.join(File.dirname(__FILE__), 'formatter/formatter_base')
-require File.join(File.dirname(__FILE__), 'formatter/formatter_json')
-require File.join(File.dirname(__FILE__), 'formatter/formatter_plain')
-require File.join(File.dirname(__FILE__), 'formatter/formatter_table')
+require File.join(File.dirname(__FILE__), 'formatter/base_formatter')
+require File.join(File.dirname(__FILE__), 'formatter/json_formatter')
+require File.join(File.dirname(__FILE__), 'formatter/plain_formatter')
+require File.join(File.dirname(__FILE__), 'formatter/table_formatter')
+
+# Loggers
+require File.join(File.dirname(__FILE__), 'logger/base_logger')
+require File.join(File.dirname(__FILE__), 'logger/console_logger')
+
+# Stores
+require File.join(File.dirname(__FILE__), 'store/base_store')

data/lib/apollo_crawler/logger/base_logger.rb

@@ -0,0 +1,6 @@
+module Apollo
+	module Logger
+		class BaseLogger
+		end # class BaseLogger
+	end # module Logger
+end # modole Apollo

data/lib/apollo_crawler/logger/console_logger.rb

@@ -0,0 +1,15 @@
+require File.join(File.dirname(__FILE__), 'base_logger') 
+
+module Apollo
+	module Logger
+		class ConsoleLogger < BaseLogger
+			def log(msg)
+				puts msg
+			end
+
+			def self.log(msg)
+				 return Logger.log(msg)
+			end
+		end # class ConsoleLogger
+	end # module Logger
+end # module Apollo

data/lib/apollo_crawler/program.rb

@@ -125,7 +125,7 @@ module Apollo
 			config = File.join(File.dirname(__FILE__), "config", "crawler.rb")
 			if(File.exists?(config))
 				if(@options[:verbose])
-					puts "Loading config '#{config}'"
+					RbConfig::DEFAULT_LOGGER "Loading config '#{config}'"
 				end
 				
 				# puts "Let's require '#{@options[:verbose]}'"
@@ -192,7 +192,8 @@ module Apollo
 
 			tmp.each do |x| 
 				klass = Object.const_get('Apollo').const_get('Crawler').const_get(x)
-				@crawlers.merge!({ x.downcase.to_s => klass})
+				name = x.to_s.downcase.gsub(Apollo::Crawler::BaseCrawler.name_re,"")
+				@crawlers.merge!({ name => klass})
 			end
 
 			if(@options[:verbose])
@@ -225,7 +226,8 @@ module Apollo
 
 			tmp.each do |x| 
 				klass = Object.const_get('Apollo').const_get('Formatter').const_get(x)
-				@formatters.merge!({ x.downcase.to_s => klass})
+				name = x.to_s.downcase.gsub(Apollo::Formatter::BaseFormatter.name_re,"")
+				@formatters.merge!({ name => klass})
 			end
 
 			if(@options[:verbose])
@@ -266,9 +268,10 @@ module Apollo
 			end
 
 			template_path = RbConfig::CRAWLER_TEMPLATE_PATH
+			puts template_path
 			if(File.exists?(template_path) == false)
 				puts "Template file '#{template_path}' does not exists!"
-				return
+				return -1
 			end
 
 			if(@options[:verbose])
@@ -301,6 +304,8 @@ module Apollo
 					end  
 				end
 			end  
+
+			return 0
 		end
 
 		def self.console_table(headings, rows)
@@ -342,8 +347,7 @@ module Apollo
 				url = args.length > 0 ? args[0] : nil
 				matcher = args.length > 1 ? args[1] : nil
 				
-				self.generate_crawler(name, url, matcher)
-				return 0
+				return self.generate_crawler(name, url, matcher)
 			end
 
 			register_modules()
@@ -356,7 +360,8 @@ module Apollo
 
 			# Look for specified formatter
 			f = @formatters.select { |k, v|
-				k.downcase == formatter_name.downcase
+				name = formatter_name.gsub(Apollo::Formatter::BaseFormatter::name_re, "")
+				k.downcase == name
 			}
 
 			if(f)
@@ -388,7 +393,9 @@ module Apollo
 			end
 
 			crawlers.each do |crawler|
-				p = @crawlers[crawler.downcase]
+				crawler_name = crawler.downcase.gsub(Apollo::Crawler::BaseCrawler.name_re, "")
+
+				p = @crawlers[crawler_name]
 				if(p == nil)
 					puts "Invalid crawler name - '#{crawler}'"
 					puts "See program help"

data/lib/apollo_crawler/store/base_store.rb

@@ -0,0 +1,6 @@
+module Apollo
+	module Store
+		class BaseStore
+		end # class BaseStore
+	end # module Store
+end # module Apollo

data/lib/apollo_crawler/version.rb

@@ -1,3 +1,3 @@
 module Apollo
-	VERSION = '0.1.8'
+	VERSION = '0.1.9'
 end # Apollo

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: apollo-crawler
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.1.9
 platform: ruby
 authors:
 - Tomas Korcak
@@ -227,30 +227,31 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ./lib/apollo_crawler/fetcher/smart_fetcher.rb
 - ./lib/apollo_crawler/fetcher/simple_fetcher.rb
-- ./lib/apollo_crawler/fetcher/fetcher_base.rb
+- ./lib/apollo_crawler/fetcher/base_fetcher.rb
 - ./lib/apollo_crawler/lib.rb
 - ./lib/apollo_crawler/version.rb
+- ./lib/apollo_crawler/logger/console_logger.rb
+- ./lib/apollo_crawler/logger/base_logger.rb
 - ./lib/apollo_crawler/program.rb
 - ./lib/apollo_crawler/config.rb
 - ./lib/apollo_crawler/cache/factory.rb
 - ./lib/apollo_crawler/cache/null_cache.rb
-- ./lib/apollo_crawler/cache/cache_base.rb
 - ./lib/apollo_crawler/cache/memory_cache.rb
-- ./lib/apollo_crawler/cache/filesystem_cache.rb
+- ./lib/apollo_crawler/cache/base_cache.rb
 - ./lib/apollo_crawler/cache/memcached_cache.rb
-- ./lib/apollo_crawler/crawler/crawler_template.rb
-- ./lib/apollo_crawler/crawler/stackoverflow_com/stackoverflow.rb
-- ./lib/apollo_crawler/crawler/xkcd_com/xkcd.rb
-- ./lib/apollo_crawler/crawler/google_com/google.rb
-- ./lib/apollo_crawler/crawler/crawler_base.rb
-- ./lib/apollo_crawler/crawler/slashdot_org/slashdot.rb
-- ./lib/apollo_crawler/crawler/ycombinator_com/hacker_news.rb
-- ./lib/apollo_crawler/formatter/formatter_base.rb
-- ./lib/apollo_crawler/formatter/formatter_plain.rb
-- ./lib/apollo_crawler/formatter/formatter_json.rb
-- ./lib/apollo_crawler/formatter/formatter_table.rb
-- ./lib/apollo_crawler/store/store_base.rb
+- ./lib/apollo_crawler/crawler/xkcd_crawler.rb
+- ./lib/apollo_crawler/crawler/google_crawler.rb
+- ./lib/apollo_crawler/crawler/slashdot_crawler.rb
+- ./lib/apollo_crawler/crawler/hacker_news_crawler.rb
+- ./lib/apollo_crawler/crawler/base_crawler.rb
+- ./lib/apollo_crawler/crawler/stackoverflow_crawler.rb
+- ./lib/apollo_crawler/formatter/table_formatter.rb
+- ./lib/apollo_crawler/formatter/base_formatter.rb
+- ./lib/apollo_crawler/formatter/json_formatter.rb
+- ./lib/apollo_crawler/formatter/plain_formatter.rb
+- ./lib/apollo_crawler/store/base_store.rb
 - ./lib/apollo_crawler.rb
 - bin/apollo-crawler
 homepage: https://github.com/korczis/apollo-crawler

data/lib/apollo_crawler/cache/filesystem_cache.rb

@@ -1,37 +0,0 @@
-require File.join(File.dirname(__FILE__), 'cache_base') 
-
-module Apollo
-	module Cache
-		class Filesystem < CacheBase
-			def initialize
-				# puts "This if Filesystem cache"
-			end
-
-			# Get value associated with key from cache
-			def get(key, *args)
-				# Not found, Create, cache and return
-				if block_given?
-					res = yield args
-				end
-				
-				return res
-			end
-
-			# Set value associated with key 
-			# Return cached value
-			def set(key, value)
-				return value
-			end
-
-			# Check if cache contains specified key
-			def contains(key)
-				return false
-			end
-
-			# Invalidate key/value pair
-			def invalidate(key)
-				return true
-			end
-		end # Filesystem
-	end # Cache
-end # Apollo

data/lib/apollo_crawler/crawler/crawler_template.rb

@@ -1,24 +0,0 @@
-module Apollo
-	module Crawler
-		class CRAWLER_CLASS_NAME < Crawler
-			@@MATCHER_ITEM = "CRAWLER_MATCHER"
-
-			def name()
-				return "CRAWLER_NAME"
-			end
-
-			def url()
-				return "CRAWLER_URL"
-			end
-
-			def extract_data(doc)
-				res = doc.xpath(@@MATCHER_ITEM).map { |i|
-					{ 
-						:text => i.text,
-						:link => URI.join(self.url, i['href'])
-					}
-				}
-			end
-		end # CRAWLER_CLASS_NAME
-	end # Crawler
-end # Apollo

data/lib/apollo_crawler/fetcher/fetcher_base.rb

@@ -1,6 +0,0 @@
-module Apollo
-	module Fetcher
-		class FetcherBase
-		end # FetcherBase
-	end # Fetcher
-end # Apollo

data/lib/apollo_crawler/formatter/formatter_base.rb

@@ -1,6 +0,0 @@
-module Apollo
-	module Formatter
-		class FormatterBase
-		end # FormatterBase
-	end # Formatter
-end # Apollo

data/lib/apollo_crawler/formatter/formatter_plain.rb

@@ -1,17 +0,0 @@
-require 'awesome_print'
-
-require File.join(File.dirname(__FILE__), 'formatter_base') 
-
-module Apollo
-	module Formatter
-		class Plain < FormatterBase
-			def format(obj)
-				return Plain.format(obj)
-			end
-
-			def self.format(obj)
-				 return obj.inspect
-			end
-		end
-	end # Formatter
-end # Apollo

data/lib/apollo_crawler/store/store_base.rb

@@ -1,6 +0,0 @@
-module Apollo
-	module Store
-		class StoreBase
-		end # StoreBase
-	end # Store
-end # Apollo