apollo-crawler 0.1.21 → 0.1.22

data/lib/apollo_crawler/cache/sqlite_cache.rb

@@ -20,11 +20,20 @@
 
 require File.join(File.dirname(__FILE__), 'base_cache') 
 
+require 'sqlite3'
+
 module Apollo
 	module Cache
 		class SqliteCache < BaseCache
+			DEFAULT_OPTIONS = {
+				:db => "apollo-crawler.db",
+				:collection => "fetched_docs"
+			}
+
 			def initilize(options = {})
 				super(options)
+
+				@options = DEFAULT_OPTIONS.merge(options)
 			end
 
 			def get(key)

data/lib/apollo_crawler/config.rb

@@ -18,102 +18,112 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
-require File.join(File.dirname(__FILE__), 'lib')
+# See http://stackoverflow.com/questions/6233124/where-to-place-access-config-file-in-gem
+
+require 'yaml'
 
-module RbConfig
-	############################################################
-	# Program - basic settings
-	############################################################
-	
-	# Directory for storing apollo-crawler data
-	PROGRAM_DIRECTORY = File.expand_path("~/.apollo-crawler")
+require File.join(File.dirname(__FILE__), 'lib')
 
-	PROGRAM_PLUGINS_DIRECTORY = File.join(PROGRAM_DIRECTORY, "plugins")
-	PROGRAM_TEMP_DIRECTORY = File.join(PROGRAM_DIRECTORY, "tmp")
+module Apollo
+	module RbConfig
+		############################################################
+		# Program - basic settings
+		############################################################
+		
+		# Directory for storing apollo-crawler data
+		PROGRAM_DIRECTORY = File.expand_path("~/.apollo-crawler")
 
-	# Basic PROGRAM_DIRECTORY structure, lazy created
-	PROGRAM_DIRECTORIES = [
-		PROGRAM_DIRECTORY,
-		PROGRAM_PLUGINS_DIRECTORY,
-		PROGRAM_TEMP_DIRECTORY
-	]
+		PROGRAM_PLUGINS_DIRECTORY = File.join(PROGRAM_DIRECTORY, "plugins")
+		PROGRAM_TEMP_DIRECTORY = File.join(PROGRAM_DIRECTORY, "tmp")
 
+		# Basic PROGRAM_DIRECTORY structure, lazy created
+		PROGRAM_DIRECTORIES = [
+			PROGRAM_DIRECTORY,
+			PROGRAM_PLUGINS_DIRECTORY,
+			PROGRAM_TEMP_DIRECTORY
+		]
 
 
-	PROGRAM_CONFIG_PATH = File.join(RbConfig::PROGRAM_DIRECTORY, "config.rb")
 
+		PROGRAM_CONFIG_PATH = File.join(RbConfig::PROGRAM_DIRECTORY, "config.rb")
 
 
-	############################################################
-	# Caches - caches implementations
-	############################################################
-	CACHES_DIR = File.join(File.dirname(__FILE__), "caches")
 
+		############################################################
+		# Caches - caches implementations
+		############################################################
+		CACHES_DIR = File.join(File.dirname(__FILE__), "caches")
 
 
-	############################################################
-	# Cache implementation used for chaching pages retreived
-	############################################################
-	#
-	# Filesystem backend
-	# CACHE_CLASS = Apollo::Cache::FilesystemCache
-	#
-	# Memcached - expects localhost:11211
-	# CACHE_CLASS = Apollo::Cache::MemcachedCache
-	#
-	# Pure naive ruby in-memory implementation
-	# CACHE_CLASS = Apollo::Cache::MemoryCache
-	#
-	# Null caching - no caching at all
-	# CACHE_CLASS = Apollo::Cache::NullCache
 
-	# Used caching mechanism by default
-	CACHE_CLASS = Apollo::Cache::MongoCache
+		############################################################
+		# Cache implementation used for chaching pages retreived
+		############################################################
+		#
+		# Filesystem backend
+		# CACHE_CLASS = Apollo::Cache::FilesystemCache
+		#
+		# Memcached - expects localhost:11211
+		# CACHE_CLASS = Apollo::Cache::MemcachedCache
+		#
+		# Pure naive ruby in-memory implementation
+		# CACHE_CLASS = Apollo::Cache::MemoryCache
+		#
+		# Null caching - no caching at all
+		# CACHE_CLASS = Apollo::Cache::NullCache
 
-	CACHE_CLASS_OPTIONS = {
-		:host => 'apollo-crawler.no-ip.org', 
-		:port => 27017, 
-		:pool_size => 5, 
-		:pool_timeout => 5,
-		:db => 'apollo-crawler',
-		:collection => 'fetched_docs'
-	}
+		CACHE_CLASS_OPTIONS_MONGO = {
+			:host => 'apollo-crawler.no-ip.org', 
+			:port => 27017, 
+			:pool_size => 5, 
+			:pool_timeout => 5,
+			:db => 'apollo-crawler',
+			:collection => 'fetched_docs'
+		}
 
+		CACHE_CLASS_OPTIONS_SQLITE3 = {
+			:db => 'apollo-crawler-db',
+			:collection => 'fetched_docs'
+		}
 
+		# Used caching mechanism by default
+		CACHE_CLASS = Apollo::Cache::SqliteCache
+		CACHE_CLASS_OPTIONS = CACHE_CLASS_OPTIONS_MONGO
 
-	############################################################
-	# Crawlers - Built-in out-of box working crawlers
-	############################################################
-	CRAWLERS_DIR = File.join(File.dirname(__FILE__), "crawler")
-	
-	# Template used for generated crawlers
-	CRAWLER_TEMPLATE_NAME = "crawler_template.trb"
+		############################################################
+		# Crawlers - Built-in out-of box working crawlers
+		############################################################
+		CRAWLERS_DIR = File.join(File.dirname(__FILE__), "crawler")
+		
+		# Template used for generated crawlers
+		CRAWLER_TEMPLATE_NAME = "crawler_template.trb"
 
-	# Path of template
-	CRAWLER_TEMPLATE_PATH = File.join(CRAWLERS_DIR, CRAWLER_TEMPLATE_NAME)
+		# Path of template
+		CRAWLER_TEMPLATE_PATH = File.join(CRAWLERS_DIR, CRAWLER_TEMPLATE_NAME)
 
 
 
-	############################################################
-	# Fetchers - used for fetching documents
-	############################################################
-	FETCHERS_DIR = File.join(File.dirname(__FILE__), "fetcher")
-	
-	DEFAULT_FETCHER = Apollo::Fetcher::SmartFetcher
+		############################################################
+		# Fetchers - used for fetching documents
+		############################################################
+		FETCHERS_DIR = File.join(File.dirname(__FILE__), "fetcher")
+		
+		DEFAULT_FETCHER = Apollo::Fetcher::SmartFetcher
 
 
-	############################################################
-	# Formatters - used for formatting crawled documents results
-	############################################################
-	FORMATTERS_DIR = File.join(File.dirname(__FILE__), "formatter")
+		############################################################
+		# Formatters - used for formatting crawled documents results
+		############################################################
+		FORMATTERS_DIR = File.join(File.dirname(__FILE__), "formatter")
 
-	# Default formatter if no other specified
-	DEFAULT_FORMATTER = Apollo::Formatter::JsonFormatter
+		# Default formatter if no other specified
+		DEFAULT_FORMATTER = Apollo::Formatter::JsonFormatter
 
 
 
-	############################################################
-	# Loggers - used for formatting output messages
-	############################################################
-	DEFAULT_LOGGER = Apollo::Logger::ConsoleLogger
-end # Config
+		############################################################
+		# Loggers - used for formatting output messages
+		############################################################
+		DEFAULT_LOGGER = Apollo::Logger::ConsoleLogger
+	end # Config
+end

data/lib/apollo_crawler/crawler/crawlers.rb

@@ -1,3 +1,23 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
 require File.join(File.dirname(__FILE__), 'base_crawler')
 require File.join(File.dirname(__FILE__), 'google_crawler')
 require File.join(File.dirname(__FILE__), 'hacker_news_crawler')

data/lib/apollo_crawler/crawler/google_crawler.rb

@@ -35,7 +35,7 @@ module Apollo
 
 			def extract_data(doc)
 				res = doc.xpath(@@MATCHER_ITEM).map { | node |
-					url = BaseCrawler.try_get_url(self.url, node['href'])
+					url = BaseCrawler.try_get_url(self.url, node['href']).to_s
 					next if url.nil?
 
 					{ 
@@ -47,7 +47,7 @@ module Apollo
 
 			def extract_links(doc)
 				res = doc.xpath("(//td[@class = 'b']/a)[last()]").map { | node |
-					res_doc = BaseCrawler.try_get_url(self.url, node['href'])
+					res_doc = BaseCrawler.try_get_url(self.url, node['href']).to_s
 					next if url.nil?
 
 					{ 

data/lib/apollo_crawler/crawler/hacker_news_crawler.rb

@@ -35,7 +35,7 @@ module Apollo
 
 			def extract_data(doc)
 				res = doc.xpath(@@MATCHER_ITEM).map { |node|
-					url = BaseCrawler.try_get_url(self.url, node['href'])
+					url = BaseCrawler.try_get_url(self.url, node['href']).to_s
 					next if url.nil?
 
 					{ 
@@ -49,7 +49,7 @@ module Apollo
 
 			def extract_links(doc)
 				res = doc.xpath("(//td[@class = 'title']/a)[last()]").map { |node|
-					url = BaseCrawler.try_get_url(self.url, node['href'])
+					url = BaseCrawler.try_get_url(self.url, node['href']).to_s
 					next if url.nil?
 
 					{ 

data/lib/apollo_crawler/crawler/slashdot_crawler.rb

@@ -35,7 +35,7 @@ module Apollo
 
 			def extract_data(doc)
 				res = doc.xpath(@@MATCHER_ITEM).map { | node |
-					url = BaseCrawler.try_get_url(self.url, node['href'])
+					url = BaseCrawler.try_get_url(self.url, node['href']).to_s
 					next if url.nil?
 
 					{ 
@@ -47,7 +47,7 @@ module Apollo
 
 			def extract_links(doc)
 				res = doc.xpath(@@MATCHER_ITEM).map { | node |
-					url = BaseCrawler.try_get_url(self.url, node['href'])
+					url = BaseCrawler.try_get_url(self.url, node['href']).to_s
 					next if url.nil?
 
 					{ 

data/lib/apollo_crawler/crawler/stackoverflow_crawler.rb

@@ -35,7 +35,7 @@ module Apollo
 
 			def extract_data(doc)
 				res = doc.xpath(@@MATCHER_ITEM).map { |node|
-					url = BaseCrawler.try_get_url(self.url, node['href'])
+					url = BaseCrawler.try_get_url(self.url, node['href']).to_s
 					next if url.nil?
 
 					{ 
@@ -49,7 +49,7 @@ module Apollo
 
 			def extract_links(doc)
 				res = doc.xpath("(//div[@class = 'pager fl']/a)[last()]").map { |node|
-					url = BaseCrawler.try_get_url(self.url, node['href'])
+					url = BaseCrawler.try_get_url(self.url, node['href']).to_s
 					next if url.nil?
 
 					{ 

data/lib/apollo_crawler/crawler/xkcd_crawler.rb

@@ -37,7 +37,7 @@ module Apollo
 				res = doc.xpath(@@MATCHER_ITEM).map { |node|
 					{ 
 						:text => node['title'],
-						:link => URI.join(self.url, node['src']),
+						:link => URI.join(self.url, node['src']).to_s
 					}
 				}
 			end
@@ -45,7 +45,7 @@ module Apollo
 			def extract_links(doc)
 				res = doc.xpath("//ul[@class = 'comicNav']/li/a[@accesskey = 'p']").map { |node|
 					{ 
-						:link => URI.join(self.url, node['href']),
+						:link => URI.join(self.url, node['href']).to_s
 					}
 				}
 				res.uniq

data/lib/apollo_crawler/crawler/youjizz_crawler.rb

@@ -33,7 +33,7 @@ module Apollo
 
 			def extract_data(doc)
 				res = doc.xpath(@@MATCHER_ITEM).map { | node |
-					link = BaseCrawler.try_get_url(self.url, node['href'])
+					link = BaseCrawler.try_get_url(self.url, node['href']).to_s
 					next if link.nil?
 
 					{ 
@@ -45,7 +45,7 @@ module Apollo
 
 			def extract_links(doc)
 				res = doc.xpath("//div[@id = 'pagination']/a").map { | node |
-					link = BaseCrawler.try_get_url(self.url, node['href'])
+					link = BaseCrawler.try_get_url(self.url, node['href']).to_s
 					next if link.nil?
 
 					{ 

data/lib/apollo_crawler/env.rb

@@ -0,0 +1,24 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+module Apollo
+	ENV = "development"
+	BASE_DIR = File.expand_path(File.join(File.dirname(__FILE__), "..", ".."))
+end

data/lib/apollo_crawler/fetcher/base_fetcher.rb

@@ -35,7 +35,7 @@ module Apollo
 				}
 			end
 
-			def self.fetch(url)
+			def self.fetch(url, options = {})
 				uri = URI.parse(url.to_s)				
 
 				# See https://github.com/lostisland/faraday

data/lib/apollo_crawler/fetcher/fetchers.rb

@@ -1,3 +1,23 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
 require File.join(File.dirname(__FILE__), 'base_fetcher')
 require File.join(File.dirname(__FILE__), 'simple_fetcher')
 require File.join(File.dirname(__FILE__), 'smart_fetcher')

data/lib/apollo_crawler/fetcher/simple_fetcher.rb

@@ -26,7 +26,7 @@ require File.join(File.dirname(__FILE__), 'base_fetcher')
 module Apollo
 	module Fetcher
 		class SimpleFetcher < BaseFetcher
-			def self.fetch(url)
+			def self.fetch(url, options = {})
 				# TODO: Throw exception ???
 				return BaseFetcher::fetch(url)
 			end

data/lib/apollo_crawler/fetcher/smart_fetcher.rb

@@ -29,7 +29,7 @@ module Apollo
 			@@DEFAULT_SLEEP = 0.1
 			@@LAST_FETCH = nil
 
-			def self.fetch(url)
+			def self.fetch(url, options = {})
 				# TODO: Throw exception ???
 				if(@@LAST_FETCH != nil)
 					now = DateTime.now

data/lib/apollo_crawler/formatter/formatters.rb

@@ -1,3 +1,23 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
 require File.join(File.dirname(__FILE__), 'base_formatter')
 require File.join(File.dirname(__FILE__), 'json_formatter')
 require File.join(File.dirname(__FILE__), 'plain_formatter')

data/lib/apollo_crawler/formatter/json_formatter.rb

@@ -25,8 +25,12 @@ require File.join(File.dirname(__FILE__), 'base_formatter')
 module Apollo
 	module Formatter
 		class JsonFormatter < BaseFormatter
+			def name()
+				return "JSON"
+			end
+
 			def format(obj)
-				return Json.format(obj)
+				return JsonFormatter.format(obj)
 			end
 
 			def self.format(obj)