apollo-crawler 0.1.14 → 0.1.15

data/bin/apollo-crawler

@@ -21,6 +21,9 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+require "rubygems"
+require "bundler/setup"
+
 require File.join(File.dirname(__FILE__), "..", "lib", "apollo_crawler")
 
 Apollo::CrawlerProgram.new.run(ARGV)

data/lib/apollo_crawler.rb

@@ -21,40 +21,26 @@
 # TODO: Make this work - DRY!
 # require File.join(File.dirname(__FILE__), 'apollo_crawler/lib')
 
-# Main
-require File.join(File.dirname(__FILE__), 'apollo_crawler/program')
-
 # Caches
-require File.join(File.dirname(__FILE__), 'apollo_crawler/cache/base_cache')
-require File.join(File.dirname(__FILE__), 'apollo_crawler/cache/factory')
-require File.join(File.dirname(__FILE__), 'apollo_crawler/cache/memcached_cache')
-require File.join(File.dirname(__FILE__), 'apollo_crawler/cache/memory_cache')
-require File.join(File.dirname(__FILE__), 'apollo_crawler/cache/mongo_cache')
-require File.join(File.dirname(__FILE__), 'apollo_crawler/cache/null_cache')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/cache/caches')
 
 # Crawlers
-require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/base_crawler')
-require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/google_crawler')
-require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/hacker_news_crawler')
-require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/slashdot_crawler')
-require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/stackoverflow_crawler')
-require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/xkcd_crawler')
-require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/youjizz_crawler')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/crawlers')
 
 # Fetchers
-require File.join(File.dirname(__FILE__), 'apollo_crawler/fetcher/base_fetcher')
-require File.join(File.dirname(__FILE__), 'apollo_crawler/fetcher/simple_fetcher')
-require File.join(File.dirname(__FILE__), 'apollo_crawler/fetcher/smart_fetcher')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/fetcher/fetchers')
 
 # Formatters
-require File.join(File.dirname(__FILE__), 'apollo_crawler/formatter/base_formatter')
-require File.join(File.dirname(__FILE__), 'apollo_crawler/formatter/json_formatter')
-require File.join(File.dirname(__FILE__), 'apollo_crawler/formatter/plain_formatter')
-require File.join(File.dirname(__FILE__), 'apollo_crawler/formatter/table_formatter')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/formatter/formatters')
+
+# Helpers
+require File.join(File.dirname(__FILE__), 'apollo_crawler/helper/helpers')
 
 # Loggers
-require File.join(File.dirname(__FILE__), 'apollo_crawler/logger/base_logger')
-require File.join(File.dirname(__FILE__), 'apollo_crawler/logger/console_logger')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/logger/loggers')
+
+# Program
+require File.join(File.dirname(__FILE__), 'apollo_crawler/program/programs')
 
 # Stores
-require File.join(File.dirname(__FILE__), 'apollo_crawler/store/base_store')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/store/stores')

data/lib/apollo_crawler/cache/base_cache.rb

@@ -25,7 +25,7 @@ module Apollo
 			end
 
 			# Get value associated with key from cache
-			def get(key, *args)
+			def try_get(key, *args)
 
 				# Not found, Create, cache and return
 				if block_given?
@@ -41,20 +41,10 @@ module Apollo
 				return value
 			end
 
-			# Check if cache contains specified key
-			def contains(key)
-				return false
+			def remove(key)
+				# self.set(key, nil)
 			end
 
-			# Invalidate key/value pair
-			def invalidate(key)
-				return true
-			end
-
-			# Clear cache
-			def clear
-				return
-			end
 		end # class BaseCache
 	end # module Cache
 end # module Apollo

data/lib/apollo_crawler/cache/caches.rb

@@ -0,0 +1,8 @@
+require File.join(File.dirname(__FILE__), 'base_cache') 
+require File.join(File.dirname(__FILE__), 'memcached_cache') 
+require File.join(File.dirname(__FILE__), 'memory_cache') 
+require File.join(File.dirname(__FILE__), 'mongo_cache') 
+require File.join(File.dirname(__FILE__), 'null_cache') 
+require File.join(File.dirname(__FILE__), 'sqlite_cache') 
+
+require File.join(File.dirname(__FILE__), 'factory') 

data/lib/apollo_crawler/cache/memcached_cache.rb

@@ -32,9 +32,13 @@ module Apollo
 				@cache = Dalli::Client.new()
 			end
 
+			def get(key)
+				@cache.get(key)
+			end
+
 			# Get value associated with key from cache
-			def get(key, *args)
-				res = @cache.get(key)
+			def try_get(key, *args)
+				res = get(key)
 
 				# Not found, Create, cache and return
 				if res.nil? && block_given?
@@ -52,21 +56,6 @@ module Apollo
 				@cache.set(key, value)
 				return key
 			end
-
-			# Check if cache contains specified key
-			def contains(key)
-				# TODO: Implement
-			end
-
-			# Invalidate key/value pair
-			def invalidate(key)
-				# TODO: Implement
-			end
-
-			# Clear cache
-			def clear
-				# TODO: Implement
-			end
 		end # class MemcachedCache
 	end # module Cache
 end # module Apollo

data/lib/apollo_crawler/cache/memory_cache.rb

@@ -29,9 +29,13 @@ module Apollo
 				@cache = {}
 			end
 
+			def get(key)
+				@cache[key]
+			end
+
 			# Get value associated with key from cache
-			def get(key, *args)
-				res = @cache[key]
+			def try_get(key, *args)
+				res = get(key)
 
 				# Not found, Create, cache and return
 				if res.nil? && block_given?
@@ -46,21 +50,6 @@ module Apollo
 			def set(key, value)
 				@cache[key] = value
 			end
-
-			# Check if cache contains specified key
-			def contains(key)
-				@cache.has_key?(key)
-			end
-
-			# Invalidate key/value pair
-			def invalidate(key)
-				@cache.delete(key)
-			end
-
-			# Clear cache
-			def clear
-				@cache.clear
-			end
 		end # class MemoryCache
 	end # module Cache
 end # module Apollo

data/lib/apollo_crawler/cache/mongo_cache.rb

@@ -38,16 +38,19 @@ module Apollo
 				super(options)
 				
 				opts = @@DEFAULT_OPTIONS.merge(options)
-				puts opts.inspect
-
+				
 				@mongo_client = Mongo::MongoClient.new(opts[:host], opts[:port], :pool_size => opts[:pool_size], :pool_timeout => opts[:pool_timeout])
 				@db = @mongo_client[opts[:db]]
 				@coll = @db[opts[:collection]]
 			end
 
+			def get(key)
+				@coll.find({:url => key})
+			end
+
 			# Get value associated with key from cache
-			def get(key, *args)
-				res = @coll.find({:url => key})
+			def try_get(key, *args)
+				res = get(key)
 
 				# Not found, Create, cache and return
 				if res.nil? || res.count < 1 && block_given?

data/lib/apollo_crawler/cache/null_cache.rb

@@ -27,10 +27,16 @@ module Apollo
 				super(options)
 			end
 
+			def get(key)
+				nil
+			end
+
 			# Get value associated with key from cache
-			def get(key, *args)
+			def try_get(key, *args)
+				res = get(key)
+
 				# Not found, Create, cache and return
-				if block_given?
+				if res.nil? && block_given?
 					res = yield args
 				end
 				
@@ -42,16 +48,6 @@ module Apollo
 			def set(key, value)
 				return value
 			end
-
-			# Check if cache contains specified key
-			def contains(key)
-				return false
-			end
-
-			# Invalidate key/value pair
-			def invalidate(key)
-				return true
-			end
 		end # NullCache
 	end # Cache
 end # Apollo

data/lib/apollo_crawler/cache/sqlite_cache.rb

@@ -0,0 +1,53 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require File.join(File.dirname(__FILE__), 'base_cache') 
+
+module Apollo
+	module Cache
+		class SqliteCache < BaseCache
+			def initilize(options = {})
+				super(options)
+			end
+
+			def get(key)
+				return nil
+			end
+
+			# Get value associated with key from cache
+			def try_get(key, *args)
+				res = get(key)
+
+				# Not found, Create, cache and return
+				if res.nil? && block_given?
+					res = yield args
+				end
+				
+				return res
+			end
+
+			# Set value associated with key 
+			# Return cached value
+			def set(key, value)
+				return value
+			end
+		end # SqliteCache
+	end # Cache
+end # Apollo

data/lib/apollo_crawler/crawler/base_crawler.rb

@@ -112,7 +112,7 @@ module Apollo
 						# Add document to queue of results
 						res << doc
 
-						enqueue_url(doc[:links]) if doc[:links]
+						enqueue_url(doc[:links].map(){ |l| l[:link] }) if doc[:links]
 					end
 
 					# Break if limit of documents to processed was reached
@@ -130,10 +130,9 @@ module Apollo
 			def enqueue_url(url)
 				urls = []
 				return urls if url.nil?
-
 				# We support both - list of urls or single url
 				if(url.kind_of?(Array))
-					urls.concat(url)
+					urls = urls.concat(url)
 				else
 					urls << url
 				end
@@ -194,7 +193,7 @@ module Apollo
 
 				# TODO: Use some (custom-made) low-level HTTTP Protocol cache - just for sure
 				cache = Apollo::Cache::Factory.instance.construct
-				metadoc = cache.get(url) do
+				metadoc = cache.try_get(url) do
 					max_attempts = 3
 					attempt_no = 0
 					success = false

data/lib/apollo_crawler/crawler/crawlers.rb

@@ -0,0 +1,7 @@
+require File.join(File.dirname(__FILE__), 'base_crawler')
+require File.join(File.dirname(__FILE__), 'google_crawler')
+require File.join(File.dirname(__FILE__), 'hacker_news_crawler')
+require File.join(File.dirname(__FILE__), 'slashdot_crawler')
+require File.join(File.dirname(__FILE__), 'stackoverflow_crawler')
+require File.join(File.dirname(__FILE__), 'xkcd_crawler')
+require File.join(File.dirname(__FILE__), 'youjizz_crawler')

data/lib/apollo_crawler/fetcher/base_fetcher.rb

@@ -21,11 +21,12 @@
 require "open-uri"
 require "nokogiri"
 
+require "em-http-request"
+
 module Apollo
 	module Fetcher
 		class BaseFetcher
 			def self.fetch(url)
-				# TODO: Throw exception ???
 				return open(url).read
 			end			
 		end # class BaseFetcher

data/lib/apollo_crawler/fetcher/fetchers.rb

@@ -0,0 +1,3 @@
+require File.join(File.dirname(__FILE__), 'base_fetcher')
+require File.join(File.dirname(__FILE__), 'simple_fetcher')
+require File.join(File.dirname(__FILE__), 'smart_fetcher')

data/lib/apollo_crawler/formatter/formatters.rb

@@ -0,0 +1,4 @@
+require File.join(File.dirname(__FILE__), 'base_formatter')
+require File.join(File.dirname(__FILE__), 'json_formatter')
+require File.join(File.dirname(__FILE__), 'plain_formatter')
+require File.join(File.dirname(__FILE__), 'table_formatter')

data/lib/apollo_crawler/helper/core_helper.rb

@@ -0,0 +1,29 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+module Apollo
+	module Helper
+		class CoreHelper
+			def self.name_re()
+				return /formatter$/
+			end
+		end # class CoreHelper
+	end # module Helper
+end # module Apollo

data/lib/apollo_crawler/helper/helpers.rb

@@ -0,0 +1 @@
+require File.join(File.dirname(__FILE__), 'core_helper')

data/lib/apollo_crawler/lib.rb

@@ -18,40 +18,25 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
-# Main
-require File.join(File.dirname(__FILE__), 'program')
-
 # Caches
-require File.join(File.dirname(__FILE__), 'cache/base_cache')
-require File.join(File.dirname(__FILE__), 'cache/factory')
-require File.join(File.dirname(__FILE__), 'cache/memcached_cache')
-require File.join(File.dirname(__FILE__), 'cache/memory_cache')
-require File.join(File.dirname(__FILE__), 'cache/mongo_cache')
-require File.join(File.dirname(__FILE__), 'cache/null_cache')
+require File.join(File.dirname(__FILE__), 'cache/caches')
 
 # Crawlers
-require File.join(File.dirname(__FILE__), 'crawler/base_crawler')
-require File.join(File.dirname(__FILE__), 'crawler/google_crawler')
-require File.join(File.dirname(__FILE__), 'crawler/hacker_news_crawler')
-require File.join(File.dirname(__FILE__), 'crawler/slashdot_crawler')
-require File.join(File.dirname(__FILE__), 'crawler/stackoverflow_crawler')
-require File.join(File.dirname(__FILE__), 'crawler/xkcd_crawler')
-require File.join(File.dirname(__FILE__), 'crawler/youjizz_crawler')
+require File.join(File.dirname(__FILE__), 'crawler/crawlers')
 
 # Fetchers
-require File.join(File.dirname(__FILE__), 'fetcher/base_fetcher')
-require File.join(File.dirname(__FILE__), 'fetcher/simple_fetcher')
-require File.join(File.dirname(__FILE__), 'fetcher/smart_fetcher')
-
+require File.join(File.dirname(__FILE__), 'fetcher/fetchers')
 # Formatters
-require File.join(File.dirname(__FILE__), 'formatter/base_formatter')
-require File.join(File.dirname(__FILE__), 'formatter/json_formatter')
-require File.join(File.dirname(__FILE__), 'formatter/plain_formatter')
-require File.join(File.dirname(__FILE__), 'formatter/table_formatter')
+require File.join(File.dirname(__FILE__), 'formatter/formatters')
+
+# Helpers
+require File.join(File.dirname(__FILE__), 'helper/helpers')
 
 # Loggers
-require File.join(File.dirname(__FILE__), 'logger/base_logger')
-require File.join(File.dirname(__FILE__), 'logger/console_logger')
+require File.join(File.dirname(__FILE__), 'logger/loggers')
+
+# Programs
+require File.join(File.dirname(__FILE__), 'program/programs')
 
 # Stores
-require File.join(File.dirname(__FILE__), 'store/base_store')
+require File.join(File.dirname(__FILE__), 'store/stores')

data/lib/apollo_crawler/logger/loggers.rb

@@ -0,0 +1,2 @@
+require File.join(File.dirname(__FILE__), 'base_logger')
+require File.join(File.dirname(__FILE__), 'console_logger')

data/lib/apollo_crawler/program/base_program.rb

@@ -0,0 +1,29 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+module Apollo
+	class BaseProgram
+		def self.require_files(files = [])
+			Dir.glob(files).each do |file|
+				require file
+			end
+		end
+	end # class BaseProgram
+end # module Apollo

data/lib/apollo_crawler/{program.rb → program/crawler_program.rb}

@@ -18,9 +18,6 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
-require "rubygems"
-require "bundler/setup"
-
 require 'json'
 
 require "thor"
@@ -39,15 +36,17 @@ require 'terminal-table'
 require 'eventmachine'
 require 'em-http'
 
-require File.join(File.dirname(__FILE__), 'version') 
+require File.join(File.dirname(__FILE__), '..', 'version') 
 
 # require File.join(File.dirname(__FILE__), 'config/crawler')
 # puts Apollo::CrawlerProgramConfig
 
+require File.join(File.dirname(__FILE__),'base_program') 
+
 module Apollo
-	class CrawlerProgram
+	class CrawlerProgram < BaseProgram
 		# Load default config
-		require File.join(File.dirname(__FILE__), "config")
+		require File.join(File.dirname(__FILE__), "..", "config")
 
 		# This hash will hold all of the options
 		# parsed from the command-line by OptionParser.
@@ -235,10 +234,7 @@ module Apollo
 				puts "Registering caches - '#{dir}'"
 			end
 
-			files = File.join(dir, "**", "*.rb")
-			Dir.glob(files).each do |file|
-				require file
-			end
+			BaseProgram.require_files(File.join(dir, "**", "*.rb"))
 
 			tmp = Apollo::Cache.constants.select { |c| 
 				Class === Apollo::Cache.const_get(c)
@@ -269,11 +265,8 @@ module Apollo
 				puts "Registering crawlers - '#{dir}'"
 			end
 
-			files = File.join(dir, "**", "*.rb")
-			Dir.glob(files).each do |file|
-				require file
-			end
-
+			BaseProgram.require_files(File.join(dir, "**", "*.rb"))
+			
 			tmp = Apollo::Crawler.constants.select { |c| 
 				Class === Apollo::Crawler.const_get(c)
 			}
@@ -303,10 +296,7 @@ module Apollo
 				puts "Registering formatters - '#{dir}'"
 			end
 
-			files = File.join(dir, "**", "*.rb")
-			Dir.glob(files).each do |file|
-				require file
-			end
+			BaseProgram.require_files(File.join(dir, "**", "*.rb"))
 
 			tmp = Apollo::Formatter.constants.select { |c| 
 				Class === Apollo::Formatter.const_get(c)
@@ -348,7 +338,7 @@ module Apollo
 			end
 		end
 
-		def generate_crawler(name, url = nil, matcher = nil)
+		def generate_crawler(name, url = nil, matcher = nil, options = @options)
 			name = name.titleize.gsub(" ", "")
 
 			if(@options[:verbose])
@@ -362,11 +352,13 @@ module Apollo
 				return -1
 			end
 
-			if(@options[:verbose])
+			if(options[:verbose])
 				puts "Using template '#{template_path}'"
 			end
 
-			dest_path = File.join(Dir.pwd, "#{name.underscore}.rb")
+			unless(options[:silent])
+				dest_path = File.join(Dir.pwd, "#{name.underscore}.rb")
+			end
 
 			url = url ? url : "http://some-url-here"
 			matcher = matcher ? matcher : "//a"
@@ -414,23 +406,31 @@ module Apollo
 			return
 		end
 
-		def run_crawlers(crawlers, args)
+		def get_crawlers_by_name(crawlers, crawler_classes = @crawlers)
+			res = []
+
 			crawlers.each do |name|
 				crawler_name = name.downcase.gsub(Apollo::Crawler::BaseCrawler.name_re, "")
 
-				crawler = @crawlers[crawler_name]
+				crawler = crawler_classes[crawler_name]
 				if(crawler == nil)
-					puts "Invalid crawler name - '#{name}'"
-					puts "See program help"
-					return 0
+					next
 				end
 
-				if(@options[:verbose])
+				res << crawler
+			end
+
+			return res
+		end
+
+		def run_crawlers(crawlers, args, options = @options)
+			crawlers.each do |crawler|
+				if(options[:verbose])
 					puts "Running '#{crawler}'"
 				end
 
 				opts = {
-					:doc_limit => @options[:doc_limit]
+					:doc_limit => options[:doc_limit]
 				}
 
 				res = crawler.new.etl(args, opts) { | docs |
@@ -442,23 +442,23 @@ module Apollo
 		end
 
 		# Get crawlers passd to cmd-line
-		def get_crawlers(args, opts = @options)
+		def get_crawlers(args, options = @options)
 			crawlers = []
 			if(args.length > 0)
 				crawlers << args.shift
 			end
 
-			if(opts[:run_all])
+			if(options[:run_all])
 				crawlers = @crawlers.keys
 			end
 
 			return crawlers
 		end
 
-		def init_program_directory(base_dir = RbConfig::PROGRAM_DIRECTORY, dirs = RbConfig::PROGRAM_DIRECTORIES, opts = @options)			
+		def init_program_directory(base_dir = RbConfig::PROGRAM_DIRECTORY, dirs = RbConfig::PROGRAM_DIRECTORIES, options = @options)			
 			dirs.each do |dir|
 				if(File.directory?(dir) == false)
-					if(opts[:verbose])
+					if(options[:verbose])
 						puts "Creating '#{dir}'"
 					end
 
@@ -469,10 +469,10 @@ module Apollo
 			init_user_config_file(File.join(File.dirname(__FILE__), 'config_user.trb'), File.join(base_dir, 'config.rb'))			
 		end
 
-		def init_user_config_file(config_path, dest_path, opts = @options)
+		def init_user_config_file(config_path, dest_path, options = @options)
 			# Create user config file
 			if(File.exists?(config_path) && File.exists?(dest_path) == false)
-				if(opts[:verbose])
+				if(options[:verbose])
 					puts "Creating user config file '#{config_path}' => '#{dest_path}'"
 				end
 
@@ -511,13 +511,15 @@ module Apollo
 				return request_exit(res_code)
 			end
 
-			crawlers = get_crawlers(args)
-			if(crawlers.empty?)
+			crawler_names = get_crawlers(args)
+			if(crawler_names.empty?)
 				puts @optparser
 				return request_exit(0)
 			end
 
-			res_code = run_crawlers(crawlers, args)
+			crawlers = get_crawlers_by_name(crawler_names, @crawlers)
+
+			res_code = run_crawlers(crawlers, args, @options)
 			return request_exit(res_code)
 		end
 

data/lib/apollo_crawler/program/programs.rb

@@ -0,0 +1,2 @@
+require File.join(File.dirname(__FILE__), 'base_program')
+require File.join(File.dirname(__FILE__), 'crawler_program')

data/lib/apollo_crawler/store/stores.rb

@@ -0,0 +1 @@
+require File.join(File.dirname(__FILE__), 'base_store')

data/lib/apollo_crawler/version.rb

@@ -19,5 +19,5 @@
 # THE SOFTWARE.
 
 module Apollo
-	VERSION = '0.1.14'
+	VERSION = '0.1.15'
 end # Apollo

metadata

@@ -1,7 +1,8 @@
 --- !ruby/object:Gem::Specification
 name: apollo-crawler
 version: !ruby/object:Gem::Version
-  version: 0.1.14
+  version: 0.1.15
+  prerelease: 
 platform: ruby
 authors:
 - Tomas Korcak
@@ -10,51 +11,42 @@ bindir: bin
 cert_chain: []
 date: 2013-03-03 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: amqp
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: awesome_print
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.1.0
 - !ruby/object:Gem::Dependency
-  name: active_support
+  name: activesupport
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.2.12
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.2.12
 - !ruby/object:Gem::Dependency
   name: dalli
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -62,6 +54,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -69,6 +62,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: fastercsv
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -76,13 +70,15 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: 1.5.5
 - !ruby/object:Gem::Dependency
-  name: ffi
+  name: eventmachine
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -90,6 +86,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -97,6 +94,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: em-http-request
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -104,6 +102,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -111,6 +110,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: em-synchrony
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -118,41 +118,47 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: eventmachine
+  name: amqp
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.9.9
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.9.9
 - !ruby/object:Gem::Dependency
-  name: writeexcel
+  name: json
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.7.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.7.1
 - !ruby/object:Gem::Dependency
-  name: json
+  name: memcache-client
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -160,13 +166,15 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: memcached
+  name: mongo
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -174,13 +182,15 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: mongo
+  name: mongoid
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -188,13 +198,15 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: mongoid
+  name: mime-types
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -202,27 +214,31 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: mime-types
+  name: nokogiri
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.5.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.5.6
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: openurl
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -230,13 +246,15 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: openurl
+  name: parallel
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -244,69 +262,79 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: parallel
+  name: rack
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.5.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.5.2
 - !ruby/object:Gem::Dependency
   name: terminal-table
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.4.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.4.5
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.17.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.17.0
 - !ruby/object:Gem::Dependency
-  name: typhoeus
+  name: writeexcel
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.6.18
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.6.18
 - !ruby/object:Gem::Dependency
   name: guard
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -314,6 +342,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -321,6 +350,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: guard-rake
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -328,6 +358,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -335,6 +366,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: guard-rspec
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -342,6 +374,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -353,57 +386,69 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ./lib/apollo_crawler/lib.rb
+- ./lib/apollo_crawler/program/base_program.rb
+- ./lib/apollo_crawler/program/programs.rb
+- ./lib/apollo_crawler/program/crawler_program.rb
 - ./lib/apollo_crawler/fetcher/smart_fetcher.rb
 - ./lib/apollo_crawler/fetcher/simple_fetcher.rb
 - ./lib/apollo_crawler/fetcher/base_fetcher.rb
-- ./lib/apollo_crawler/lib.rb
+- ./lib/apollo_crawler/fetcher/fetchers.rb
 - ./lib/apollo_crawler/version.rb
-- ./lib/apollo_crawler/logger/console_logger.rb
-- ./lib/apollo_crawler/logger/base_logger.rb
-- ./lib/apollo_crawler/program.rb
-- ./lib/apollo_crawler/config.rb
-- ./lib/apollo_crawler/cache/factory.rb
-- ./lib/apollo_crawler/cache/null_cache.rb
-- ./lib/apollo_crawler/cache/memory_cache.rb
-- ./lib/apollo_crawler/cache/base_cache.rb
-- ./lib/apollo_crawler/cache/mongo_cache.rb
-- ./lib/apollo_crawler/cache/memcached_cache.rb
-- ./lib/apollo_crawler/crawler/xkcd_crawler.rb
-- ./lib/apollo_crawler/crawler/google_crawler.rb
-- ./lib/apollo_crawler/crawler/youjizz_crawler.rb
 - ./lib/apollo_crawler/crawler/slashdot_crawler.rb
+- ./lib/apollo_crawler/crawler/stackoverflow_crawler.rb
 - ./lib/apollo_crawler/crawler/hacker_news_crawler.rb
+- ./lib/apollo_crawler/crawler/youjizz_crawler.rb
+- ./lib/apollo_crawler/crawler/google_crawler.rb
 - ./lib/apollo_crawler/crawler/base_crawler.rb
-- ./lib/apollo_crawler/crawler/stackoverflow_crawler.rb
+- ./lib/apollo_crawler/crawler/xkcd_crawler.rb
+- ./lib/apollo_crawler/crawler/crawlers.rb
+- ./lib/apollo_crawler/logger/loggers.rb
+- ./lib/apollo_crawler/logger/console_logger.rb
+- ./lib/apollo_crawler/logger/base_logger.rb
+- ./lib/apollo_crawler/config.rb
 - ./lib/apollo_crawler/formatter/table_formatter.rb
 - ./lib/apollo_crawler/formatter/base_formatter.rb
 - ./lib/apollo_crawler/formatter/json_formatter.rb
 - ./lib/apollo_crawler/formatter/plain_formatter.rb
+- ./lib/apollo_crawler/formatter/formatters.rb
+- ./lib/apollo_crawler/cache/memcached_cache.rb
+- ./lib/apollo_crawler/cache/memory_cache.rb
+- ./lib/apollo_crawler/cache/mongo_cache.rb
+- ./lib/apollo_crawler/cache/null_cache.rb
+- ./lib/apollo_crawler/cache/factory.rb
+- ./lib/apollo_crawler/cache/caches.rb
+- ./lib/apollo_crawler/cache/sqlite_cache.rb
+- ./lib/apollo_crawler/cache/base_cache.rb
 - ./lib/apollo_crawler/store/base_store.rb
+- ./lib/apollo_crawler/store/stores.rb
+- ./lib/apollo_crawler/helper/core_helper.rb
+- ./lib/apollo_crawler/helper/helpers.rb
 - ./lib/apollo_crawler.rb
 - bin/apollo-crawler
 homepage: https://github.com/korczis/apollo-crawler
 licenses:
 - MIT
-metadata: {}
 post_install_message: Thanks for installing Apollo Crawler!
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.0
+rubygems_version: 1.8.23
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: Apollo Platform Crawler
 test_files: []

checksums.yaml

@@ -1,15 +0,0 @@
----
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    NDZmNjQ3N2FkZmVkYjc5NjQ3NjJiZTMyZGFlYjY4ODA0ZDgwYWE2OA==
-  data.tar.gz: !binary |-
-    OTA2NmJhOTEyYmJiMTFiYTFmZWFjNzY3YmEzYjYyZTc0MzZlMDk1Mw==
-!binary "U0hBNTEy":
-  metadata.gz: !binary |-
-    MDk1NDcwN2FhYjA0YTVjNTA4N2FhOWRjNzcyMTJlMDg3ZjlmNmM2NTQyMjY2
-    ZTZjNjk1YjQ2MmYxMGU2MzViNmJkYjU3OTFlMDk2MTEzMjE2MTdkMGU3NWQ4
-    ZDdlN2Y0YTc1MjQ4NmRiZDc2ZGExMTkwMmViODVkYjY2MmI1YjI=
-  data.tar.gz: !binary |-
-    MmZlM2IxNjUyMDg3Yjk3YjE5ODQ3OWM4NWY2NzIwNzEwODQ0OWJlMGI3MmQ5
-    ZmJkY2E3NDljYTJiMjhmYzMxYzY2ZTZlMTJmYTAyYjA4NWMxYTdkYmU1ZGUz
-    YzAwMzlhODY0NGQyMzNlMTc5MjQ1OTI0NzEzZmY2NmExMjA0Zjg=