apollo-crawler 0.1.13 → 0.1.14

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    OTQxNjRiOTJiNzc5MWZkZTg1NmE0YjA2OTE3YjQ3NmM5MjBhMTA2NQ==
+    NDZmNjQ3N2FkZmVkYjc5NjQ3NjJiZTMyZGFlYjY4ODA0ZDgwYWE2OA==
   data.tar.gz: !binary |-
-    NDM0OWUwMzM3OGNkNzk5YWMwNjMwYzFlNzUwNDc2MzFlZDZhMzJiMA==
+    OTA2NmJhOTEyYmJiMTFiYTFmZWFjNzY3YmEzYjYyZTc0MzZlMDk1Mw==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    YjY4ZTNmMmIxZmUwMmE1OWMxNzVmZTQyYmQ2NTQ0MDExM2IzM2Q4MmVlYmEz
-    MmJiNzAxMDAzNGM4ODIwODViYjk4YTU5OWY2NGU1ODU2MmYxMTI0NzdiMDg5
-    NTI5OWQ3MjkzZTRjZDAzNWYxZTc5YmNmZGM3OTc3OGFlNDNkZjM=
+    MDk1NDcwN2FhYjA0YTVjNTA4N2FhOWRjNzcyMTJlMDg3ZjlmNmM2NTQyMjY2
+    ZTZjNjk1YjQ2MmYxMGU2MzViNmJkYjU3OTFlMDk2MTEzMjE2MTdkMGU3NWQ4
+    ZDdlN2Y0YTc1MjQ4NmRiZDc2ZGExMTkwMmViODVkYjY2MmI1YjI=
   data.tar.gz: !binary |-
-    M2NjZTJmYzFkNGFkNzQzOWY0ZTRhMjBmMThkYTIxYWFkNGJjYmFhZDgyMTQ4
-    ZWNlM2M1NGU2M2E3MjAwZjcyYmVmNmM0OTVmODQxMmY1ZmM5YWEyZDAwZWYy
-    YzMyYWMyNTBhYjg0YWRiZWNlMjcxNjQzOTMyMDlhYWU5ZjBjYjc=
+    MmZlM2IxNjUyMDg3Yjk3YjE5ODQ3OWM4NWY2NzIwNzEwODQ0OWJlMGI3MmQ5
+    ZmJkY2E3NDljYTJiMjhmYzMxYzY2ZTZlMTJmYTAyYjA4NWMxYTdkYmU1ZGUz
+    YzAwMzlhODY0NGQyMzNlMTc5MjQ1OTI0NzEzZmY2NmExMjA0Zjg=

data/lib/apollo_crawler/cache/base_cache.rb

@@ -21,6 +21,9 @@
 module Apollo
 	module Cache
 		class BaseCache
+			def initialize(options = {})
+			end
+
 			# Get value associated with key from cache
 			def get(key, *args)
 

data/lib/apollo_crawler/cache/factory.rb

@@ -45,7 +45,7 @@ module Apollo
 					return @cache
 				end
 
-				res = RbConfig::CACHE_CLASS.new
+				res = RbConfig::CACHE_CLASS.new(RbConfig::CACHE_CLASS_OPTIONS)
 
 				@cache = res
 				return res

data/lib/apollo_crawler/cache/memcached_cache.rb

@@ -26,7 +26,9 @@ module Apollo
 		class MemcachedCache < BaseCache
 			@cache = nil
 
-			def initialize
+			def initialize(options = {})
+				super(options)
+				
 				@cache = Dalli::Client.new()
 			end
 

data/lib/apollo_crawler/cache/memory_cache.rb

@@ -25,7 +25,7 @@ module Apollo
 		class MemoryCache < BaseCache
 			@cache = nil
 
-			def initialize
+			def initialize(options = {})
 				@cache = {}
 			end
 

data/lib/apollo_crawler/cache/mongo_cache.rb

@@ -25,14 +25,29 @@ require 'mongo'
 module Apollo
 	module Cache
 		class MongoCache < BaseCache
-			def initialize
-				@mongo_client = Mongo::MongoClient.new('localhost', 27017, :pool_size => 5, :pool_timeout => 5)
-				@db = @mongo_client['apollo-crawler']
+			@@DEFAULT_OPTIONS = {
+				:host => 'localhost', 
+				:port => 27017, 
+				:pool_size => 5, 
+				:pool_timeout => 5,
+				:db => 'apollo-crawler',
+				:collection => 'cached_docs'
+			}
+
+			def initialize(options = @@DEFAULT_OPTIONS)
+				super(options)
+				
+				opts = @@DEFAULT_OPTIONS.merge(options)
+				puts opts.inspect
+
+				@mongo_client = Mongo::MongoClient.new(opts[:host], opts[:port], :pool_size => opts[:pool_size], :pool_timeout => opts[:pool_timeout])
+				@db = @mongo_client[opts[:db]]
+				@coll = @db[opts[:collection]]
 			end
 
 			# Get value associated with key from cache
 			def get(key, *args)
-				res = @db['docs'].find({:url => key})
+				res = @coll.find({:url => key})
 
 				# Not found, Create, cache and return
 				if res.nil? || res.count < 1 && block_given?
@@ -46,7 +61,7 @@ module Apollo
 			# Set value associated with key 
 			# Return cached value
 			def set(key, value)
-				@db['docs'].insert(value)
+				@coll.insert(value)
 				return value
 			end
 		end # MongoCache

data/lib/apollo_crawler/cache/null_cache.rb

@@ -23,6 +23,10 @@ require File.join(File.dirname(__FILE__), 'base_cache')
 module Apollo
 	module Cache
 		class NullCache < BaseCache
+			def initilize(options = {})
+				super(options)
+			end
+
 			# Get value associated with key from cache
 			def get(key, *args)
 				# Not found, Create, cache and return

data/lib/apollo_crawler/config.rb

@@ -21,6 +21,29 @@
 require File.join(File.dirname(__FILE__), 'lib')
 
 module RbConfig
+	############################################################
+	# Program - basic settings
+	############################################################
+	
+	# Directory for storing apollo-crawler data
+	PROGRAM_DIRECTORY = File.expand_path("~/.apollo-crawler")
+
+	PROGRAM_PLUGINS_DIRECTORY = File.join(PROGRAM_DIRECTORY, "plugins")
+	PROGRAM_TEMP_DIRECTORY = File.join(PROGRAM_DIRECTORY, "tmp")
+
+	# Basic PROGRAM_DIRECTORY structure, lazy created
+	PROGRAM_DIRECTORIES = [
+		PROGRAM_DIRECTORY,
+		PROGRAM_PLUGINS_DIRECTORY,
+		PROGRAM_TEMP_DIRECTORY
+	]
+
+
+
+	PROGRAM_CONFIG_PATH = File.join(RbConfig::PROGRAM_DIRECTORY, "config.rb")
+
+
+
 	############################################################
 	# Caches - caches implementations
 	############################################################
@@ -45,7 +68,16 @@ module RbConfig
 	# CACHE_CLASS = Apollo::Cache::NullCache
 
 	# Used caching mechanism by default
-	CACHE_CLASS = Apollo::Cache::MongoCache
+	CACHE_CLASS = Apollo::Cache::MemoryCache
+
+	CACHE_CLASS_OPTIONS = {
+		:host => 'localhost', 
+		:port => 27017, 
+		:pool_size => 5, 
+		:pool_timeout => 5,
+		:db => 'apollo-crawler',
+		:collection => 'cached_docs'
+	}
 
 
 

data/lib/apollo_crawler/crawler/base_crawler.rb

@@ -27,9 +27,11 @@ module Apollo
 			
 
 			@backlog = nil
+			@visited = nil
 
 			def initialize
 				@backlog = []
+				@visited = []
 			end
 
 			def self.name_re()
@@ -82,12 +84,7 @@ module Apollo
 					return nil
 				end
 
-				# We support both - list of urls or single url
-				if(url.kind_of?(Array))
-					@backlog.concat(url)
-				else
-					@backlog << url
-				end
+				enqueue_url(url)
 
 				# Counter of processed documents (pages)
 				docs_processed = 0
@@ -103,6 +100,8 @@ module Apollo
 					# Increase counter of processed documents
 					docs_processed = docs_processed + 1
 
+					@visited << url
+
 					# Process document if was successfuly retreived
 					if(!doc.nil?)
 						# TODO: Use log4r and log it only on info level
@@ -113,25 +112,39 @@ module Apollo
 						# Add document to queue of results
 						res << doc
 
-						# If
-						if(doc[:links].nil? == false)
-							doc[:links].each do |link|
-								url = link[:link].to_s
-								# TODO: Use log4r and log it only on info level
-								#puts url
-
-								# TODO: Check if it is unique
-								@backlog << url
-							end
-						end
+						enqueue_url(doc[:links]) if doc[:links]
 					end
 
 					# Break if limit of documents to processed was reached
 					break if opts[:doc_limit] && docs_processed >= opts[:doc_limit]
 				end
+
+				# Return processed document
 				return res
 			end
 
+			def url_processed?(url)
+				return @backlog.include?(url) || @visited.include?(url)
+			end
+
+			def enqueue_url(url)
+				urls = []
+				return urls if url.nil?
+
+				# We support both - list of urls or single url
+				if(url.kind_of?(Array))
+					urls.concat(url)
+				else
+					urls << url
+				end
+
+				urls.each do |u|
+					if(url_processed?(u) == false)
+						@backlog << u
+					end
+				end
+			end
+
 			def process_url(url)
 				doc = self.fetch_document(url)
 				if(doc.nil?)

data/lib/apollo_crawler/fetcher/smart_fetcher.rb

@@ -26,7 +26,7 @@ require File.join(File.dirname(__FILE__), 'base_fetcher')
 module Apollo
 	module Fetcher
 		class SmartFetcher < BaseFetcher
-			@@DEFAULT_SLEEP = 1.0
+			@@DEFAULT_SLEEP = 0.1
 			@@LAST_FETCH = nil
 
 			def self.fetch(url)

data/lib/apollo_crawler/program.rb

@@ -36,6 +36,9 @@ require 'active_support/inflector'
 
 require 'terminal-table'
 
+require 'eventmachine'
+require 'em-http'
+
 require File.join(File.dirname(__FILE__), 'version') 
 
 # require File.join(File.dirname(__FILE__), 'config/crawler')
@@ -43,9 +46,6 @@ require File.join(File.dirname(__FILE__), 'version')
 
 module Apollo
 	class CrawlerProgram
-		@@PROGRAM_DIR = File.expand_path("~/.apollo-crawler")
-		@@CONFIG_PATH = File.join(@@PROGRAM_DIR, "config.rb")
-
 		# Load default config
 		require File.join(File.dirname(__FILE__), "config")
 
@@ -64,10 +64,14 @@ module Apollo
 			@crawlers = {}
 			@formatter = RbConfig::DEFAULT_FORMATTER
 			@formatters = {}
+
+			at_exit { 
+				at_exit_handler
+			}
 		end
 
 		# Initialize command-line options
-		def init_options
+		def init_options()
 			@options = {}
 			
 			@options[:doc_limit] = nil
@@ -87,7 +91,9 @@ module Apollo
 			]
 
 			@options[:generate_crawler] = nil
+		end
 
+		def init_options_parser()
 			@optparser = OptionParser.new do | opts |
 				opts.banner = "Usage: apollo-crawler [OPTIONS] CRAWLER_NAME [START_URL]"
 
@@ -184,9 +190,8 @@ module Apollo
 			return nil
 		end
 
-		def init_formatter()
+		def init_formatter(formatter_name = "json")
 			# Set default formatter here
-			formatter_name = "json"
 			if(@options[:formatter])
 				formatter_name = @options[:formatter]
 			end
@@ -198,15 +203,15 @@ module Apollo
 			}
 
 			if(f)
-				@formatter = f[f.keys[0]] 
+				return f[f.keys[0]] 
 			end
-		end
+
+			return nil
+		end	
 
 		# Load global options first
 		# Merge it with local options (if they exists)
-		def load_config_file()
-			config = @@CONFIG_PATH
-
+		def load_config_file(config = RbConfig::PROGRAM_CONFIG_PATH)
 			if(File.exists?(config))
 				if(@options[:verbose])
 					puts "Loading config '#{config}'"
@@ -398,14 +403,14 @@ module Apollo
 		end
 
 		# List available crawlers
-		def list_crawlers()
-			CrawlerProgram.console_table(['name', 'class'], @crawlers)
+		def list_crawlers(crawlers = @crawlers)
+			CrawlerProgram.console_table(['name', 'class'], crawlers)
 			return
 		end
 
 		# List available formatters
-		def list_formatters()
-			CrawlerProgram.console_table(['name', 'class'], @formatters)
+		def list_formatters(formatters = @formatters)
+			CrawlerProgram.console_table(['name', 'class'], formatters)
 			return
 		end
 
@@ -429,19 +434,7 @@ module Apollo
 				}
 
 				res = crawler.new.etl(args, opts) { | docs |
-					if(docs.nil?)
-						next
-					end
-
-					if(docs.kind_of?(Array) == false)
-						docs = [docs]
-					end
-
-					if @options[:silent] != true
-						docs.each do |doc|
-							puts @formatter.format(doc)
-						end
-					end
+					process_docs_handler(docs)
 				}
 			end
 
@@ -449,29 +442,40 @@ module Apollo
 		end
 
 		# Get crawlers passd to cmd-line
-		def get_crawlers(args)
+		def get_crawlers(args, opts = @options)
 			crawlers = []
 			if(args.length > 0)
 				crawlers << args.shift
 			end
 
-			if(@options[:run_all])
+			if(opts[:run_all])
 				crawlers = @crawlers.keys
 			end
 
 			return crawlers
 		end
 
-		def init_program_directory()
-			dir = File.expand_path("~/.apollo-crawler")
-			if(File.directory?(dir) == false)
-				FileUtils.mkpath(dir)
+		def init_program_directory(base_dir = RbConfig::PROGRAM_DIRECTORY, dirs = RbConfig::PROGRAM_DIRECTORIES, opts = @options)			
+			dirs.each do |dir|
+				if(File.directory?(dir) == false)
+					if(opts[:verbose])
+						puts "Creating '#{dir}'"
+					end
+
+					FileUtils.mkpath(dir)
+				end
 			end
 
-			config_path = File.join(File.dirname(__FILE__), 'config_user.trb')
-			dest_path = File.join(dir, 'config.rb')
+			init_user_config_file(File.join(File.dirname(__FILE__), 'config_user.trb'), File.join(base_dir, 'config.rb'))			
+		end
 
+		def init_user_config_file(config_path, dest_path, opts = @options)
+			# Create user config file
 			if(File.exists?(config_path) && File.exists?(dest_path) == false)
+				if(opts[:verbose])
+					puts "Creating user config file '#{config_path}' => '#{dest_path}'"
+				end
+
 				FileUtils.cp(config_path, dest_path)
 			end
 		end
@@ -479,10 +483,11 @@ module Apollo
 		# Init program
 		def init_program(args)
 			init_options()
+			init_options_parser()
 
 			parse_options(args)
 
-			init_program_directory()
+			init_program_directory(RbConfig::PROGRAM_DIRECTORY, RbConfig::PROGRAM_DIRECTORIES)
 
 			load_config_file()
 
@@ -493,20 +498,68 @@ module Apollo
 				return res
 			end
 
-			init_formatter()
+			@formatter = init_formatter()
+
+			return nil
 		end
 
 		# Run Program
 		def run(args = ARGV)
-			init_program(args)
+			res_code = init_program(args)
+			
+			if res_code.nil? == false
+				return request_exit(res_code)
+			end
 
 			crawlers = get_crawlers(args)
 			if(crawlers.empty?)
 				puts @optparser
-				return 0
+				return request_exit(0)
+			end
+
+			res_code = run_crawlers(crawlers, args)
+			return request_exit(res_code)
+		end
+
+		def request_exit(code = 0)
+			begin
+				exit(0)
+			rescue SystemExit => e
+				# puts "rescued a SystemExit exception, reason: '#{e.to_s}'"
+			end
+
+			return code
+		end
+
+		def process_docs_handler(docs, options = @options, formatter = @formatter)
+			if(docs.nil?)
+				return docs
 			end
 
-			return run_crawlers(crawlers, args)
+			if(docs.kind_of?(Array) == false)
+				docs = [docs]
+			end
+
+			if options[:silent] != true
+				docs.each do |doc|
+					puts formatter.format(doc)
+				end
+			end
+
+			return docs
+		end
+
+		# At Exit handler
+		def at_exit_handler()
+			if(@options[:verbose])
+				puts "Running at_exit_handler" 
+			end
+
+			# TODO: Flush caches
+			# TODO: End gracefully
+
+			# Force exit event machine
+			# EventMachine.stop
 		end
 	end
 end

data/lib/apollo_crawler/version.rb

@@ -19,5 +19,5 @@
 # THE SOFTWARE.
 
 module Apollo
-	VERSION = '0.1.13'
+	VERSION = '0.1.14'
 end # Apollo

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: apollo-crawler
 version: !ruby/object:Gem::Version
-  version: 0.1.13
+  version: 0.1.14
 platform: ruby
 authors:
 - Tomas Korcak
@@ -94,6 +94,34 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: em-http-request
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: em-synchrony
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: eventmachine
   requirement: !ruby/object:Gem::Requirement
@@ -262,6 +290,20 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: typhoeus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: guard
   requirement: !ruby/object:Gem::Requirement