apollo-crawler 0.1.21 → 0.1.22

data/lib/apollo_crawler/formatter/plain_formatter.rb

@@ -25,6 +25,10 @@ require File.join(File.dirname(__FILE__), 'base_formatter')
 module Apollo
 	module Formatter
 		class PlainFormatter < BaseFormatter
+			def name()
+				return "Plain"
+			end
+
 			def format(obj)
 				return Plain.format(obj)
 			end

data/lib/apollo_crawler/formatter/table_formatter.rb

@@ -25,6 +25,10 @@ require File.join(File.dirname(__FILE__), 'base_formatter')
 module Apollo
 	module Formatter
 		class TableFormatter < BaseFormatter
+			def name()
+				return "Table"
+			end
+
 			def format(obj)
 				return Table.format(obj)
 			end

data/lib/apollo_crawler/helper/amqp_helper.rb

@@ -0,0 +1,26 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+module Apollo
+	module Helper
+		module Amqp
+		end # Amqp
+	end # module Helper
+end # module Apollo

data/lib/apollo_crawler/helper/core_helper.rb

@@ -18,12 +18,32 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+class Class
+	def subclasses(base=self)
+		res = ObjectSpace.each_object(Class).select { |klass| klass < base }
+	end
+end
+
 module Apollo
 	module Helper
-		class CoreHelper
-			def self.name_re()
-				return /formatter$/
+		module Core
+
+			# Gets a subclasses of specified class with it's hierarchy
+			def self.get_classes(base=nil)
+				res = []
+				
+				if(base != nil)
+					res = base.subclasses
+				end
+
+				res.each do |tmp|
+					# puts tmp.inspect
+					s = self.get_classes(tmp)
+					res << s unless s.empty?
+				end
+
+				return res.flatten
 			end
-		end # class CoreHelper
+		end # Core
 	end # module Helper
 end # module Apollo

data/lib/apollo_crawler/helper/helpers.rb

@@ -1 +1,23 @@
-require File.join(File.dirname(__FILE__), 'core_helper')
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require File.join(File.dirname(__FILE__), 'amqp_helper')
+require File.join(File.dirname(__FILE__), 'core_helper')
+require File.join(File.dirname(__FILE__), 'mongo_helper')

data/lib/apollo_crawler/helper/mongo_helper.rb

@@ -0,0 +1,26 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+module Apollo
+	module Helper
+		module Mongo
+		end # Mongo
+	end # module Helper
+end # module Apollo

data/lib/apollo_crawler/lib.rb

@@ -18,12 +18,18 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+# Environment first 
+require File.join(File.dirname(__FILE__), 'env')
+
+# Adapters
+require File.join(File.dirname(__FILE__), 'adapter/adapters')
+
+# Agents
+require File.join(File.dirname(__FILE__), 'agent/agents')
+
 # Caches
 require File.join(File.dirname(__FILE__), 'cache/caches')
 
-# CLI - Command Line Interface
-require File.join(File.dirname(__FILE__), 'cli/cli')
-
 # Crawlers
 require File.join(File.dirname(__FILE__), 'crawler/crawlers')
 
@@ -39,6 +45,9 @@ require File.join(File.dirname(__FILE__), 'helper/helpers')
 # Loggers
 require File.join(File.dirname(__FILE__), 'logger/loggers')
 
+# Programs
+require File.join(File.dirname(__FILE__), 'planner/planners')
+
 # Programs
 require File.join(File.dirname(__FILE__), 'program/programs')
 

data/lib/apollo_crawler/logger/loggers.rb

@@ -1,2 +1,22 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
 require File.join(File.dirname(__FILE__), 'base_logger')
 require File.join(File.dirname(__FILE__), 'console_logger')

data/lib/apollo_crawler/planner/base_planner.rb

@@ -0,0 +1,26 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+module Apollo
+	module Planner
+		class BasePlanner		
+		end # class BasePlanner
+	end # module Planner
+end # module Apollo

data/lib/apollo_crawler/planner/planners.rb

@@ -0,0 +1,22 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require File.join(File.dirname(__FILE__), 'base_planner')
+require File.join(File.dirname(__FILE__), 'smart_planner')

data/lib/apollo_crawler/planner/smart_planner.rb

@@ -0,0 +1,28 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require File.join(File.dirname(__FILE__),'base_planner') 
+
+module Apollo
+	module Planner
+		class SmartPlanner < BasePlanner
+		end # class SmartPlanner
+	end # module Planner
+end # module Apollo

data/lib/apollo_crawler/program/base_program.rb

@@ -18,12 +18,142 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+require 'yaml'
+
 module Apollo
 	class BaseProgram
+		CONFIG_DIR = File.join(Apollo::BASE_DIR, "config")
+		
+		DEFAULT_OPTIONS = {
+			:env => Apollo::ENV,
+			:verbose => false
+		}
+
+		attr_accessor :config
+		attr_accessor :options
+		attr_accessor :optparser
+
+		attr_accessor :mongo
+		attr_accessor :mongo_db
+
+		def initialize
+			self.config = {}
+			self.options = DEFAULT_OPTIONS
+			self.optparser = nil
+
+			self.mongo = nil
+			self.mongo_db = nil
+		end
+
+		def self.get_config_path(config)
+			return File.join(CONFIG_DIR, "#{config}.yml")
+		end
+
+		def load_configs(dir = CONFIG_DIR, options = DEFAULT_OPTIONS)
+			Dir[File.join(dir, "**/*.yml")].each do |c|
+				puts "Loading config #{c}" if options[:verbose]
+				self.load_config_file(c, options)
+			end
+
+			return self.config
+		end
+
+		def load_config_file(config_file, options = DEFAULT_OPTIONS)
+			return nil unless File.exists?(config_file)
+
+			config_name = File.basename(config_file.downcase, ".yml")
+
+			res = YAML.load(File.read(config_file))
+			return nil unless res
+
+			res = {config_name => res[options[:env]]}
+			self.config.merge! res
+			
+			if(options[:verbose])
+				puts "Loaded config '#{config_file}' => #{res[config_name]}"
+			end
+
+			return res[config_name]
+		end
+
+		def load_config(config_name, options = DEFAULT_OPTIONS)
+			path = BaseProgram.get_config_path(config_name)
+			return load_config_file(path, options)
+		end
+
 		def self.require_files(files = [])
 			Dir.glob(files).each do |file|
 				require file
 			end
 		end
+
+		# Parse the options passed to command-line
+		def parse_options(args = ARGV)
+			res = []
+
+			# Parse the command-line. Remember there are two forms
+			# of the parse method. The 'parse' method simply parses
+			# ARGV, while the 'parse!' method parses ARGV and removes
+			# any options found there, as well as any parameters for
+			# the options. What's left is the list of files to resize.
+			begin
+				optparser.parse!(args)
+			rescue Exception => e
+				return nil
+			end
+
+			return res
+		end
+
+		def process_options(args)
+			return nil
+		end
+
+		def init_options()
+			return nil
+		end
+
+		# Init program
+		def init_program(args)
+			res = nil
+			res = init_options()
+
+			begin
+				res = parse_options(args)
+			rescue Exception => e
+				puts "Unable to parse options"
+				return res unless res.nil?
+			end
+
+			res = process_options(args)
+			return res unless res.nil?
+
+			confs = load_configs(BaseProgram::CONFIG_DIR, @options)
+			puts "Loaded configs => #{confs.inspect}" if @options[:verbose]
+
+			# Init Mongo Connection
+			init_mongo()
+
+			return nil
+		end
+
+		# Run Program
+		def run(args = ARGV)
+			res = init_program(args)
+			return res unless res.nil?
+
+			puts "Running environment '#{@options[:env]}'" if @options[:verbose]
+		end
+
+		def request_exit(code = 0)
+			puts "Exiting app"
+			begin
+				exit(code)
+			rescue SystemExit => e
+				# puts "rescued a SystemExit exception, reason: '#{e.to_s}'"
+			end
+
+			return code
+		end
 	end # class BaseProgram
 end # module Apollo

data/lib/apollo_crawler/program/console_program.rb

@@ -0,0 +1,177 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require 'json'
+
+require "thor"
+
+require "open-uri"
+require "nokogiri"
+
+require "pp"
+require "optparse"
+
+require 'active_support'
+require 'active_support/inflector'
+
+require 'terminal-table'
+
+require 'eventmachine'
+require 'em-http'
+
+require 'fileutils'
+
+require File.join(File.dirname(__FILE__), '..', 'version') 
+
+require File.join(File.dirname(__FILE__), 'base_program') 
+
+module Apollo
+	# Apollo Crawler Base Directory
+	APOLLO_CONSOLE_BASE_DIR = File.join(File.dirname(__FILE__), "..")
+
+	class ConsoleProgram < BaseProgram
+		# Load default config
+		require File.join(File.dirname(__FILE__), "..", "config")
+
+		# This hash will hold all of the options
+		# parsed from the command-line by OptionParser.
+		@options = nil
+		@optparser = nil
+
+		# Initializer - Constructor
+		def initialize
+			super
+			
+			@options = {}
+		end
+
+		# Initialize command-line options
+		def init_options()
+			@options[:env] = Apollo::ENV	
+
+			@options[:verbose] = false
+			@options[:version] = nil
+		end
+
+		def init_options_parser()
+			@optparser = OptionParser.new do | opts |
+				opts.banner = "Usage: apollo-console [OPTIONS]"
+
+				opts.separator ""
+      			opts.separator "Specific options:"
+
+				# This displays the help screen, all programs are
+				# assumed to have this option.
+				opts.on('-h', '--help', 'Display this screen') do
+					@options[:show_help] = true
+				end
+
+				opts.on('-e', '--environment [NAME]', "Environment used, default '#{@options[:env]}'") do |name|
+					@options[:env] = name
+				end
+
+				opts.on('-v', '--verbose', 'Enable verbose output') do
+					@options[:verbose] = true
+				end
+
+				opts.on('-V', '--version', 'Show version info') do
+					@options[:version] = true
+				end
+
+				opts.on('-s', '--silent', 'Silent mode - do not print processed document') do
+					@options[:silent] = true
+				end	
+			end
+		end
+
+		# Parse the options passed to command-line
+		def parse_options(args = ARGV)
+			# Parse the command-line. Remember there are two forms
+			# of the parse method. The 'parse' method simply parses
+			# ARGV, while the 'parse!' method parses ARGV and removes
+			# any options found there, as well as any parameters for
+			# the options. What's left is the list of files to resize.
+			@optparser.parse!(args)
+		end
+
+		def process_options(args)
+			if(@options[:version])
+				puts Apollo::VERSION
+				return 0
+			end
+
+			if(@options[:show_help])
+				puts @optparser
+				return 0
+			end
+
+			return nil
+		end
+
+
+		# Init program
+		def init_program(args)
+			init_options()
+			init_options_parser()
+
+
+			parse_options(args)
+
+			res = process_options(args)
+			if res != nil
+				return res
+			end
+
+			return nil
+		end
+
+		# Run Program
+		def run(args = ARGV)
+			res_code = init_program(args)
+
+			if res_code.nil? == false
+				return request_exit(res_code)
+			end
+
+			if(@options[:verbose])
+				puts "Running environment '#{@options[:env]}'"
+			end
+
+			# if(ARGV.length < 1)
+			# 	puts @optparser
+			# 	return 0
+			# end
+
+			# Here we start
+
+			return request_exit(res_code)
+		end
+
+		def request_exit(code = 0)
+			begin
+				exit(0)
+			rescue SystemExit => e
+				# puts "rescued a SystemExit exception, reason: '#{e.to_s}'"
+			end
+
+			return code
+		end
+	end
+end