apollo-crawler 0.1.21 → 0.1.22

data/lib/apollo_crawler/program/crawler_program.rb

@@ -45,26 +45,41 @@ require File.join(File.dirname(__FILE__), '..', 'version')
 
 require File.join(File.dirname(__FILE__),'base_program') 
 
+
+# Hack
+class String
+  def to_class
+    self.split('::').inject(Object) do |mod, class_name|
+      mod.const_get(class_name)
+    end
+  end
+end
+
 module Apollo
+	# Apollo Crawler Base Directory
+	APOLLO_CRAWLER_BASE_DIR = File.join(File.dirname(__FILE__), "..")
+
+	# Modules enabled
+	APOLLO_CRAWLER_MODULES = [
+		"agent",
+		"cache",
+		"crawler",
+		"formatter"
+	]
+
 	class CrawlerProgram < BaseProgram
 		# Load default config
 		require File.join(File.dirname(__FILE__), "..", "config")
 
 		# This hash will hold all of the options
 		# parsed from the command-line by OptionParser.
-		@caches = nil
-		@crawlers = nil
-		@formatter = nil
-		@formatters = nil
 		@options = nil
 		@optparser = nil
 
 		# Initializer - Constructor
 		def initialize
-			@caches = {}
-			@crawlers = {}
-			@formatter = RbConfig::DEFAULT_FORMATTER
-			@formatters = {}
+			super
+			
 			@options = {}
 
 			at_exit { 
@@ -72,11 +87,46 @@ module Apollo
 			}
 		end
 
+		def self.get_modules_paths(modules = APOLLO_CRAWLER_MODULES)
+			res = modules.map do |name|
+				Dir[File.join(APOLLO_CRAWLER_BASE_DIR, name, "*.rb")].each do |path|
+					path
+				end
+			end
+
+			res.flatten.sort
+		end
+
+		def self.register_modules(modules = APOLLO_CRAWLER_MODULES)
+			get_modules_paths(modules).each do |file| 
+				# puts "Adding module '#{file}'"
+				require file
+			end
+		end
+
+		# Show tabular data in form of CLI table
+		def self.console_table(headings, rows)
+			rows = rows.map do |o| 
+				i = o.new
+
+				res = []
+				headings.each do |h|
+					res << i.instance_eval(h)
+				end
+				res
+			end
+
+			table = Terminal::Table.new :headings => headings, :rows => rows
+			puts table
+		end
+
 		# Initialize command-line options
 		def init_options()
+			@options[:env] = Apollo::ENV	
+
 			@options[:doc_limit] = nil
 			@options[:verbose] = false
-			@options[:version] = false
+			@options[:version] = nil
 			
 			@options[:cache_dirs] = [
 				RbConfig::CACHES_DIR
@@ -103,13 +153,17 @@ module Apollo
 				# This displays the help screen, all programs are
 				# assumed to have this option.
 				opts.on('-h', '--help', 'Display this screen') do
-					@options[:show_help]
+					@options[:show_help] = true
 				end
 
 				opts.on('-a', '--all', 'Run all crawlers') do
 					@options[:run_all] = true
 				end	
 
+				opts.on('-e', '--environment [NAME]', "Environment used, default '#{@options[:env]}'") do |name|
+					@options[:env] = name
+				end
+
 				opts.on('-f', '--format [NAME]', "Formatter used") do |name|
 					@options[:formatter] = name
 				end
@@ -142,6 +196,10 @@ module Apollo
 					@options[:list_formatters] = true
 				end			
 
+				# opts.on('-q', '--query [QUERY]', 'Query crawler database for phrase') do |query|
+				# 	@options[:query] = query
+				# end	
+
 				opts.on('-s', '--silent', 'Silent mode - do not print processed document') do
 					@options[:silent] = true
 				end	
@@ -178,37 +236,20 @@ module Apollo
 			end
 
 			if(@options[:list_formatters])
-				list_formatters()
+				objs = Apollo::Formatter::BaseFormatter.subclasses
+				CrawlerProgram.console_table(['name', 'self.class'], objs)
 				return 0
 			end
 
 			if(@options[:list_crawlers])
-				list_crawlers()
+				objs = Apollo::Crawler::BaseCrawler.subclasses
+				CrawlerProgram.console_table(['name', 'self.class'], objs)
 				return 0
 			end
 
 			return nil
 		end
 
-		def init_formatter(formatter_name = "json")
-			# Set default formatter here
-			if(@options[:formatter])
-				formatter_name = @options[:formatter]
-			end
-
-			# Look for specified formatter
-			f = @formatters.select { |k, v|
-				name = formatter_name.gsub(Apollo::Formatter::BaseFormatter::name_re, "")
-				k.downcase == name
-			}
-
-			if(f)
-				return f[f.keys[0]] 
-			end
-
-			return nil
-		end	
-
 		# Load global options first
 		# Merge it with local options (if they exists)
 		def load_config_file(config = RbConfig::PROGRAM_CONFIG_PATH)
@@ -229,116 +270,6 @@ module Apollo
 			end
 		end
 
-		# Register caches
-		def register_cache(dir)			
-			if(@options[:verbose]) 
-				puts "Registering caches - '#{dir}'"
-			end
-
-			BaseProgram.require_files(File.join(dir, "**", "*.rb"))
-
-			tmp = Apollo::Cache.constants.select { |c| 
-				Class === Apollo::Cache.const_get(c)
-			}
-
-			tmp.each do |x| 
-				klass = Object.const_get('Apollo').const_get('Cache').const_get(x)
-				@caches.merge!({ x.downcase.to_s => klass})
-			end
-
-			if(@options[:verbose])
-				@caches.each do |cache, klass|
-					name = klass
-
-					# klass.ancestors.include?(Apollo::Caches::Cache)
-					if  name == "Apollo::Caches::Cache"
-						next
-					end
-
-					puts "Registered cache '#{cache}' -> '#{name}'"
-				end
-			end
-		end
-
-		# Register crawlers
-		def register_crawlers(dir)			
-			if(@options[:verbose]) 
-				puts "Registering crawlers - '#{dir}'"
-			end
-
-			BaseProgram.require_files(File.join(dir, "**", "*.rb"))
-			
-			tmp = Apollo::Crawler.constants.select { |c| 
-				Class === Apollo::Crawler.const_get(c)
-			}
-
-			tmp.each do |x| 
-				klass = Object.const_get('Apollo').const_get('Crawler').const_get(x)
-				name = x.to_s.downcase.gsub(Apollo::Crawler::BaseCrawler.name_re,"")
-				@crawlers.merge!({ name => klass})
-			end
-
-			if(@options[:verbose])
-				@crawlers.each do |crawler, klass|
-					name = klass.new.class.name
-
-					if  name == "Apollo::Crawler::Crawler"
-						next
-					end
-
-					puts "Registered crawler '#{crawler}' -> '#{name}'"
-				end
-			end
-		end
-
-		# Register formatters
-		def register_formatters(dir)			
-			if(@options[:verbose]) 
-				puts "Registering formatters - '#{dir}'"
-			end
-
-			BaseProgram.require_files(File.join(dir, "**", "*.rb"))
-
-			tmp = Apollo::Formatter.constants.select { |c| 
-				Class === Apollo::Formatter.const_get(c)
-			}
-
-			tmp.each do |x| 
-				klass = Object.const_get('Apollo').const_get('Formatter').const_get(x)
-				name = x.to_s.downcase.gsub(Apollo::Formatter::BaseFormatter.name_re,"")
-				@formatters.merge!({ name => klass})
-			end
-
-			if(@options[:verbose])
-				@formatters.each do |formatter, klass|
-					name = klass.new.class.name
-
-					if  name == "Apollo::Formatters::Formatter"
-						next
-					end
-
-					puts "Registered formatter '#{formatter}' -> '#{name}'"
-				end
-			end
-		end
-
-		def register_modules()
-			# Register caches which can be used
-			@options[:cache_dirs].each do |dir|
-				register_cache(dir)
-			end
-
-			# Register sites which can be crawled
-			@options[:crawler_dirs].each do |dir|
-				register_crawlers(dir)
-			end
-
-			# Register sites which can be crawled
-			@options[:formatter_dirs].each do |dir|
-				register_formatters(dir)
-			end
-		end
-
 		def generate_crawler(name, url = nil, matcher = nil, options = @options)
 			name = name.titleize.gsub(" ", "")
 
@@ -389,39 +320,26 @@ module Apollo
 			return 0
 		end
 
-		# Show tabular data in form of CLI table
-		def self.console_table(headings, rows)
-			table = Terminal::Table.new :headings => headings, :rows => rows
-			puts table
-		end
-
-		# List available crawlers
-		def list_crawlers(crawlers = @crawlers)
-			CrawlerProgram.console_table(['name', 'class'], crawlers)
-			return
-		end
+		def get_crawlers_by_name(crawlers, crawler_classes = Apollo::Crawler::BaseCrawler.subclasses)
+			res = []
+			crawlers.each do |crawler|
+				next if crawler.nil?
 
-		# List available formatters
-		def list_formatters(formatters = @formatters)
-			CrawlerProgram.console_table(['name', 'class'], formatters)
-			return
-		end
+				crawler_classes.each do |klass|
+					next if klass.nil?
 
-		def get_crawlers_by_name(crawlers, crawler_classes = @crawlers)
-			res = []
+					crawler_name = crawler.to_s.split('::').last.downcase
+					klass_name = klass.to_s.split('::').last.downcase.gsub("crawler", "")
 
-			crawlers.each do |name|
-				crawler_name = name.downcase.gsub(Apollo::Crawler::BaseCrawler.name_re, "")
+					# puts "#{crawler_name} => #{klass_name}"
 
-				crawler = crawler_classes[crawler_name]
-				if(crawler == nil)
-					next
+					if crawler_name == klass_name || crawler_name == "#{klass_name}crawler"
+						res << klass
+						break
+					end
 				end
-
-				res << crawler
 			end
-
-			return res
+			res
 		end
 
 		def run_crawlers(crawlers, args, options = @options)
@@ -434,9 +352,16 @@ module Apollo
 					:doc_limit => options[:doc_limit]
 				}
 
-				res = crawler.new.etl(args, opts) { | docs |
-					process_docs_handler(docs)
-				}
+				# Run crawlers
+				instance = crawler.new
+
+				if(args.nil? || args.empty?)
+					args = instance.url
+				end
+
+				res = instance.etl(args, opts) do | docs |
+					process_docs_handler(docs, options, Apollo::Formatter::JsonFormatter.new)
+				end
 			end
 
 			return 0
@@ -486,39 +411,61 @@ module Apollo
 			init_options()
 			init_options_parser()
 
+			CrawlerProgram.register_modules()
+
 			parse_options(args)
 
 			init_program_directory(RbConfig::PROGRAM_DIRECTORY, RbConfig::PROGRAM_DIRECTORIES)
 
 			load_config_file()
 
-			register_modules()
-
 			res = process_options(args)
 			if res != nil
 				return res
 			end
 
-			@formatter = init_formatter()
-
 			return nil
 		end
 
+		def run_query(query, options = {})
+			if(options[:verbose])
+				puts "Investigating query '#{query}'"
+			end
+
+			return 0
+		end
+
 		# Run Program
 		def run(args = ARGV)
 			res_code = init_program(args)
-			
+
 			if res_code.nil? == false
 				return request_exit(res_code)
 			end
 
+			if(@options[:verbose])
+				puts "Running environment '#{@options[:env]}'"
+			end
+
+			# Look for query
+			if(@options[:query])
+				res_code = run_query(@options[:query], @options)
+				return request_exit(res_code)
+			end
+
+			# Parse remaining arguments as crawlers
 			crawler_names = get_crawlers(args)
-			if(crawler_names.empty?)
+			if(crawler_names.nil? || crawler_names.empty?)
 				puts @optparser
 				return request_exit(0)
-			end
+			end	
 
-			crawlers = get_crawlers_by_name(crawler_names, @crawlers)
+			# Get crawlers by their names
+			crawlers = get_crawlers_by_name(crawler_names, Apollo::Crawler::BaseCrawler.subclasses)
+			if(crawlers.nil? || crawlers.empty?)
+				puts @optparser
+				return request_exit(0)
+			end	
 
 			res_code = run_crawlers(crawlers, args, @options)
 			return request_exit(res_code)
@@ -534,7 +481,7 @@ module Apollo
 			return code
 		end
 
-		def process_docs_handler(docs, options = @options, formatter = @formatter)
+		def process_docs_handler(docs, options = options, formatter)
 			if(docs.nil?)
 				return docs
 			end
@@ -554,9 +501,9 @@ module Apollo
 
 		# At Exit handler
 		def at_exit_handler()
-			if(@options[:verbose])
-				puts "Running at_exit_handler" 
-			end
+			# if(@options[:verbose])
+			# 	puts "Running at_exit_handler" 
+			# end
 
 			# TODO: Flush caches
 			# TODO: End gracefully

data/lib/apollo_crawler/program/platform_program.rb

@@ -0,0 +1,137 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require 'json'
+
+require "thor"
+
+require "open-uri"
+require "nokogiri"
+
+require "pp"
+require "optparse"
+
+require 'active_support'
+require 'active_support/inflector'
+
+require 'terminal-table'
+
+require 'eventmachine'
+require 'em-http'
+
+require 'fileutils'
+
+require 'mongoid'
+
+require File.join(File.dirname(__FILE__), '..', 'version')
+
+require File.join(File.dirname(__FILE__),'base_program')
+
+module Apollo
+	# Apollo Crawler Base Directory
+	APOLLO_PLATFORM_BASE_DIR = File.join(File.dirname(__FILE__), "..")
+
+	class PlatformProgram < BaseProgram
+		# Load default config
+		require File.join(File.dirname(__FILE__), "..", "config")
+
+		DEFAULT_OPTIONS = {
+			:version => nil
+		}
+
+		# Initializer - Constructor
+		def initialize
+			super
+			
+			self.options.merge!(DEFAULT_OPTIONS)
+		end
+
+		def init_options()
+			self.optparser = OptionParser.new do | opts |
+				opts.banner = "Usage: apollo-platform [OPTIONS]"
+
+				opts.separator ""
+      			opts.separator "Specific options:"
+
+				# This displays the help screen, all programs are
+				# assumed to have this option.
+				opts.on('-h', '--help', 'Display this screen') do
+					self.options[:show_help] = true
+				end
+
+				opts.on('-e', '--environment [NAME]', "Environment used, default '#{options[:env]}'") do |name|
+					self.options[:env] = name
+				end
+
+				opts.on('-v', '--verbose', 'Enable verbose output') do
+					self.options[:verbose] = true
+				end
+
+				opts.on('-V', '--version', 'Show version info') do
+					self.options[:version] = true
+				end
+			end
+		end
+
+		def process_options(args)
+			if(self.options[:version])
+				puts Apollo::VERSION
+				return 0
+			end
+
+			if(self.options[:show_help])
+				puts optparser
+				return 0
+			end
+
+			# Return nil, it means program can freely continue.
+			return nil
+		end
+
+		def init_mongo()
+			self.mongo = Mongo::Connection.new(self.config['mongo']['host'])
+			self.mongo_db = self.mongo.db(self.config['mongo']['db'])
+
+			if(self.options[:verbose])
+				puts "(Mongo) Connection Inited: #{self.mongo.inspect}"
+				puts "(Mongo) Database Inited: #{self.mongo_db.inspect}"
+			end
+
+			return self.mongo
+		end
+
+		# Run Program
+		def run(args = ARGV)
+			res = super(args)
+			return res unless res.nil?
+			
+			# Print classes
+			# puts Apollo::Crawler::BaseCrawler.subclasses.inspect
+
+			# Here we start
+			if(ARGV.length < 1)
+				puts optparser
+				return 0
+			end
+
+			return request_exit(res_code)
+		end
+	end
+end