apollo-crawler 0.1.5 → 0.1.6

data/lib/apollo_crawler/fetcher/fetcher_base.rb

@@ -0,0 +1,6 @@
+module Apollo
+	module Fetcher
+		class FetcherBase
+		end # FetcherBase
+	end # Fetcher
+end # Apollo

data/lib/apollo_crawler/fetcher/simple_fetcher.rb

@@ -0,0 +1,8 @@
+require File.join(File.dirname(__FILE__), 'fetcher_base') 
+
+module Apollo
+	module Fetcher
+		class SimpleFetcher < FetcherBase
+		end # Crawler
+	end # Fetcher
+end # Apollo

data/lib/apollo_crawler/formatter/formatter_base.rb

@@ -0,0 +1,6 @@
+module Apollo
+	module Formatter
+		class FormatterBase
+		end # FormatterBase
+	end # Formatter
+end # Apollo

data/lib/apollo_crawler/{formatters → formatter}/formatter_json.rb

@@ -1,17 +1,17 @@
-require 'json'
-
-require File.join(File.dirname(__FILE__), '..', 'formatter') 
-
-module Apollo
-	module Formatters
-		class Json < Formatter
-			def format(obj)
-				return Json.format(obj)
-			end
-
-			def self.format(obj)
-				 return JSON.pretty_generate(obj)
-			end
-		end
-	end # Formatters
-end # Apollo
+require 'json'
+
+require File.join(File.dirname(__FILE__), 'formatter_base') 
+
+module Apollo
+	module Formatter
+		class Json < FormatterBase
+			def format(obj)
+				return Json.format(obj)
+			end
+
+			def self.format(obj)
+				 return JSON.pretty_generate(obj)
+			end
+		end
+	end # Formatter
+end # Apollo

data/lib/apollo_crawler/{formatters → formatter}/formatter_plain.rb

@@ -1,17 +1,17 @@
-require 'awesome_print'
-
-require File.join(File.dirname(__FILE__), '..', 'formatter') 
-
-module Apollo
-	module Formatters
-		class Plain < Formatter
-			def format(obj)
-				return Plain.format(obj)
-			end
-
-			def self.format(obj)
-				 return obj.inspect
-			end
-		end
-	end # Formatters
-end # Apollo
+require 'awesome_print'
+
+require File.join(File.dirname(__FILE__), 'formatter_base') 
+
+module Apollo
+	module Formatter
+		class Plain < FormatterBase
+			def format(obj)
+				return Plain.format(obj)
+			end
+
+			def self.format(obj)
+				 return obj.inspect
+			end
+		end
+	end # Formatter
+end # Apollo

data/lib/apollo_crawler/{formatters → formatter}/formatter_table.rb

@@ -1,35 +1,35 @@
-require 'terminal-table'
-
-require File.join(File.dirname(__FILE__), '..', 'formatter') 
-
-module Apollo
-	module Formatters
-		class Table < Formatter
-			def format(obj)
-				return Table.format(obj)
-			end
-
-			def self.format(obj)
-				headings = []
-				if(obj[:data].length > 0)
-					headings = obj[:data][0].keys
-				end
-
-				rows = []
-				obj[:data].each do |line|
-					next if (line.nil? || line.empty?)
-					
-					data = []
-					headings.each do |column|
-						data << line[column]
-					end
-
-					rows << data
-				end
-				
-				table = Terminal::Table.new :headings => headings, :rows => rows
-				return table
-			end
-		end
-	end # Formatters
-end # Apollo
+require 'terminal-table'
+
+require File.join(File.dirname(__FILE__), 'formatter_base') 
+
+module Apollo
+	module Formatter
+		class Table < FormatterBase
+			def format(obj)
+				return Table.format(obj)
+			end
+
+			def self.format(obj)
+				headings = []
+				if(obj[:data].length > 0)
+					headings = obj[:data][0].keys
+				end
+
+				rows = []
+				obj[:data].each do |line|
+					next if (line.nil? || line.empty?)
+					
+					data = []
+					headings.each do |column|
+						data << line[column]
+					end
+
+					rows << data
+				end
+				
+				table = Terminal::Table.new :headings => headings, :rows => rows
+				return table
+			end
+		end
+	end # Formatter
+end # Apollo

data/lib/apollo_crawler/lib.rb

@@ -0,0 +1,28 @@
+# Main
+require File.join(File.dirname(__FILE__), 'program')
+
+# Caches
+require File.join(File.dirname(__FILE__), 'cache/cache_base')
+require File.join(File.dirname(__FILE__), 'cache/factory')
+require File.join(File.dirname(__FILE__), 'cache/filesystem_cache')
+require File.join(File.dirname(__FILE__), 'cache/memcached_cache')
+require File.join(File.dirname(__FILE__), 'cache/memory_cache')
+require File.join(File.dirname(__FILE__), 'cache/null_cache')
+
+# Crawlers
+require File.join(File.dirname(__FILE__), 'crawler/crawler_base')
+require File.join(File.dirname(__FILE__), 'crawler/google_com/google')
+require File.join(File.dirname(__FILE__), 'crawler/slashdot_org/slashdot')
+require File.join(File.dirname(__FILE__), 'crawler/stackoverflow_com/stackoverflow')
+require File.join(File.dirname(__FILE__), 'crawler/xkcd_com/xkcd')
+require File.join(File.dirname(__FILE__), 'crawler/ycombinator_com/hacker_news')
+
+# Fetchers
+require File.join(File.dirname(__FILE__), 'fetcher/fetcher_base')
+require File.join(File.dirname(__FILE__), 'fetcher/simple_fetcher')
+
+# Formatters
+require File.join(File.dirname(__FILE__), 'formatter/formatter_base')
+require File.join(File.dirname(__FILE__), 'formatter/formatter_json')
+require File.join(File.dirname(__FILE__), 'formatter/formatter_plain')
+require File.join(File.dirname(__FILE__), 'formatter/formatter_table')

data/lib/apollo_crawler/program.rb

@@ -0,0 +1,406 @@
+require "rubygems"
+require "bundler/setup"
+
+require 'json'
+
+require "thor"
+
+require "open-uri"
+require "nokogiri"
+
+require "pp"
+require "optparse"
+
+require 'active_support'
+require 'active_support/inflector'
+
+require 'terminal-table'
+
+require File.join(File.dirname(__FILE__), 'version') 
+
+# require File.join(File.dirname(__FILE__), 'config/crawler')
+# puts Apollo::CrawlerProgramConfig
+
+module Apollo
+	class CrawlerProgram
+		require File.join(File.dirname(__FILE__), "config")
+
+		# This hash will hold all of the options
+		# parsed from the command-line by OptionParser.
+		@caches = nil
+		@crawlers = nil
+		@formatter = nil
+		@formatters = nil
+		@options = nil
+		@optparser = nil
+
+		# Initializer - Constructor
+		def initialize
+			@caches = {}
+			@crawlers = {}
+			@formatter = RbConfig::DEFAULT_FORMATTER
+			@formatters = {}
+		end
+
+		# Initialize command-line options
+		def init_options
+			@options = {}
+			@options[:verbose] = false
+			@options[:version] = false
+			@options[:cache_dirs] = [
+				RbConfig::CACHES_DIR
+			]
+			@options[:crawler_dirs] = [
+				RbConfig::CRAWLERS_DIR
+			]
+			@options[:formatter_dirs] = [
+				RbConfig::FORMATTERS_DIR
+			]
+			@options[:generate_crawler] = nil
+
+			@optparser = OptionParser.new do | opts |
+				opts.banner = "Usage: apollo-crawler [OPTIONS] CRAWLER_NAME [START_URL]"
+
+				opts.separator ""
+      			opts.separator "Specific options:"
+
+				# This displays the help screen, all programs are
+				# assumed to have this option.
+				opts.on('-h', '--help', 'Display this screen') do
+					@options[:show_help]
+				end
+
+				opts.on('-a', '--all', 'Run all crawlers') do
+					@options[:run_all] = true
+				end	
+
+				opts.on('-f', '--format [NAME]', "Formatter used") do |name|
+					@options[:formatter] = name
+				end
+
+				opts.on('-g', '--generate [NAME]', "Generate scaffold for new crawler") do |name|
+					@options[:generate_crawler] = name
+				end
+
+				opts.on('-i', '--include [PATH]', 'Include additional crawler or crawler directory') do |path|
+					@options[:crawler_dirs] << path
+				end
+
+				opts.on('-v', '--verbose', 'Enable verbose output') do
+					@options[:verbose] = true
+				end
+
+				opts.on('-V', '--version', 'Show version info') do
+					@options[:version] = true
+				end
+
+				opts.on('-l', '--list-crawlers', 'List of crawlers') do
+					@options[:list_crawlers] = true
+				end
+
+				opts.on(nil, '--list-formatters', 'List of formatters available') do
+					@options[:list_formatters] = true
+				end				
+			end
+		end
+
+		# Parse the options passed to command-line
+		def parse_options(args = ARGV)
+			# Parse the command-line. Remember there are two forms
+			# of the parse method. The 'parse' method simply parses
+			# ARGV, while the 'parse!' method parses ARGV and removes
+			# any options found there, as well as any parameters for
+			# the options. What's left is the list of files to resize.
+			@optparser.parse!(args)
+		end
+
+		# Load global options first
+		# Merge it with local options (if they exists)
+		def load_config_file()
+			config = File.join(File.dirname(__FILE__), "config", "crawler.rb")
+			if(File.exists?(config))
+				if(@options[:verbose])
+					puts "Loading config '#{config}'"
+				end
+				
+				# puts "Let's require '#{@options[:verbose]}'"
+				require config
+			else
+				if(@options[:verbose])
+					# TODO: Add support for initial rake task generation
+					#          Something like this:
+					#          rake config:init # Initializes config files with
+					#            their defaults (if not exists already)        
+					puts "Default config does not exist, skipping - '#{config}'"
+				end
+			end
+		end
+
+		# Register caches
+		def register_cache(dir)			
+			if(@options[:verbose]) 
+				puts "Registering caches - '#{dir}'"
+			end
+
+			files = File.join(dir, "**", "*.rb")
+			Dir.glob(files).each do |file|
+				require file
+			end
+
+			tmp = Apollo::Cache.constants.select { |c| 
+				Class === Apollo::Cache.const_get(c)
+			}
+
+			tmp.each do |x| 
+				klass = Object.const_get('Apollo').const_get('Cache').const_get(x)
+				@caches.merge!({ x.downcase.to_s => klass})
+			end
+
+			if(@options[:verbose])
+				@caches.each do |cache, klass|
+					name = klass
+
+					# klass.ancestors.include?(Apollo::Caches::Cache)
+					if  name == "Apollo::Caches::Cache"
+						next
+					end
+
+					puts "Registered cache '#{cache}' -> '#{name}'"
+				end
+			end
+		end
+
+		# Register crawlers
+		def register_crawlers(dir)			
+			if(@options[:verbose]) 
+				puts "Registering crawlers - '#{dir}'"
+			end
+
+			files = File.join(dir, "**", "*.rb")
+			Dir.glob(files).each do |file|
+				require file
+			end
+
+			tmp = Apollo::Crawler.constants.select { |c| 
+				Class === Apollo::Crawler.const_get(c)
+			}
+
+			tmp.each do |x| 
+				klass = Object.const_get('Apollo').const_get('Crawler').const_get(x)
+				@crawlers.merge!({ x.downcase.to_s => klass})
+			end
+
+			if(@options[:verbose])
+				@crawlers.each do |crawler, klass|
+					name = klass.new.class.name
+
+					if  name == "Apollo::Crawler::Crawler"
+						next
+					end
+
+					puts "Registered crawler '#{crawler}' -> '#{name}'"
+				end
+			end
+		end
+
+		# Register formatters
+		def register_formatters(dir)			
+			if(@options[:verbose]) 
+				puts "Registering formatters - '#{dir}'"
+			end
+
+			files = File.join(dir, "**", "*.rb")
+			Dir.glob(files).each do |file|
+				require file
+			end
+
+			tmp = Apollo::Formatter.constants.select { |c| 
+				Class === Apollo::Formatter.const_get(c)
+			}
+
+			tmp.each do |x| 
+				klass = Object.const_get('Apollo').const_get('Formatter').const_get(x)
+				@formatters.merge!({ x.downcase.to_s => klass})
+			end
+
+			if(@options[:verbose])
+				@formatters.each do |formatter, klass|
+					name = klass.new.class.name
+
+					if  name == "Apollo::Formatters::Formatter"
+						next
+					end
+
+					puts "Registered formatter '#{formatter}' -> '#{name}'"
+				end
+			end
+		end
+
+		def generate_crawler(name, url = nil, matcher = nil)
+			name = name.titleize.gsub(" ", "")
+
+			if(@options[:verbose])
+				puts "Generating new crawler '#{name}'"
+			end
+
+			template_path = RbConfig::CRAWLER_TEMPLATE_PATH
+			if(File.exists?(template_path) == false)
+				puts "Template file '#{template_path}' does not exists!"
+				return
+			end
+
+			if(@options[:verbose])
+				puts "Using template '#{template_path}'"
+			end
+
+			dest_path = File.join(Dir.pwd, "#{name.underscore}.rb")
+
+			url = url ? url : "http://some-url-here"
+			matcher = matcher ? matcher : "//a"
+			
+			placeholders = {
+				"CRAWLER_CLASS_NAME" => name,
+				"CRAWLER_NAME" => name.titleize,
+				"CRAWLER_URL"  => url,
+				"CRAWLER_MATCHER" => matcher
+			}
+
+			puts "Generating crawler '#{name.titleize}', class: '#{name}', path: '#{dest_path}'"
+
+			File.open(template_path, 'r') do |tmpl|
+				File.open(dest_path, 'w') do |crawler|  
+					while line = tmpl.gets  
+						#puts line
+						placeholders.each do |k, v|
+							line.gsub!(k, v)
+						end
+						
+						crawler.puts line
+					end  
+				end
+			end  
+		end
+
+		def run(args = ARGV)
+			puts "#{ARGV.inspect}"
+
+			init_options()
+
+			parse_options(args)
+
+			if(@options[:version])
+				puts Apollo::VERSION
+				return 0
+			end
+
+			if(@options[:show_help])
+				puts @optparser
+				return 0
+			end
+
+			load_config_file()
+
+			if(@options[:generate_crawler])
+				name = @options[:generate_crawler]
+				url = args.length > 0 ? args[0] : nil
+				matcher = args.length > 1 ? args[1] : nil
+				
+				self.generate_crawler(name, url, matcher)
+				return 0
+			end
+
+			# Register caches which can be used
+			@options[:cache_dirs].each do |dir|
+				register_cache(dir)
+			end
+
+			# Register sites which can be crawled
+			@options[:crawler_dirs].each do |dir|
+				register_crawlers(dir)
+			end
+
+			# Register sites which can be crawled
+			@options[:formatter_dirs].each do |dir|
+				register_formatters(dir)
+			end
+
+			# Set default formatter here
+			formatter_name = "json"
+			if(@options[:formatter])
+				formatter_name = @options[:formatter]
+			end
+
+			# Look for specified formatter
+			f = @formatters.select { |k, v|
+				k.downcase == formatter_name.downcase
+			}
+
+			if(f)
+				@formatter = f[f.keys[0]] 
+			end
+
+			if(@options[:list_formatters])
+				headings = ['name', 'class']
+				rows = @formatters
+
+				table = Terminal::Table.new :headings => headings, :rows => rows
+				
+				puts table
+				return 0
+			end
+
+			if(@options[:list_crawlers])
+				headings = ['name', 'class']
+				rows = @crawlers
+
+				table = Terminal::Table.new :headings => headings, :rows => rows
+				
+				puts table
+				return 0
+			end
+
+			crawlers = []
+			if(args.length > 0)
+				crawlers << args.shift
+			end
+
+			if(@options[:run_all])
+				crawlers = @crawlers.keys
+			end
+
+			if(crawlers.empty?)
+				puts @optparser
+				return 0
+			end
+
+			crawlers.each do |crawler|
+				p = @crawlers[crawler.downcase]
+				if(p == nil)
+					puts "Invalid crawler name - '#{crawler}'"
+					puts "See program help"
+					return 0
+				end
+
+				if(@options[:verbose])
+					puts "Running '#{crawler}'"
+				end
+
+				res = p.new.etl(args) { | docs |
+					if(docs.nil?)
+						next
+					end
+
+					if(docs.kind_of?(Array) == false)
+						docs = [docs]
+					end
+
+					docs.each do |doc|
+						puts @formatter.format(doc)
+					end
+				}
+			end
+
+			return 0
+		end
+	end
+end