apollo-crawler 0.1.5 → 0.1.6

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NGY5ZWY3NDQwYjlkNmY2Yjk2OTFmOTc2MDFmMGFjMmE5YjNkMThlNg==
+    ZjAyNjFlMDZkZTI3NjNjZjI0MjZjZmUwNjY5ZTIwM2MwMzBhNTA3NA==
   data.tar.gz: !binary |-
-    OGExM2U1ZWZiNmQ5Y2U5OWYxOTYzMTMzYWQ2MzBiMmRmOTAzODE3ZQ==
+    ZWY0YzI5ZjMxZmNkNGI0Y2FlYWI2ODZmZGQzOWUwNzI0OTU3NjcyMg==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    ZmFlZDI2NGUzNjI3MTgzNGI3Y2ViMGI2MGIzYzVkY2VkZmMxN2Y0YmQ1Njg1
-    ZGI4Yjg2ZDdkNTNjZWM5NGE5Mjc0MTBlYTA3NDA1MDNjMTNlOGRkNTkzZWUw
-    NGQzMjA3ZTA2ZjA4YmNhMTc4OGQ3NThhMDk5OWQ2MDM3NDgxYTU=
+    MmQ0YjM1NzZhZDk3NWM5ODBlMmNlMzVlYjE0MGRlZTM5NjQ0MWI0ZWJlZDI0
+    YzcwMGE3Zjc0NzBlMTAzZmY1MWFhNzhkMzdiNTdlZDYyM2I5Y2FhM2IzNjE5
+    MjAzZDE1ZjUyNWE3ZGU1YWYzZTJmYWYwZjAxZjI2YmRiYjY2ZWY=
   data.tar.gz: !binary |-
-    ODE1MjQ1ZjU2ZjBlODgwN2UzYTE4ZmEwZjRhZGMxMmY3YjU5OTk4ZTNhOTMz
-    ZjEwN2JhMjY1Nzc5NTdiMzMzZmU1NTg2NTAyZWMxNjcwZDNiODQzNGVkNmY3
-    MzQ2MWE0NzI5OTRhYTliODQ3ZWE2YzljYzk5NjFjYzI3MDM4NWU=
+    OGNkYWRkNGNlMmI0ZDhmNjgzMWY4ZjUyNTBhZWZiNDlhYWUzZDRmZmFkYzU4
+    NTIwMmZjNzE0OGQ2Yzg3M2M3YjExMzg3YjhkNGVhMjg4MjAzY2MzZTg4N2Y2
+    MzEwM2UyMGZlZDRlOGIxMjFmOTA3YzA4NjgzZGYwNTVkODEzZmQ=

data/bin/apollo-crawler

@@ -1,410 +1,12 @@
-#! /usr/bin/env ruby
-
-# encoding: utf-8
-
-require "rubygems"
-require "bundler/setup"
-
-require 'json'
-
-require "thor"
-
-require "open-uri"
-require "nokogiri"
-
-require "pp"
-require "optparse"
-
-require 'active_support'
-require 'active_support/inflector'
-
-require 'terminal-table'
-
-require File.join(File.dirname(__FILE__), '..', 'lib', 'apollo_crawler', 'version') 
-
-module Apollo
-	class CrawlerProgram
-		@@CACHES_DIR = File.join(File.dirname(__FILE__), "..", "lib", "apollo_crawler", "caches")
-		@@CRAWLERS_DIR = File.join(File.dirname(__FILE__), "..", "lib", "apollo_crawler", "crawlers")
-		@@FORMATTERS_DIR = File.join(File.dirname(__FILE__), "..", "lib", "apollo_crawler", "formatters")
-		@@CRAWLER_TEMPLATE_NAME = "crawler_template.rb"
-
-		# This hash will hold all of the options
-		# parsed from the command-line by
-		# OptionParser.
-		@options = nil
-		@optparser = nil
-		@caches = nil
-		@crawlers = nil
-		@formatters = nil
-		@formatter = nil
-
-		# Initializer - Constructor
-		def initialize
-			@caches = {}
-			@crawlers = {}
-			@formatters = {}
-		end
-
-		# Initialize command-line options
-		def init_options
-			@options = {}
-			@options[:verbose] = false
-			@options[:version] = false
-			@options[:cache_dirs] = [
-				@@CACHES_DIR
-			]
-			@options[:crawler_dirs] = [
-				@@CRAWLERS_DIR
-			]
-			@options[:formatter_dirs] = [
-				@@FORMATTERS_DIR
-			]
-			@options[:generate_crawler] = nil
-
-			@optparser = OptionParser.new do | opts |
-				opts.banner = "Usage: apollo-crawler [OPTIONS] CRAWLER_NAME [START_URL]"
-
-				opts.separator ""
-      			opts.separator "Specific options:"
-
-				# This displays the help screen, all programs are
-				# assumed to have this option.
-				opts.on('-h', '--help', 'Display this screen') do
-					puts opts
-					exit
-				end
-
-				opts.on('-a', '--all', 'Run all crawlers') do
-					@options[:run_all] = true
-				end	
-
-				opts.on('-f', '--format [NAME]', "Formatter used") do |name|
-					@options[:formatter] = name
-				end
-
-				opts.on('-g', '--generate [NAME]', "Generate scaffold for new crawler") do |name|
-					@options[:generate_crawler] = name
-				end
-
-				opts.on('-i', '--include [PATH]', 'Include additional crawler or crawler directory') do |path|
-					@options[:crawler_dirs] << path
-				end
-
-				opts.on('-v', '--verbose', 'Enable verbose output') do
-					@options[:verbose] = true
-				end
-
-				opts.on('-V', '--version', 'Show version info') do
-					@options[:version] = true
-				end
-
-				opts.on('-l', '--list-crawlers', 'List of crawlers') do
-					@options[:list_crawlers] = true
-				end
-
-				opts.on(nil, '--list-formatters', 'List of formatters available') do
-					@options[:list_formatters] = true
-				end				
-			end
-		end
-
-		# Parse the options passed to command-line
-		def parse_options
-			# Parse the command-line. Remember there are two forms
-			# of the parse method. The 'parse' method simply parses
-			# ARGV, while the 'parse!' method parses ARGV and removes
-			# any options found there, as well as any parameters for
-			# the options. What's left is the list of files to resize.
-			@optparser.parse!
-		end
-
-		# Load global options first
-		# Merge it with local options (if they exists)
-		def load_config_file()
-			config = File.join(File.dirname(__FILE__), "config", "crawler.rb")
-			if(File.exists?(config))
-				if(@options[:verbose])
-					puts "Loading config '#{config}'"
-				end
-				
-				# puts "Let's require '#{@options[:verbose]}'"
-				require config
-			else
-				if(@options[:verbose])
-					# TODO: Add support for initial rake task generation
-					#          Something like this:
-					#          rake config:init # Initializes config files with
-					#            their defaults (if not exists already)        
-					puts "Default config does not exist, skipping - '#{config}'"
-				end
-			end
-		end
-
-		# Register caches
-		def register_cache(dir)			
-			if(@options[:verbose]) 
-				puts "Registering caches - '#{dir}'"
-			end
-
-			files = File.join(dir, "**", "*.rb")
-			Dir.glob(files).each do |file|
-				require file
-			end
-
-			tmp = Apollo::Caches.constants.select { |c| 
-				Class === Apollo::Caches.const_get(c)
-			}
-
-			tmp.each do |x| 
-				klass = Object.const_get('Apollo').const_get('Caches').const_get(x)
-				@caches.merge!({ x.downcase.to_s => klass})
-			end
-
-			if(@options[:verbose])
-				@caches.each do |cache, klass|
-					name = klass
-
-					# klass.ancestors.include?(Apollo::Caches::Cache)
-					if  name == "Apollo::Caches::Cache"
-						next
-					end
-
-					puts "Registered cache '#{cache}' -> '#{name}'"
-				end
-			end
-		end
-
-		# Register crawlers
-		def register_crawlers(dir)			
-			if(@options[:verbose]) 
-				puts "Registering crawlers - '#{dir}'"
-			end
-
-			files = File.join(dir, "**", "*.rb")
-			Dir.glob(files).each do |file|
-				require file
-			end
-
-			tmp = Apollo::Crawlers.constants.select { |c| 
-				Class === Apollo::Crawlers.const_get(c)
-			}
-
-			tmp.each do |x| 
-				klass = Object.const_get('Apollo').const_get('Crawlers').const_get(x)
-				@crawlers.merge!({ x.downcase.to_s => klass})
-			end
-
-			if(@options[:verbose])
-				@crawlers.each do |crawler, klass|
-					name = klass.new.class.name
-
-					if  name == "Apollo::Crawlers::Crawler"
-						next
-					end
-
-					puts "Registered crawler '#{crawler}' -> '#{name}'"
-				end
-			end
-		end
-
-		# Register formatters
-		def register_formatters(dir)			
-			if(@options[:verbose]) 
-				puts "Registering formatters - '#{dir}'"
-			end
-
-			files = File.join(dir, "**", "*.rb")
-			Dir.glob(files).each do |file|
-				require file
-			end
-
-			tmp = Apollo::Formatters.constants.select { |c| 
-				Class === Apollo::Formatters.const_get(c)
-			}
-
-			tmp.each do |x| 
-				klass = Object.const_get('Apollo').const_get('Formatters').const_get(x)
-				@formatters.merge!({ x.downcase.to_s => klass})
-			end
-
-			if(@options[:verbose])
-				@formatters.each do |formatter, klass|
-					name = klass.new.class.name
-
-					if  name == "Apollo::Formatters::Formatter"
-						next
-					end
-
-					puts "Registered formatter '#{formatter}' -> '#{name}'"
-				end
-			end
-		end
-
-		def generate_crawler(name, url = nil, matcher = nil)
-			name = name.titleize.gsub(" ", "")
-
-			if(@options[:verbose])
-				puts "Generating new crawler '#{name}'"
-			end
-
-			template_path = File.join(File.dirname(__FILE__), '..', 'lib', 'apollo_crawler', @@CRAWLER_TEMPLATE_NAME)
-			if(File.exists?(template_path) == false)
-				puts "Template file '#{template_path}' does not exists!"
-				return
-			end
-
-			if(@options[:verbose])
-				puts "Using template '#{template_path}'"
-			end
-
-			dest_path = File.join(Dir.pwd, "#{name.underscore}.rb")
-
-			url = url ? url : "http://some-url-here"
-			matcher = matcher ? matcher : "//a"
-			
-			placeholders = {
-				"CRAWLER_CLASS_NAME" => name,
-				"CRAWLER_NAME" => name.titleize,
-				"CRAWLER_URL"  => url,
-				"CRAWLER_MATCHER" => matcher
-			}
-
-			puts "Generating crawler '#{name.titleize}', class: '#{name}', path: '#{dest_path}'"
-
-			File.open(template_path, 'r') do |tmpl|
-				File.open(dest_path, 'w') do |crawler|  
-					while line = tmpl.gets  
-						#puts line
-						placeholders.each do |k, v|
-							line.gsub!(k, v)
-						end
-						
-						crawler.puts line
-					end  
-				end
-			end  
-		end
-
-		def run
-			init_options()
-
-			parse_options()
-
-			if(@options[:version])
-				puts Apollo::VERSION
-				exit
-			end
-
-			load_config_file()
-
-			if(@options[:generate_crawler])
-				name = @options[:generate_crawler]
-				url = ARGV.length > 0 ? ARGV[0] : nil
-				matcher = ARGV.length > 1 ? ARGV[1] : nil
-				
-				self.generate_crawler(name, url, matcher)
-				exit
-			end
-
-			# Register caches which can be used
-			@options[:cache_dirs].each do |dir|
-				register_cache(dir)
-			end
-
-			# Register sites which can be crawled
-			@options[:crawler_dirs].each do |dir|
-				register_crawlers(dir)
-			end
-
-			# Register sites which can be crawled
-			@options[:formatter_dirs].each do |dir|
-				register_formatters(dir)
-			end
-
-			# Set default formatter here
-			formatter_name = "json"
-			if(@options[:formatter])
-				formatter_name = @options[:formatter]
-			end
-
-			# Look for specified formatter
-			f = @formatters.select { |k, v|
-				k.downcase == formatter_name.downcase
-			}
-
-			if(f)
-				@formatter = f[f.keys[0]] 
-			end
-
-			if(@options[:list_formatters])
-				headings = ['name', 'class']
-				rows = @formatters
-
-				table = Terminal::Table.new :headings => headings, :rows => rows
-				
-				puts table
-				return
-			end
-
-			if(@options[:list_crawlers])
-				headings = ['name', 'class']
-				rows = @crawlers
-
-				table = Terminal::Table.new :headings => headings, :rows => rows
-				
-				puts table
-				return
-			end
-
-
-
-			crawlers = []
-			if(ARGV.length > 0)
-				crawlers << ARGV.shift
-			end
-
-			if(@options[:run_all])
-				crawlers = @crawlers.keys
-			end
-
-			if(crawlers.empty?)
-				puts @optparser
-				exit
-			end
-
-			crawlers.each do |crawler|
-				p = @crawlers[crawler.downcase]
-				if(p == nil)
-					puts "Invalid crawler name - '#{crawler}'"
-					puts "See program help"
-					next
-				end
-
-				if(@options[:verbose])
-					puts "Running '#{crawler}'"
-				end
-
-				res = p.new.etl(ARGV) { | docs |
-					if(docs.nil?)
-						next
-					end
-
-					if(docs.kind_of?(Array) == false)
-						docs = [docs]
-					end
-
-					docs.each do |doc|
-						puts @formatter.format(doc)
-					end
-				}
-			end
-		end
-	end
-end
-
-if __FILE__ == $0
-	Apollo::CrawlerProgram.new.run()
-else
-	Apollo::CrawlerProgram.new.run()
-end
+#! /usr/bin/env ruby
+
+# encoding: utf-8
+
+require File.join(File.dirname(__FILE__), "..", "lib", "apollo_crawler")
+
+Apollo::CrawlerProgram.new.run(ARGV)
+
+# TODO: Maybe consider enabling this?
+# if __FILE__ == $0
+#	Apollo::CrawlerProgram.new.run(ARGV)
+# end

data/lib/apollo_crawler.rb

@@ -1,20 +1,31 @@
-# Main
-require 'apollo_crawler/cache'
-require 'apollo_crawler/crawler'
-require 'apollo_crawler/formatter'
-
-# Caches
-require 'apollo_crawler/caches/factory'
-require 'apollo_crawler/caches/memory_cache'
-require 'apollo_crawler/caches/null_cache'
-
-# Crawlers
-require 'apollo_crawler/crawlers/alexa_com/alexa'
-require 'apollo_crawler/crawlers/firmy_cz/firmy'
-require 'apollo_crawler/crawlers/slashdot_org/slashdot'
-require 'apollo_crawler/crawlers/ycombinator_com/hacker_news'
-
-# Formatters
-require 'apollo_crawler/formatters/formatter_json'
-require 'apollo_crawler/formatters/formatter_plain'
-require 'apollo_crawler/formatters/formatter_table'
+# TODO: Make this work - DRY!
+# require File.join(File.dirname(__FILE__), 'apollo_crawler/lib')
+
+# Main
+require File.join(File.dirname(__FILE__), 'apollo_crawler/program')
+
+# Caches
+require File.join(File.dirname(__FILE__), 'apollo_crawler/cache/cache_base')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/cache/factory')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/cache/filesystem_cache')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/cache/memcached_cache')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/cache/memory_cache')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/cache/null_cache')
+
+# Crawlers
+require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/crawler_base')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/google_com/google')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/slashdot_org/slashdot')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/stackoverflow_com/stackoverflow')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/xkcd_com/xkcd')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/ycombinator_com/hacker_news')
+
+# Fetchers
+require File.join(File.dirname(__FILE__), 'apollo_crawler/fetcher/fetcher_base')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/fetcher/simple_fetcher')
+
+# Formatters
+require File.join(File.dirname(__FILE__), 'apollo_crawler/formatter/formatter_base')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/formatter/formatter_json')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/formatter/formatter_plain')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/formatter/formatter_table')