apollo-crawler 0.1.4 → 0.1.5

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MzZkOTQ1YzViYTM2ODQwMjNlNzdmNjAwMmYzNTRkZGZkOWNmMmE5ZQ==
+    NGY5ZWY3NDQwYjlkNmY2Yjk2OTFmOTc2MDFmMGFjMmE5YjNkMThlNg==
   data.tar.gz: !binary |-
-    MGM1ZDM4YmUxODI1NWE3M2RkYTg5Nzk5MWVlZjY0NzljNWQ2YmZkOA==
+    OGExM2U1ZWZiNmQ5Y2U5OWYxOTYzMTMzYWQ2MzBiMmRmOTAzODE3ZQ==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    Yjk1NWU1OWE5ZmE3MzBjYzE5YjM0MjgxMzJhODRkMTFkOWQ3OGMyM2ExMDBl
-    N2RhN2E0YzUyMTU0MDVmMmY3NGFhZDU2OGNlYjcxYTFmYjc0YTY5NTNlOTVi
-    MzA0YzJlZmQ1OTA2NWQ3Mzg1MTJiNWQ3NDU0MDBjMTU0YWE3MTc=
+    ZmFlZDI2NGUzNjI3MTgzNGI3Y2ViMGI2MGIzYzVkY2VkZmMxN2Y0YmQ1Njg1
+    ZGI4Yjg2ZDdkNTNjZWM5NGE5Mjc0MTBlYTA3NDA1MDNjMTNlOGRkNTkzZWUw
+    NGQzMjA3ZTA2ZjA4YmNhMTc4OGQ3NThhMDk5OWQ2MDM3NDgxYTU=
   data.tar.gz: !binary |-
-    NjY1ZWJhMTcwY2VjNzllNjliZWZiNmM2YWIyMGRlYTRlZDdkZjI2OWVmNmYz
-    MGU1MGFjNTM1OTU0ZWQwMDk4YWNmNTYzYzcwZjUwZjI1MTIwZTA5YjRhODZm
-    Y2UwZjUyZjVjNTQ2MWE2MzdiZDk3MjJlYjQ3YjRjYWFiNTk2ODU=
+    ODE1MjQ1ZjU2ZjBlODgwN2UzYTE4ZmEwZjRhZGMxMmY3YjU5OTk4ZTNhOTMz
+    ZjEwN2JhMjY1Nzc5NTdiMzMzZmU1NTg2NTAyZWMxNjcwZDNiODQzNGVkNmY3
+    MzQ2MWE0NzI5OTRhYTliODQ3ZWE2YzljYzk5NjFjYzI3MDM4NWU=

data/bin/apollo-crawler

@@ -1,405 +1,410 @@
-#! /usr/bin/env ruby
-
-# encoding: utf-8
-
-require "rubygems"
-require "bundler/setup"
-
-require 'json'
-
-require "thor"
-
-require "open-uri"
-require "nokogiri"
-
-require "pp"
-require "optparse"
-
-require 'active_support'
-require 'active_support/inflector'
-
-require 'terminal-table'
-
-require File.join(File.dirname(__FILE__), '..', 'lib', 'apollo_crawler', 'version') 
-
-module Apollo
-	class CrawlerProgram
-		@@CACHES_DIR = File.join(File.dirname(__FILE__), "..", "lib", "apollo_crawler", "caches")
-		@@CRAWLERS_DIR = File.join(File.dirname(__FILE__), "..", "lib", "apollo_crawler", "crawlers")
-		@@FORMATTERS_DIR = File.join(File.dirname(__FILE__), "..", "lib", "apollo_crawler", "formatters")
-		@@CRAWLER_TEMPLATE_NAME = "crawler_template.rb"
-
-		# This hash will hold all of the options
-		# parsed from the command-line by
-		# OptionParser.
-		@options = nil
-		@optparser = nil
-		@caches = nil
-		@crawlers = nil
-		@formatters = nil
-		@formatter = nil
-
-		# Initializer - Constructor
-		def initialize
-			@caches = {}
-			@crawlers = {}
-			@formatters = {}
-		end
-
-		# Initialize command-line options
-		def init_options
-			@options = {}
-			@options[:verbose] = false
-			@options[:version] = false
-			@options[:cache_dirs] = [
-				@@CACHES_DIR
-			]
-			@options[:crawler_dirs] = [
-				@@CRAWLERS_DIR
-			]
-			@options[:formatter_dirs] = [
-				@@FORMATTERS_DIR
-			]
-			@options[:generate_crawler] = nil
-
-			@optparser = OptionParser.new do | opts |
-				# This displays the help screen, all programs are
-				# assumed to have this option.
-				opts.on('-h', '--help', 'Display this screen') do
-					puts opts
-					exit
-				end
-
-				opts.on('-a', '--all', 'Run all crawlers') do
-					@options[:run_all] = true
-				end	
-
-				opts.on('-f', '--format [NAME]', "Formatter used") do |name|
-					@options[:formatter] = name
-				end
-
-				opts.on('-g', '--generate [NAME]', "Generate scaffold for new crawler") do |name|
-					@options[:generate_crawler] = name
-				end
-
-				opts.on('-i', '--include [PATH]', 'Include additional crawler or crawler directory') do |path|
-					@options[:crawler_dirs] << path
-				end
-
-				opts.on('-v', '--verbose', 'Enable verbose output') do
-					@options[:verbose] = true
-				end
-
-				opts.on('-V', '--version', 'Show version info') do
-					@options[:version] = true
-				end
-
-				opts.on('-l', '--list-crawlers', 'List of crawlers') do
-					@options[:list_crawlers] = true
-				end
-
-				opts.on(nil, '--list-formatters', 'List of formatters available') do
-					@options[:list_formatters] = true
-				end				
-			end
-		end
-
-		# Parse the options passed to command-line
-		def parse_options
-			# Parse the command-line. Remember there are two forms
-			# of the parse method. The 'parse' method simply parses
-			# ARGV, while the 'parse!' method parses ARGV and removes
-			# any options found there, as well as any parameters for
-			# the options. What's left is the list of files to resize.
-			@optparser.parse!
-		end
-
-		# Load global options first
-		# Merge it with local options (if they exists)
-		def load_config_file()
-			config = File.join(File.dirname(__FILE__), "config", "crawler.rb")
-			if(File.exists?(config))
-				if(@options[:verbose])
-					puts "Loading config '#{config}'"
-				end
-				
-				# puts "Let's require '#{@options[:verbose]}'"
-				require config
-			else
-				if(@options[:verbose])
-					# TODO: Add support for initial rake task generation
-					#          Something like this:
-					#          rake config:init # Initializes config files with
-					#            their defaults (if not exists already)        
-					puts "Default config does not exist, skipping - '#{config}'"
-				end
-			end
-		end
-
-		# Register caches
-		def register_cache(dir)			
-			if(@options[:verbose]) 
-				puts "Registering caches - '#{dir}'"
-			end
-
-			files = File.join(dir, "**", "*.rb")
-			Dir.glob(files).each do |file|
-				require file
-			end
-
-			tmp = Apollo::Caches.constants.select { |c| 
-				Class === Apollo::Caches.const_get(c)
-			}
-
-			tmp.each do |x| 
-				klass = Object.const_get('Apollo').const_get('Caches').const_get(x)
-				@caches.merge!({ x.downcase.to_s => klass})
-			end
-
-			if(@options[:verbose])
-				@caches.each do |cache, klass|
-					name = klass
-
-					# klass.ancestors.include?(Apollo::Caches::Cache)
-					if  name == "Apollo::Caches::Cache"
-						next
-					end
-
-					puts "Registered cache '#{cache}' -> '#{name}'"
-				end
-			end
-		end
-
-		# Register crawlers
-		def register_crawlers(dir)			
-			if(@options[:verbose]) 
-				puts "Registering crawlers - '#{dir}'"
-			end
-
-			files = File.join(dir, "**", "*.rb")
-			Dir.glob(files).each do |file|
-				require file
-			end
-
-			tmp = Apollo::Crawlers.constants.select { |c| 
-				Class === Apollo::Crawlers.const_get(c)
-			}
-
-			tmp.each do |x| 
-				klass = Object.const_get('Apollo').const_get('Crawlers').const_get(x)
-				@crawlers.merge!({ x.downcase.to_s => klass})
-			end
-
-			if(@options[:verbose])
-				@crawlers.each do |crawler, klass|
-					name = klass.new.class.name
-
-					if  name == "Apollo::Crawlers::Crawler"
-						next
-					end
-
-					puts "Registered crawler '#{crawler}' -> '#{name}'"
-				end
-			end
-		end
-
-		# Register formatters
-		def register_formatters(dir)			
-			if(@options[:verbose]) 
-				puts "Registering formatters - '#{dir}'"
-			end
-
-			files = File.join(dir, "**", "*.rb")
-			Dir.glob(files).each do |file|
-				require file
-			end
-
-			tmp = Apollo::Formatters.constants.select { |c| 
-				Class === Apollo::Formatters.const_get(c)
-			}
-
-			tmp.each do |x| 
-				klass = Object.const_get('Apollo').const_get('Formatters').const_get(x)
-				@formatters.merge!({ x.downcase.to_s => klass})
-			end
-
-			if(@options[:verbose])
-				@formatters.each do |formatter, klass|
-					name = klass.new.class.name
-
-					if  name == "Apollo::Formatters::Formatter"
-						next
-					end
-
-					puts "Registered formatter '#{formatter}' -> '#{name}'"
-				end
-			end
-		end
-
-		def generate_crawler(name, url = nil, matcher = nil)
-			name = name.titleize.gsub(" ", "")
-
-			if(@options[:verbose])
-				puts "Generating new crawler '#{name}'"
-			end
-
-			template_path = File.join(File.dirname(__FILE__), '..', 'lib', 'apollo_crawler', @@CRAWLER_TEMPLATE_NAME)
-			if(File.exists?(template_path) == false)
-				puts "Template file '#{template_path}' does not exists!"
-				return
-			end
-
-			if(@options[:verbose])
-				puts "Using template '#{template_path}'"
-			end
-
-			dest_path = File.join(Dir.pwd, "#{name.underscore}.rb")
-
-			url = url ? url : "http://some-url-here"
-			matcher = matcher ? matcher : "//a"
-			
-			placeholders = {
-				"CRAWLER_CLASS_NAME" => name,
-				"CRAWLER_NAME" => name.titleize,
-				"CRAWLER_URL"  => url,
-				"CRAWLER_MATCHER" => matcher
-			}
-
-			puts "Generating crawler '#{name.titleize}', class: '#{name}', path: '#{dest_path}'"
-
-			File.open(template_path, 'r') do |tmpl|
-				File.open(dest_path, 'w') do |crawler|  
-					while line = tmpl.gets  
-						#puts line
-						placeholders.each do |k, v|
-							line.gsub!(k, v)
-						end
-						
-						crawler.puts line
-					end  
-				end
-			end  
-		end
-
-		def run
-			init_options()
-
-			parse_options()
-
-			if(@options[:version])
-				puts Apollo::VERSION
-				exit
-			end
-
-			load_config_file()
-
-			if(@options[:generate_crawler])
-				name = @options[:generate_crawler]
-				url = ARGV.length > 0 ? ARGV[0] : nil
-				matcher = ARGV.length > 1 ? ARGV[1] : nil
-				
-				self.generate_crawler(name, url, matcher)
-				exit
-			end
-
-			# Register caches which can be used
-			@options[:cache_dirs].each do |dir|
-				register_cache(dir)
-			end
-
-			# Register sites which can be crawled
-			@options[:crawler_dirs].each do |dir|
-				register_crawlers(dir)
-			end
-
-			# Register sites which can be crawled
-			@options[:formatter_dirs].each do |dir|
-				register_formatters(dir)
-			end
-
-			# Set default formatter here
-			formatter_name = "json"
-			if(@options[:formatter])
-				formatter_name = @options[:formatter]
-			end
-
-			# Look for specified formatter
-			f = @formatters.select { |k, v|
-				k.downcase == formatter_name.downcase
-			}
-
-			if(f)
-				@formatter = f[f.keys[0]] 
-			end
-
-			if(@options[:list_formatters])
-				headings = ['name', 'class']
-				rows = @formatters
-
-				table = Terminal::Table.new :headings => headings, :rows => rows
-				
-				puts table
-				return
-			end
-
-			if(@options[:list_crawlers])
-				headings = ['name', 'class']
-				rows = @crawlers
-
-				table = Terminal::Table.new :headings => headings, :rows => rows
-				
-				puts table
-				return
-			end
-
-
-
-			crawlers = []
-			if(ARGV.length > 0)
-				crawlers << ARGV.shift
-			end
-
-			if(@options[:run_all])
-				crawlers = @crawlers.keys
-			end
-
-			if(crawlers.empty?)
-				puts @optparser
-				exit
-			end
-
-			crawlers.each do |crawler|
-				p = @crawlers[crawler.downcase]
-				if(p == nil)
-					puts "Invalid crawler name - '#{crawler}'"
-					puts "See program help"
-					next
-				end
-
-				if(@options[:verbose])
-					puts "Running '#{crawler}'"
-				end
-
-				res = p.new.etl(ARGV) { | docs |
-					if(docs.nil?)
-						next
-					end
-
-					if(docs.kind_of?(Array) == false)
-						docs = [docs]
-					end
-
-					docs.each do |doc|
-						puts @formatter.format(doc)
-					end
-				}
-			end
-		end
-	end
-end
-
-if __FILE__ == $0
-	Apollo::CrawlerProgram.new.run()
-else
-	Apollo::CrawlerProgram.new.run()
-end
+#! /usr/bin/env ruby
+
+# encoding: utf-8
+
+require "rubygems"
+require "bundler/setup"
+
+require 'json'
+
+require "thor"
+
+require "open-uri"
+require "nokogiri"
+
+require "pp"
+require "optparse"
+
+require 'active_support'
+require 'active_support/inflector'
+
+require 'terminal-table'
+
+require File.join(File.dirname(__FILE__), '..', 'lib', 'apollo_crawler', 'version') 
+
+module Apollo
+	class CrawlerProgram
+		@@CACHES_DIR = File.join(File.dirname(__FILE__), "..", "lib", "apollo_crawler", "caches")
+		@@CRAWLERS_DIR = File.join(File.dirname(__FILE__), "..", "lib", "apollo_crawler", "crawlers")
+		@@FORMATTERS_DIR = File.join(File.dirname(__FILE__), "..", "lib", "apollo_crawler", "formatters")
+		@@CRAWLER_TEMPLATE_NAME = "crawler_template.rb"
+
+		# This hash will hold all of the options
+		# parsed from the command-line by
+		# OptionParser.
+		@options = nil
+		@optparser = nil
+		@caches = nil
+		@crawlers = nil
+		@formatters = nil
+		@formatter = nil
+
+		# Initializer - Constructor
+		def initialize
+			@caches = {}
+			@crawlers = {}
+			@formatters = {}
+		end
+
+		# Initialize command-line options
+		def init_options
+			@options = {}
+			@options[:verbose] = false
+			@options[:version] = false
+			@options[:cache_dirs] = [
+				@@CACHES_DIR
+			]
+			@options[:crawler_dirs] = [
+				@@CRAWLERS_DIR
+			]
+			@options[:formatter_dirs] = [
+				@@FORMATTERS_DIR
+			]
+			@options[:generate_crawler] = nil
+
+			@optparser = OptionParser.new do | opts |
+				opts.banner = "Usage: apollo-crawler [OPTIONS] CRAWLER_NAME [START_URL]"
+
+				opts.separator ""
+      			opts.separator "Specific options:"
+
+				# This displays the help screen, all programs are
+				# assumed to have this option.
+				opts.on('-h', '--help', 'Display this screen') do
+					puts opts
+					exit
+				end
+
+				opts.on('-a', '--all', 'Run all crawlers') do
+					@options[:run_all] = true
+				end	
+
+				opts.on('-f', '--format [NAME]', "Formatter used") do |name|
+					@options[:formatter] = name
+				end
+
+				opts.on('-g', '--generate [NAME]', "Generate scaffold for new crawler") do |name|
+					@options[:generate_crawler] = name
+				end
+
+				opts.on('-i', '--include [PATH]', 'Include additional crawler or crawler directory') do |path|
+					@options[:crawler_dirs] << path
+				end
+
+				opts.on('-v', '--verbose', 'Enable verbose output') do
+					@options[:verbose] = true
+				end
+
+				opts.on('-V', '--version', 'Show version info') do
+					@options[:version] = true
+				end
+
+				opts.on('-l', '--list-crawlers', 'List of crawlers') do
+					@options[:list_crawlers] = true
+				end
+
+				opts.on(nil, '--list-formatters', 'List of formatters available') do
+					@options[:list_formatters] = true
+				end				
+			end
+		end
+
+		# Parse the options passed to command-line
+		def parse_options
+			# Parse the command-line. Remember there are two forms
+			# of the parse method. The 'parse' method simply parses
+			# ARGV, while the 'parse!' method parses ARGV and removes
+			# any options found there, as well as any parameters for
+			# the options. What's left is the list of files to resize.
+			@optparser.parse!
+		end
+
+		# Load global options first
+		# Merge it with local options (if they exists)
+		def load_config_file()
+			config = File.join(File.dirname(__FILE__), "config", "crawler.rb")
+			if(File.exists?(config))
+				if(@options[:verbose])
+					puts "Loading config '#{config}'"
+				end
+				
+				# puts "Let's require '#{@options[:verbose]}'"
+				require config
+			else
+				if(@options[:verbose])
+					# TODO: Add support for initial rake task generation
+					#          Something like this:
+					#          rake config:init # Initializes config files with
+					#            their defaults (if not exists already)        
+					puts "Default config does not exist, skipping - '#{config}'"
+				end
+			end
+		end
+
+		# Register caches
+		def register_cache(dir)			
+			if(@options[:verbose]) 
+				puts "Registering caches - '#{dir}'"
+			end
+
+			files = File.join(dir, "**", "*.rb")
+			Dir.glob(files).each do |file|
+				require file
+			end
+
+			tmp = Apollo::Caches.constants.select { |c| 
+				Class === Apollo::Caches.const_get(c)
+			}
+
+			tmp.each do |x| 
+				klass = Object.const_get('Apollo').const_get('Caches').const_get(x)
+				@caches.merge!({ x.downcase.to_s => klass})
+			end
+
+			if(@options[:verbose])
+				@caches.each do |cache, klass|
+					name = klass
+
+					# klass.ancestors.include?(Apollo::Caches::Cache)
+					if  name == "Apollo::Caches::Cache"
+						next
+					end
+
+					puts "Registered cache '#{cache}' -> '#{name}'"
+				end
+			end
+		end
+
+		# Register crawlers
+		def register_crawlers(dir)			
+			if(@options[:verbose]) 
+				puts "Registering crawlers - '#{dir}'"
+			end
+
+			files = File.join(dir, "**", "*.rb")
+			Dir.glob(files).each do |file|
+				require file
+			end
+
+			tmp = Apollo::Crawlers.constants.select { |c| 
+				Class === Apollo::Crawlers.const_get(c)
+			}
+
+			tmp.each do |x| 
+				klass = Object.const_get('Apollo').const_get('Crawlers').const_get(x)
+				@crawlers.merge!({ x.downcase.to_s => klass})
+			end
+
+			if(@options[:verbose])
+				@crawlers.each do |crawler, klass|
+					name = klass.new.class.name
+
+					if  name == "Apollo::Crawlers::Crawler"
+						next
+					end
+
+					puts "Registered crawler '#{crawler}' -> '#{name}'"
+				end
+			end
+		end
+
+		# Register formatters
+		def register_formatters(dir)			
+			if(@options[:verbose]) 
+				puts "Registering formatters - '#{dir}'"
+			end
+
+			files = File.join(dir, "**", "*.rb")
+			Dir.glob(files).each do |file|
+				require file
+			end
+
+			tmp = Apollo::Formatters.constants.select { |c| 
+				Class === Apollo::Formatters.const_get(c)
+			}
+
+			tmp.each do |x| 
+				klass = Object.const_get('Apollo').const_get('Formatters').const_get(x)
+				@formatters.merge!({ x.downcase.to_s => klass})
+			end
+
+			if(@options[:verbose])
+				@formatters.each do |formatter, klass|
+					name = klass.new.class.name
+
+					if  name == "Apollo::Formatters::Formatter"
+						next
+					end
+
+					puts "Registered formatter '#{formatter}' -> '#{name}'"
+				end
+			end
+		end
+
+		def generate_crawler(name, url = nil, matcher = nil)
+			name = name.titleize.gsub(" ", "")
+
+			if(@options[:verbose])
+				puts "Generating new crawler '#{name}'"
+			end
+
+			template_path = File.join(File.dirname(__FILE__), '..', 'lib', 'apollo_crawler', @@CRAWLER_TEMPLATE_NAME)
+			if(File.exists?(template_path) == false)
+				puts "Template file '#{template_path}' does not exists!"
+				return
+			end
+
+			if(@options[:verbose])
+				puts "Using template '#{template_path}'"
+			end
+
+			dest_path = File.join(Dir.pwd, "#{name.underscore}.rb")
+
+			url = url ? url : "http://some-url-here"
+			matcher = matcher ? matcher : "//a"
+			
+			placeholders = {
+				"CRAWLER_CLASS_NAME" => name,
+				"CRAWLER_NAME" => name.titleize,
+				"CRAWLER_URL"  => url,
+				"CRAWLER_MATCHER" => matcher
+			}
+
+			puts "Generating crawler '#{name.titleize}', class: '#{name}', path: '#{dest_path}'"
+
+			File.open(template_path, 'r') do |tmpl|
+				File.open(dest_path, 'w') do |crawler|  
+					while line = tmpl.gets  
+						#puts line
+						placeholders.each do |k, v|
+							line.gsub!(k, v)
+						end
+						
+						crawler.puts line
+					end  
+				end
+			end  
+		end
+
+		def run
+			init_options()
+
+			parse_options()
+
+			if(@options[:version])
+				puts Apollo::VERSION
+				exit
+			end
+
+			load_config_file()
+
+			if(@options[:generate_crawler])
+				name = @options[:generate_crawler]
+				url = ARGV.length > 0 ? ARGV[0] : nil
+				matcher = ARGV.length > 1 ? ARGV[1] : nil
+				
+				self.generate_crawler(name, url, matcher)
+				exit
+			end
+
+			# Register caches which can be used
+			@options[:cache_dirs].each do |dir|
+				register_cache(dir)
+			end
+
+			# Register sites which can be crawled
+			@options[:crawler_dirs].each do |dir|
+				register_crawlers(dir)
+			end
+
+			# Register sites which can be crawled
+			@options[:formatter_dirs].each do |dir|
+				register_formatters(dir)
+			end
+
+			# Set default formatter here
+			formatter_name = "json"
+			if(@options[:formatter])
+				formatter_name = @options[:formatter]
+			end
+
+			# Look for specified formatter
+			f = @formatters.select { |k, v|
+				k.downcase == formatter_name.downcase
+			}
+
+			if(f)
+				@formatter = f[f.keys[0]] 
+			end
+
+			if(@options[:list_formatters])
+				headings = ['name', 'class']
+				rows = @formatters
+
+				table = Terminal::Table.new :headings => headings, :rows => rows
+				
+				puts table
+				return
+			end
+
+			if(@options[:list_crawlers])
+				headings = ['name', 'class']
+				rows = @crawlers
+
+				table = Terminal::Table.new :headings => headings, :rows => rows
+				
+				puts table
+				return
+			end
+
+
+
+			crawlers = []
+			if(ARGV.length > 0)
+				crawlers << ARGV.shift
+			end
+
+			if(@options[:run_all])
+				crawlers = @crawlers.keys
+			end
+
+			if(crawlers.empty?)
+				puts @optparser
+				exit
+			end
+
+			crawlers.each do |crawler|
+				p = @crawlers[crawler.downcase]
+				if(p == nil)
+					puts "Invalid crawler name - '#{crawler}'"
+					puts "See program help"
+					next
+				end
+
+				if(@options[:verbose])
+					puts "Running '#{crawler}'"
+				end
+
+				res = p.new.etl(ARGV) { | docs |
+					if(docs.nil?)
+						next
+					end
+
+					if(docs.kind_of?(Array) == false)
+						docs = [docs]
+					end
+
+					docs.each do |doc|
+						puts @formatter.format(doc)
+					end
+				}
+			end
+		end
+	end
+end
+
+if __FILE__ == $0
+	Apollo::CrawlerProgram.new.run()
+else
+	Apollo::CrawlerProgram.new.run()
+end