apollo-crawler 0.0.17 → 0.0.18

data/bin/apollo-crawler

@@ -1,12 +1,170 @@
-#!/usr/bin/env ruby
-
-begin
-  require 'apollo_crawler'
-  require 'launchy'
-rescue LoadError
-  require 'rubygems'
-  require 'apollo_crawler'
-  require 'launchy'
+#! /usr/bin/env ruby
+
+require "rubygems"
+require "bundler/setup"
+
+require 'json'
+
+require "thor"
+
+require "open-uri"
+require "nokogiri"
+
+require "pp"
+require "optparse"
+
+module Crawler
+	class Program
+		# This hash will hold all of the options
+		# parsed from the command-line by
+		# OptionParser.
+		@options = nil
+		@optparser = nil
+		@plugins = nil
+
+		# Initializer - Constructor
+		def initialize
+			@plugins = {}
+		end
+
+		# Initialize command-line options
+		def init_options
+			@options = {}
+			@options[:verbose] = false
+
+			@optparser = OptionParser.new do | opts |
+				# This displays the help screen, all programs are
+				# assumed to have this option.
+				opts.on('-h', '--help', 'Display this screen') do
+					puts opts
+					exit
+				end
+
+				opts.on('-a', '--all', 'Run all plugins') do
+					@options[:run_all] = true
+				end	
+
+				opts.on('-v', '--verbose', 'Enable verbose output') do
+					@options[:verbose] = true
+				end
+
+				opts.on('-l', '--list-plugins', 'List of plugins') do
+					@options[:list_plugins] = true
+				end				
+			end
+		end
+
+		# Parse the options passed to command-line
+		def parse_options
+			# Parse the command-line. Remember there are two forms
+			# of the parse method. The 'parse' method simply parses
+			# ARGV, while the 'parse!' method parses ARGV and removes
+			# any options found there, as well as any parameters for
+			# the options. What's left is the list of files to resize.
+			@optparser.parse!
+		end
+
+		# Load global options first
+		# Merge it with local options (if they exists)
+		def load_config_file()
+			config = File.join(File.dirname(__FILE__), "config", "crawler.rb")
+			puts "Inspecting #{config} ..."
+			if(File.exists?(config))
+				if(@options[:verbose])
+					puts "Loading config '#{config}'"
+				end
+				
+				# puts "Let's require '#{@options[:verbose]}'"
+				require config
+			else
+				if(@options[:verbose])
+					# TODO: Add support for initial rake task generation
+					#          Something like this:
+					#          rake config:init # Initializes config files with
+					#            their defaults (if not exists already)        
+					puts "Default config does not exist, skipping - '#{config}'"
+				end
+			end
+		end
+
+		# Register plugins (specific crawlers)
+		def register_plugins()
+			dir = File.join(File.dirname(__FILE__), "..", "lib", "apollo_crawler", "plugins")
+			if(@options[:verbose]) 
+				puts "Registering plugins - '#{dir}'"
+			end
+
+			sites = File.join(dir, "**", "*.rb")
+			Dir.glob(sites).each do |site|
+				require site
+			end
+
+			tmp = Apollo::Crawler::Plugins.constants.select { |c| 
+				Class === Apollo::Crawler::Plugins.const_get(c)
+			}
+
+			tmp.each do |x| 
+				klass = Object.const_get('Apollo').const_get('Crawler').const_get('Plugins').const_get(x)
+				@plugins.merge!({ x.downcase.to_s => klass})
+			end
+
+			if(@options[:verbose])
+				@plugins.each do |plugin, klass|
+					name = klass.new.class.name
+
+					if  name == "Apollo::Crawler::Plugins::Plugin"
+						next
+					end
+
+					puts "Registered '#{plugin}' -> '#{name}'"
+				end
+			end
+		end
+
+		def run
+			init_options()
+			
+			load_config_file()
+
+			parse_options()
+
+			# Register sites which can be crawled
+			register_plugins()
+
+			if(@options[:list_plugins])
+				puts "Listing plugins"
+				puts "----------------------------------------"
+				i = 0
+				@plugins.sort.each do |plugin, klass|
+					instance = klass.new
+					# puts klass.class_eval("@@NAME")
+					puts "(#{i}) #{plugin} - #{instance.name}"
+					i += 1
+				end
+				puts "----------------------------------------"
+				return
+			end
+
+			plugins = ARGV
+
+			if(@options[:run_all])
+				plugins = @plugins.keys
+			end
+
+			if(plugins.empty?)
+				puts @optparser
+			end
+
+			plugins.each do |plugin|
+				p = @plugins[plugin.downcase].new
+
+				# puts "Running '#{plugin}'"
+				puts JSON.pretty_generate(p.run)
+			end
+		end
+	end
 end
 
-Launchy::Cli.new.run( ARGV, ENV )
+if __FILE__ == $0
+	Crawler::Program.new.run()
+end

data/lib/apollo_crawler/version.rb

@@ -1,5 +1,5 @@
 module Apollo
 	module Crawler
-		VERSION = '0.0.17'
+		VERSION = '0.0.18'
 	end # Crawler
 end # Apollo

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: apollo-crawler
 version: !ruby/object:Gem::Version
-  version: 0.0.17
+  version: 0.0.18
   prerelease: 
 platform: ruby
 authors:
@@ -18,7 +18,6 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- ./main.rb
 - ./lib/apollo_crawler/version.rb
 - ./lib/apollo_crawler/crawler.rb
 - ./lib/apollo_crawler/plugins/slashdot_org/slashdot.rb

data/main.rb

@@ -1,170 +0,0 @@
-#! /usr/bin/env ruby
-
-require "rubygems"
-require "bundler/setup"
-
-require 'json'
-
-require "thor"
-
-require "open-uri"
-require "nokogiri"
-
-require "pp"
-require "optparse"
-
-module Crawler
-	class Program
-		# This hash will hold all of the options
-		# parsed from the command-line by
-		# OptionParser.
-		@options = nil
-		@optparser = nil
-		@plugins = nil
-
-		# Initializer - Constructor
-		def initialize
-			@plugins = {}
-		end
-
-		# Initialize command-line options
-		def init_options
-			@options = {}
-			@options[:verbose] = false
-
-			@optparser = OptionParser.new do | opts |
-				# This displays the help screen, all programs are
-				# assumed to have this option.
-				opts.on('-h', '--help', 'Display this screen') do
-					puts opts
-					exit
-				end
-
-				opts.on('-a', '--all', 'Run all plugins') do
-					@options[:run_all] = true
-				end	
-
-				opts.on('-v', '--verbose', 'Enable verbose output') do
-					@options[:verbose] = true
-				end
-
-				opts.on('-l', '--list-plugins', 'List of plugins') do
-					@options[:list_plugins] = true
-				end				
-			end
-		end
-
-		# Parse the options passed to command-line
-		def parse_options
-			# Parse the command-line. Remember there are two forms
-			# of the parse method. The 'parse' method simply parses
-			# ARGV, while the 'parse!' method parses ARGV and removes
-			# any options found there, as well as any parameters for
-			# the options. What's left is the list of files to resize.
-			@optparser.parse!
-		end
-
-		# Load global options first
-		# Merge it with local options (if they exists)
-		def load_config_file()
-			config = File.join(File.dirname(__FILE__), "config", "crawler.rb")
-			puts "Inspecting #{config} ..."
-			if(File.exists?(config))
-				if(@options[:verbose])
-					puts "Loading config '#{config}'"
-				end
-				
-				# puts "Let's require '#{@options[:verbose]}'"
-				require config
-			else
-				if(@options[:verbose])
-					# TODO: Add support for initial rake task generation
-					#          Something like this:
-					#          rake config:init # Initializes config files with
-					#            their defaults (if not exists already)        
-					puts "Default config does not exist, skipping - '#{config}'"
-				end
-			end
-		end
-
-		# Register plugins (specific crawlers)
-		def register_plugins()
-			dir = File.join(File.dirname(__FILE__), "lib", "apollo_crawler", "plugins")
-			if(@options[:verbose]) 
-				puts "Registering plugins - '#{dir}'"
-			end
-
-			sites = File.join(dir, "**", "*.rb")
-			Dir.glob(sites).each do |site|
-				require site
-			end
-
-			tmp = Apollo::Crawler::Plugins.constants.select { |c| 
-				Class === Apollo::Crawler::Plugins.const_get(c)
-			}
-
-			tmp.each do |x| 
-				klass = Object.const_get('Apollo').const_get('Crawler').const_get('Plugins').const_get(x)
-				@plugins.merge!({ x.downcase.to_s => klass})
-			end
-
-			if(@options[:verbose])
-				@plugins.each do |plugin, klass|
-					name = klass.new.class.name
-
-					if  name == "Apollo::Crawler::Plugins::Plugin"
-						next
-					end
-
-					puts "Registered '#{plugin}' -> '#{name}'"
-				end
-			end
-		end
-
-		def run
-			init_options()
-			
-			load_config_file()
-
-			parse_options()
-
-			# Register sites which can be crawled
-			register_plugins()
-
-			if(@options[:list_plugins])
-				puts "Listing plugins"
-				puts "----------------------------------------"
-				i = 0
-				@plugins.sort.each do |plugin, klass|
-					instance = klass.new
-					# puts klass.class_eval("@@NAME")
-					puts "(#{i}) #{plugin} - #{instance.name}"
-					i += 1
-				end
-				puts "----------------------------------------"
-				return
-			end
-
-			plugins = ARGV
-
-			if(@options[:run_all])
-				plugins = @plugins.keys
-			end
-
-			if(plugins.empty?)
-				puts @optparser
-			end
-
-			plugins.each do |plugin|
-				p = @plugins[plugin.downcase].new
-
-				# puts "Running '#{plugin}'"
-				puts JSON.pretty_generate(p.run)
-			end
-		end
-	end
-end
-
-if __FILE__ == $0
-	Crawler::Program.new.run()
-end