apollo-crawler 0.0.40 → 0.0.41

data/bin/apollo-crawler

@@ -25,6 +25,7 @@ require File.join(File.dirname(__FILE__), '..', 'lib', 'apollo_crawler', 'versio
 module Crawler
 	class Program
 		@@PLUGIN_DIR = File.join(File.dirname(__FILE__), "..", "lib", "apollo_crawler", "plugins")
+		@@FORMATTERS_DIR = File.join(File.dirname(__FILE__), "..", "lib", "apollo_crawler", "formatters")
 		@@PLUGIN_TEMPLATE_NAME = "plugin_template.rb"
 
 		# This hash will hold all of the options
@@ -33,10 +34,12 @@ module Crawler
 		@options = nil
 		@optparser = nil
 		@plugins = nil
+		@formatters = nil
 
 		# Initializer - Constructor
 		def initialize
 			@plugins = {}
+			@formatters = {}
 		end
 
 		# Initialize command-line options
@@ -47,6 +50,9 @@ module Crawler
 			@options[:plugin_dirs] = [
 				@@PLUGIN_DIR
 			]
+			@options[:formatter_dirs] = [
+				@@FORMATTERS_DIR
+			]
 			@options[:generate_plugin] = nil
 
 			@optparser = OptionParser.new do | opts |
@@ -115,15 +121,48 @@ module Crawler
 			end
 		end
 
+		# Register formatters
+		def register_formatters(dir)			
+			if(@options[:verbose]) 
+				puts "Registering formatters - '#{dir}'"
+			end
+
+			files = File.join(dir, "**", "*.rb")
+			Dir.glob(files).each do |file|
+				require file
+			end
+
+			tmp = Apollo::Crawler::Formatters.constants.select { |c| 
+				Class === Apollo::Crawler::Formatters.const_get(c)
+			}
+
+			tmp.each do |x| 
+				klass = Object.const_get('Apollo').const_get('Crawler').const_get('Formatters').const_get(x)
+				@formatters.merge!({ x.downcase.to_s => klass})
+			end
+
+			if(@options[:verbose])
+				@formatters.each do |formatter, klass|
+					name = klass.new.class.name
+
+					if  name == "Apollo::Crawler::Formatters::Formatter"
+						next
+					end
+
+					puts "Registered formatter '#{formatter}' -> '#{name}'"
+				end
+			end
+		end
+
 		# Register plugins (specific crawlers)
 		def register_plugins(dir)			
 			if(@options[:verbose]) 
 				puts "Registering plugins - '#{dir}'"
 			end
 
-			sites = File.join(dir, "**", "*.rb")
-			Dir.glob(sites).each do |site|
-				require site
+			files = File.join(dir, "**", "*.rb")
+			Dir.glob(files).each do |file|
+				require file
 			end
 
 			tmp = Apollo::Crawler::Plugins.constants.select { |c| 
@@ -143,7 +182,7 @@ module Crawler
 						next
 					end
 
-					puts "Registered '#{plugin}' -> '#{name}'"
+					puts "Registered plugin '#{plugin}' -> '#{name}'"
 				end
 			end
 		end
@@ -219,6 +258,11 @@ module Crawler
 				register_plugins(dir)
 			end
 
+			# Register sites which can be crawled
+			@options[:formatter_dirs].each do |dir|
+				register_formatters(dir)
+			end
+
 			if(@options[:list_plugins])
 				headings = ['name', 'class']
 				rows = @plugins
@@ -257,7 +301,8 @@ module Crawler
 					next
 				end
 
-				puts JSON.pretty_generate(res)
+				puts Apollo::Crawler::Formatters::Json.format(res)
+				# puts Apollo::Crawler::Formatters::Plain.format(res)
 			end
 		end
 	end

data/lib/apollo_crawler.rb

@@ -1,8 +1,13 @@
 # require 'apollo_crawler/plugin'
 
 require 'apollo_crawler/crawler'
+require 'apollo_crawler/formatter'
 require 'apollo_crawler/plugin'
 
+# Formatters
+require 'apollo_crawler/formatters/formatter_plain'
+
+# Plugins
 require 'apollo_crawler/plugins/alexa_com/alexa'
 require 'apollo_crawler/plugins/firmy_cz/firmy'
 require 'apollo_crawler/plugins/slashdot_org/slashdot'

data/lib/apollo_crawler/formatter.rb

@@ -0,0 +1,8 @@
+module Apollo
+	module Crawler
+		module Formatters
+			class Formatter
+			end
+		end # Formatters
+	end # Crawler
+end # Apollo

data/lib/apollo_crawler/formatters/formatter_json.rb

@@ -0,0 +1,19 @@
+require 'json'
+
+require File.join(File.dirname(__FILE__), '..', 'formatter') 
+
+module Apollo
+	module Crawler
+		module Formatters
+			class Json < Formatter
+				def format(obj)
+					return Plain.format(obj)
+				end
+
+				def self.format(obj)
+					 return JSON.pretty_generate(obj)
+				end
+			end
+		end # Formatters
+	end # Crawler
+end # Apollo

data/lib/apollo_crawler/formatters/formatter_plain.rb

@@ -0,0 +1,19 @@
+require 'awesome_print'
+
+require File.join(File.dirname(__FILE__), '..', 'formatter') 
+
+module Apollo
+	module Crawler
+		module Formatters
+			class Plain < Formatter
+				def format(obj)
+					return Plain.format(obj)
+				end
+
+				def self.format(obj)
+					 return obj.inspect
+				end
+			end
+		end # Formatters
+	end # Crawler
+end # Apollo

data/lib/apollo_crawler/version.rb

@@ -1,5 +1,5 @@
 module Apollo
 	module Crawler
-		VERSION = '0.0.40'
+		VERSION = '0.0.41'
 	end # Crawler
 end # Apollo

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: apollo-crawler
 version: !ruby/object:Gem::Version
-  version: 0.0.40
+  version: 0.0.41
   prerelease: 
 platform: ruby
 authors:
@@ -27,6 +27,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: active_support
   requirement: !ruby/object:Gem::Requirement
@@ -210,8 +226,11 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ./lib/apollo_crawler/formatters/formatter_plain.rb
+- ./lib/apollo_crawler/formatters/formatter_json.rb
 - ./lib/apollo_crawler/version.rb
 - ./lib/apollo_crawler/crawler.rb
+- ./lib/apollo_crawler/formatter.rb
 - ./lib/apollo_crawler/plugin_template.rb
 - ./lib/apollo_crawler/plugins/xkcd_com/xkcd.rb
 - ./lib/apollo_crawler/plugins/slashdot_org/slashdot.rb