apollo-crawler 0.0.25 → 0.0.26

data/bin/apollo-crawler

@@ -18,6 +18,7 @@ require File.join(File.dirname(__FILE__), '..', 'lib', 'apollo_crawler', 'versio
 module Crawler
 	class Program
 		@@PLUGIN_DIR = File.join(File.dirname(__FILE__), "..", "lib", "apollo_crawler", "plugins")
+		@@PLUGIN_TEMPLATE_NAME = "plugin_template.rb"
 
 		# This hash will hold all of the options
 		# parsed from the command-line by
@@ -39,6 +40,7 @@ module Crawler
 			@options[:plugin_dirs] = [
 				@@PLUGIN_DIR
 			]
+			@options[:generate_plugin] = nil
 
 			@optparser = OptionParser.new do | opts |
 				# This displays the help screen, all programs are
@@ -52,6 +54,10 @@ module Crawler
 					@options[:run_all] = true
 				end	
 
+				opts.on('-g', '--generate [NAME]') do |name|
+					@options[:generate_plugin] = name
+				end
+
 				opts.on('-i', '--include [PATH]', 'Include additional plugins or plugin directories') do |path|
 					@options[:plugin_dirs] << path
 				end
@@ -84,7 +90,6 @@ module Crawler
 		# Merge it with local options (if they exists)
 		def load_config_file()
 			config = File.join(File.dirname(__FILE__), "config", "crawler.rb")
-			puts "Inspecting #{config} ..."
 			if(File.exists?(config))
 				if(@options[:verbose])
 					puts "Loading config '#{config}'"
@@ -136,6 +141,46 @@ module Crawler
 			end
 		end
 
+		def generate_plugin(name)
+			if(@options[:verbose])
+				puts "Generating new plugin '#{name}'"
+			end
+
+			template_path = File.join(File.dirname(__FILE__), '..', 'lib', 'apollo_crawler', @@PLUGIN_TEMPLATE_NAME)
+			if(File.exists?(template_path) == false)
+				puts "Template file '#{template_path}' does not exists!"
+				return
+			end
+
+			if(@options[:verbose])
+				puts "Using template '#{template_path}'"
+			end
+
+			dest_path = File.join(Dir.pwd, "#{name}.rb")
+			if(@options[:verbose])
+				puts "Generating '#{dest_path}'"
+			end
+
+			placeholders = {
+				"PLUGIN_NAME" => name,
+				"PLUGIN_URL"  => "http://some-url-here",
+				"PLUGIN_MATCHER" => "//a"
+			}
+
+			File.open(template_path, 'r') do |tmpl|
+				File.open(dest_path, 'w') do |plugin|  
+					while line = tmpl.gets  
+						#puts line
+						placeholders.each do |k, v|
+							line.gsub!(k, v)
+						end
+						
+						plugin.puts line
+					end  
+				end
+			end  
+		end
+
 		def run
 			init_options()
 
@@ -143,11 +188,17 @@ module Crawler
 
 			if(@options[:version])
 				puts Apollo::Crawler::VERSION
-				return
+				exit
 			end
 
 			load_config_file()
 
+			if(@options[:generate_plugin])
+				name = @options[:generate_plugin]
+				self.generate_plugin(name)
+				exit
+			end
+
 			# Register sites which can be crawled
 			@options[:plugin_dirs].each do |dir|
 				register_plugins(dir)
@@ -178,10 +229,15 @@ module Crawler
 			end
 
 			plugins.each do |plugin|
-				p = @plugins[plugin.downcase].new
+				p = @plugins[plugin.downcase]
+				if(p == nil)
+					puts "Invalid plugin name - '#{plugin}'"
+					puts "See program help"
+					next
+				end
 
 				# puts "Running '#{plugin}'"
-				puts JSON.pretty_generate(p.run)
+				puts JSON.pretty_generate(p.new.run)
 			end
 		end
 	end

data/lib/apollo_crawler/plugin_template.rb

@@ -0,0 +1,35 @@
+
+module Apollo
+	module Crawler
+		module Plugins
+			# PARAMATRIZE: Plugin class name
+			class PLUGIN_NAME < Plugin
+				@@URL = "PLUGIN_URL"
+
+				@@MATCHER_ITEM = "PLUGIN_MATCHER"
+
+				def name()
+					return "PLUGIN_NAME"
+				end
+
+				def run()
+					# TODO: Encapsulate and make more robust => invalid hostname, timeouts and so
+					doc = Nokogiri::HTML(open(@@URL))
+
+					res = doc.xpath(@@MATCHER_ITEM).map { |i|
+						{ 
+							:text => i.text,
+							:link => URI.join(@@URL, i['href'])
+						}
+					}
+
+					return { 
+						:plugin => self.class.name,
+						:title => doc.title,
+						:res => res
+					}
+				end
+			end
+		end # Plugins
+	end # Crawler
+end # Apollo

data/lib/apollo_crawler/version.rb

@@ -1,5 +1,5 @@
 module Apollo
 	module Crawler
-		VERSION = '0.0.25'
+		VERSION = '0.0.26'
 	end # Crawler
 end # Apollo

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: apollo-crawler
 version: !ruby/object:Gem::Version
-  version: 0.0.25
+  version: 0.0.26
   prerelease: 
 platform: ruby
 authors:
@@ -20,6 +20,7 @@ extra_rdoc_files: []
 files:
 - ./lib/apollo_crawler/version.rb
 - ./lib/apollo_crawler/crawler.rb
+- ./lib/apollo_crawler/plugin_template.rb
 - ./lib/apollo_crawler/plugins/slashdot_org/slashdot.rb
 - ./lib/apollo_crawler/plugins/firmy_cz/firmy.rb
 - ./lib/apollo_crawler/plugins/alexa_com/alexa.rb