apollo-crawler 0.0.26 → 0.0.28

data/bin/apollo-crawler

@@ -1,5 +1,7 @@
 #! /usr/bin/env ruby
 
+# encoding: utf-8
+
 require "rubygems"
 require "bundler/setup"
 
@@ -13,6 +15,10 @@ require "nokogiri"
 require "pp"
 require "optparse"
 
+require 'active_support'
+require 'active_support/inflector'
+
+
 require File.join(File.dirname(__FILE__), '..', 'lib', 'apollo_crawler', 'version') 
 
 module Crawler
@@ -141,7 +147,9 @@ module Crawler
 			end
 		end
 
-		def generate_plugin(name)
+		def generate_plugin(name, url = nil, matcher = nil)
+			name = name.titleize.gsub(" ", "")
+
 			if(@options[:verbose])
 				puts "Generating new plugin '#{name}'"
 			end
@@ -156,15 +164,19 @@ module Crawler
 				puts "Using template '#{template_path}'"
 			end
 
-			dest_path = File.join(Dir.pwd, "#{name}.rb")
+			dest_path = File.join(Dir.pwd, "#{name.underscore}.rb")
 			if(@options[:verbose])
 				puts "Generating '#{dest_path}'"
 			end
 
+			url = url ? url : "http://some-url-here"
+			matcher = matcher ? matcher : "//a"
+			
 			placeholders = {
-				"PLUGIN_NAME" => name,
-				"PLUGIN_URL"  => "http://some-url-here",
-				"PLUGIN_MATCHER" => "//a"
+				"PLUGIN_CLASS_NAME" => name,
+				"PLUGIN_NAME" => name.titleize,
+				"PLUGIN_URL"  => url,
+				"PLUGIN_MATCHER" => matcher
 			}
 
 			File.open(template_path, 'r') do |tmpl|
@@ -195,7 +207,10 @@ module Crawler
 
 			if(@options[:generate_plugin])
 				name = @options[:generate_plugin]
-				self.generate_plugin(name)
+				url = ARGV.length > 0 ? ARGV[0] : nil
+				matcher = ARGV.length > 1 ? ARGV[1] : nil
+				
+				self.generate_plugin(name, url, matcher)
 				exit
 			end
 

data/lib/apollo_crawler/plugin_template.rb

@@ -1,9 +1,10 @@
+require 'iconv'
 
 module Apollo
 	module Crawler
 		module Plugins
 			# PARAMATRIZE: Plugin class name
-			class PLUGIN_NAME < Plugin
+			class PLUGIN_CLASS_NAME < Plugin
 				@@URL = "PLUGIN_URL"
 
 				@@MATCHER_ITEM = "PLUGIN_MATCHER"
@@ -13,9 +14,11 @@ module Apollo
 				end
 
 				def run()
-					# TODO: Encapsulate and make more robust => invalid hostname, timeouts and so
-					doc = Nokogiri::HTML(open(@@URL))
+					ic = Iconv.new("UTF-8//IGNORE", "UTF-8")
 
+					# TODO: Encapsulate and make more robust => invalid hostname, timeouts and so
+					doc = Nokogiri::HTML(ic.iconv(open(@@URL).read))
+					
 					res = doc.xpath(@@MATCHER_ITEM).map { |i|
 						{ 
 							:text => i.text,

data/lib/apollo_crawler/plugins/alexa_com/alexa.rb

@@ -1,3 +1,5 @@
+require 'iconv'
+
 require File.join(File.dirname(__FILE__), '..', '..', 'plugin') 
 
 module Apollo
@@ -14,9 +16,10 @@ module Apollo
 				end
 
 				def run()
-					# TODO: Encapsulate and make more robust => invalid hostname, timeouts and so
-					doc = Nokogiri::HTML(open(@@URL))
+					ic = Iconv.new("UTF-8//IGNORE", "UTF-8")
 
+					# TODO: Encapsulate and make more robust => invalid hostname, timeouts and so
+					doc = Nokogiri::HTML(ic.iconv(open(@@URL).read))
 					res = doc.xpath(@@MATCHER_ITEM).map { |i|
 						{ 
 							:text => i.text,

data/lib/apollo_crawler/plugins/firmy_cz/firmy.rb

@@ -1,3 +1,5 @@
+require 'iconv'
+
 require File.join(File.dirname(__FILE__), '..', '..', 'plugin') 
 
 module Apollo
@@ -14,7 +16,10 @@ module Apollo
 				end
 
 				def run()
-					doc = Nokogiri::HTML(open(@@URL))
+					ic = Iconv.new("UTF-8//IGNORE", "UTF-8")
+
+					# TODO: Encapsulate and make more robust => invalid hostname, timeouts and so
+					doc = Nokogiri::HTML(ic.iconv(open(@@URL).read))
 
 					res = doc.xpath(@@MATCHER_ITEM).map { |i|
 						{ 

data/lib/apollo_crawler/plugins/slashdot_org/slashdot.rb

@@ -1,3 +1,5 @@
+require 'iconv'
+
 require File.join(File.dirname(__FILE__), '..', '..', 'plugin') 
 
 module Apollo
@@ -14,7 +16,10 @@ module Apollo
 				end
 
 				def run()
-					doc = Nokogiri::HTML(open(@@URL))
+					ic = Iconv.new("UTF-8//IGNORE", "UTF-8")
+
+					# TODO: Encapsulate and make more robust => invalid hostname, timeouts and so
+					doc = Nokogiri::HTML(ic.iconv(open(@@URL).read))
 
 					res = doc.xpath(@@MATCHER_ITEM).map { |i|
 						{ 

data/lib/apollo_crawler/plugins/ycombinator_com/hacker_news.rb

@@ -1,3 +1,5 @@
+require 'iconv'
+
 require File.join(File.dirname(__FILE__), '..', '..', 'plugin') 
 
 module Apollo
@@ -14,7 +16,10 @@ module Apollo
 				end
 
 				def run()
-					doc = Nokogiri::HTML(open(@@URL))
+					ic = Iconv.new("UTF-8//IGNORE", "UTF-8")
+
+					# TODO: Encapsulate and make more robust => invalid hostname, timeouts and so
+					doc = Nokogiri::HTML(ic.iconv(open(@@URL).read))
 
 					res = doc.xpath(@@MATCHER_ITEM).map { |i|
 						{ 

data/lib/apollo_crawler/version.rb

@@ -1,5 +1,5 @@
 module Apollo
 	module Crawler
-		VERSION = '0.0.26'
+		VERSION = '0.0.28'
 	end # Crawler
 end # Apollo

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: apollo-crawler
 version: !ruby/object:Gem::Version
-  version: 0.0.26
+  version: 0.0.28
   prerelease: 
 platform: ruby
 authors: