apollo-crawler 0.0.46 → 0.0.47

data/bin/apollo-crawler

@@ -308,7 +308,10 @@ module Apollo
 				return
 			end
 
-			crawlers = ARGV
+			crawlers = []
+			if(ARGV.length > 0)
+				crawlers << ARGV.shift
+			end
 
 			if(@options[:run_all])
 				crawlers = @crawlers.keys
@@ -331,16 +334,18 @@ module Apollo
 					puts "Running '#{crawler}'"
 				end
 
-				res = p.new.etl
+				res = p.new.etl(ARGV)
 				if(res.nil?)
 					next
 				end
 
-				# puts Apollo::Crawler::Formatters::Json.format(res)
-				# puts Apollo::Crawler::Formatters::Plain.format(res)
-				# puts Apollo::Crawler::Formatters::Table.format(res)
+				if(res.kind_of?(Array) == false)
+					res = [res]
+				end
 
-				puts @formatter.format(res)
+				res.each do |tmp|
+					puts @formatter.format(tmp)
+				end
 			end
 		end
 	end

data/lib/apollo_crawler/crawler.rb

@@ -4,6 +4,11 @@ require "nokogiri"
 module Apollo
 	module Crawlers
 		class Crawler
+			@backlog = nil
+
+			def initialize
+				@backlog = []
+			end
 
 			# Name of the crawler
 			def name
@@ -20,11 +25,31 @@ module Apollo
 			# - (3) Go to (0) eventually
 			def etl(url=nil)
 				# Look for passed URL use default instead and fail if it is not valid
-				url = url ? url : self.url
+				if(url.empty?)
+					url = self.url
+				end
+
 				if(url.nil?)
 					return nil
 				end
 
+				if(url.kind_of?(Array))
+					@backlog.concat(url)
+				else
+					@backlog << url
+				end
+
+				res = []
+				while(@backlog.empty? == false)
+					url = @backlog.shift
+
+					# puts "Processing '#{url}'"
+					res << self.process_url(url)
+				end
+				return res
+			end
+
+			def process_url(url)
 				# Try fetch document
 				doc = self.fetch_document(url)
 				if(doc.nil?)
@@ -48,14 +73,18 @@ module Apollo
 
 			# Fetch document
 			def fetch_document(url)
-				ic = Iconv.new("UTF-8//IGNORE", "UTF-8")
+				if(url == nil)
+					url = self.url
+				end
 
-				if(self.url.nil?)
+				if(url.nil?)
 					return nil
 				end
 
+				raw = open(self.url).read
+
 				# TODO: Encapsulate and make more robust => invalid hostname, timeouts and so
-				doc = Nokogiri::HTML(ic.iconv(open(self.url).read))
+				doc = Nokogiri::HTML(raw)
 				return doc
 			end
 

data/lib/apollo_crawler/crawler_template.rb

@@ -1,8 +1,6 @@
-require 'iconv'
-
 module Apollo
 	module Crawlers
-		class CRAWLER_CLASS_NAME < Apollo::Crawler::Crawlers::Crawler
+		class CRAWLER_CLASS_NAME < Crawler
 			@@MATCHER_ITEM = "CRAWLER_MATCHER"
 
 			def name()

data/lib/apollo_crawler/crawlers/alexa_com/alexa.rb

@@ -1,10 +1,8 @@
-require 'iconv'
-
 require File.join(File.dirname(__FILE__), '..', '..', 'crawler') 
 
 module Apollo
 	module Crawlers
-		class Alexa < Apollo::Crawlers::Crawler
+		class Alexa < Crawler
 			@@MATCHER_ITEM = "//div[@id = 'alphabetically']/ul/li/a"
 
 			def name()

data/lib/apollo_crawler/crawlers/firmy_cz/firmy.rb

@@ -1,10 +1,8 @@
-require 'iconv'
-
 require File.join(File.dirname(__FILE__), '..', '..', 'crawler') 
 
 module Apollo
 	module Crawlers
-		class Firmy < Apollo::Crawlers::Crawler
+		class Firmy < Crawler
 			@@MATCHER_ITEM = "//div[@id = 'alphabetically']/ul/li/a"
 
 			def name()

data/lib/apollo_crawler/crawlers/slashdot_org/slashdot.rb

@@ -1,10 +1,8 @@
-require 'iconv'
-
 require File.join(File.dirname(__FILE__), '..', '..', 'crawler') 
 
 module Apollo
 	module Crawlers
-		class Slashdot < Apollo::Crawlers::Crawler
+		class Slashdot < Crawler
 			@@MATCHER_ITEM = "//article/header/h2/span/a"
 
 			def name

data/lib/apollo_crawler/crawlers/stackoverflow_com/stackoverflow.rb

@@ -1,10 +1,8 @@
-require 'iconv'
-
 require File.join(File.dirname(__FILE__), '..', '..', 'crawler') 
 
 module Apollo
 	module Crawlers
-		class StackOverflow < Apollo::Crawlers::Crawler
+		class StackOverflow < Crawler
 			@@MATCHER_ITEM = "//div[@class = 'summary']/h3/a"
 
 			def name

data/lib/apollo_crawler/crawlers/xkcd_com/xkcd.rb

@@ -1,10 +1,8 @@
-require 'iconv'
-
 require File.join(File.dirname(__FILE__), '..', '..', 'crawler') 
 
 module Apollo
 	module Crawlers
-		class Xkcd < Apollo::Crawlers::Crawler
+		class Xkcd < Crawler
 			@@MATCHER_ITEM = "//div[@id = 'comic']/img"
 
 			def name()

data/lib/apollo_crawler/crawlers/ycombinator_com/hacker_news.rb

@@ -1,10 +1,8 @@
-require 'iconv'
-
 require File.join(File.dirname(__FILE__), '..', '..', 'crawler') 
 
 module Apollo
 	module Crawlers
-		class HackerNews < Apollo::Crawlers::Crawler
+		class HackerNews < Crawler
 			@@MATCHER_ITEM = "//td[@class = 'title']/a"
 
 			def name

data/lib/apollo_crawler/version.rb

@@ -1,5 +1,5 @@
 module Apollo
 	module Crawler
-		VERSION = '0.0.46'
+		VERSION = '0.0.47'
 	end # Crawler
 end # Apollo

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: apollo-crawler
 version: !ruby/object:Gem::Version
-  version: 0.0.46
+  version: 0.0.47
   prerelease: 
 platform: ruby
 authors:
@@ -107,22 +107,6 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: iconv
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement