apollo-crawler 0.0.45 → 0.0.46

data/bin/apollo-crawler

@@ -22,8 +22,8 @@ require 'terminal-table'
 
 require File.join(File.dirname(__FILE__), '..', 'lib', 'apollo_crawler', 'version') 
 
-module Crawler
-	class Program
+module Apollo
+	class CrawlerProgram
 		@@CRAWLERS_DIR = File.join(File.dirname(__FILE__), "..", "lib", "apollo_crawler", "crawlers")
 		@@FORMATTERS_DIR = File.join(File.dirname(__FILE__), "..", "lib", "apollo_crawler", "formatters")
 		@@CRAWLER_TEMPLATE_NAME = "crawler_template.rb"
@@ -141,12 +141,12 @@ module Crawler
 				require file
 			end
 
-			tmp = Apollo::Crawler::Formatters.constants.select { |c| 
-				Class === Apollo::Crawler::Formatters.const_get(c)
+			tmp = Apollo::Formatters.constants.select { |c| 
+				Class === Apollo::Formatters.const_get(c)
 			}
 
 			tmp.each do |x| 
-				klass = Object.const_get('Apollo').const_get('Crawler').const_get('Formatters').const_get(x)
+				klass = Object.const_get('Apollo').const_get('Formatters').const_get(x)
 				@formatters.merge!({ x.downcase.to_s => klass})
 			end
 
@@ -154,7 +154,7 @@ module Crawler
 				@formatters.each do |formatter, klass|
 					name = klass.new.class.name
 
-					if  name == "Apollo::Crawler::Formatters::Formatter"
+					if  name == "Apollo::Formatters::Formatter"
 						next
 					end
 
@@ -174,12 +174,12 @@ module Crawler
 				require file
 			end
 
-			tmp = Apollo::Crawler::Crawlers.constants.select { |c| 
-				Class === Apollo::Crawler::Crawlers.const_get(c)
+			tmp = Apollo::Crawlers.constants.select { |c| 
+				Class === Apollo::Crawlers.const_get(c)
 			}
 
 			tmp.each do |x| 
-				klass = Object.const_get('Apollo').const_get('Crawler').const_get('Crawlers').const_get(x)
+				klass = Object.const_get('Apollo').const_get('Crawlers').const_get(x)
 				@crawlers.merge!({ x.downcase.to_s => klass})
 			end
 
@@ -187,7 +187,7 @@ module Crawler
 				@crawlers.each do |crawler, klass|
 					name = klass.new.class.name
 
-					if  name == "Apollo::Crawler::Crawlers::Crawler"
+					if  name == "Apollo::Crawlers::Crawler"
 						next
 					end
 
@@ -347,7 +347,7 @@ module Crawler
 end
 
 if __FILE__ == $0
-	Crawler::Program.new.run()
+	Apollo::CrawlerProgram.new.run()
 else
-	Crawler::Program.new.run()
+	Apollo::CrawlerProgram.new.run()
 end

data/lib/apollo_crawler/crawler.rb

@@ -2,75 +2,73 @@ require "open-uri"
 require "nokogiri"
 
 module Apollo
-	module Crawler
-		module Crawlers
-			class Crawler
+	module Crawlers
+		class Crawler
 
-				# Name of the crawler
-				def name
-					return "Crawler Base" 
+			# Name of the crawler
+			def name
+				return "Crawler Base" 
+			end
+
+			def url
+				return nil
+			end
+
+			# - (0) Figure out URL
+			# - (1) Extract Data
+			# - (2) Extract Links
+			# - (3) Go to (0) eventually
+			def etl(url=nil)
+				# Look for passed URL use default instead and fail if it is not valid
+				url = url ? url : self.url
+				if(url.nil?)
+					return nil
 				end
 
-				def url
+				# Try fetch document
+				doc = self.fetch_document(url)
+				if(doc.nil?)
 					return nil
 				end
 
-				# - (0) Figure out URL
-				# - (1) Extract Data
-				# - (2) Extract Links
-				# - (3) Go to (0) eventually
-				def etl(url=nil)
-					# Look for passed URL use default instead and fail if it is not valid
-					url = url ? url : self.url
-					if(url.nil?)
-						return nil
-					end
+				# Try extract data from document
+				data = self.extract_data(doc)
 
-					# Try fetch document
-					doc = self.fetch_document(url)
-					if(doc.nil?)
-						return nil
-					end
+				# Try extract links for another documents 
+				links = self.extract_links(doc)
 
-					# Try extract data from document
-					data = self.extract_data(doc)
+				# Return ETL result
+				return { 
+					:crawler => self.class.name,
+					:title => doc.title,
+					:data => data,
+					:links => links
+				}
+			end
 
-					# Try extract links for another documents 
-					links = self.extract_links(doc)
+			# Fetch document
+			def fetch_document(url)
+				ic = Iconv.new("UTF-8//IGNORE", "UTF-8")
 
-					# Return ETL result
-					return { 
-						:crawler => self.class.name,
-						:title => doc.title,
-						:data => data,
-						:links => links
-					}
+				if(self.url.nil?)
+					return nil
 				end
 
-				# Fetch document
-				def fetch_document(url)
-					ic = Iconv.new("UTF-8//IGNORE", "UTF-8")
-
-					if(self.url.nil?)
-						return nil
-					end
-
-					# TODO: Encapsulate and make more robust => invalid hostname, timeouts and so
-					doc = Nokogiri::HTML(ic.iconv(open(self.url).read))
-					return doc
-				end
+				# TODO: Encapsulate and make more robust => invalid hostname, timeouts and so
+				doc = Nokogiri::HTML(ic.iconv(open(self.url).read))
+				return doc
+			end
 
-				# Extracts data from document
-				def extract_data(doc)
-					res = []
-					return res
-				end
+			# Extracts data from document
+			def extract_data(doc)
+				res = []
+				return res
+			end
 
-				# Extract links to another documents from this document
-				def extract_links(doc)
-					res = []
-					return res
-				end
+			# Extract links to another documents from this document
+			def extract_links(doc)
+				res = []
+				return res
 			end
 		end
 	end

data/lib/apollo_crawler/crawler_template.rb

@@ -1,28 +1,26 @@
 require 'iconv'
 
 module Apollo
-	module Crawler
-		module Crawlers
-			class CRAWLER_CLASS_NAME < Apollo::Crawler::Crawlers::Crawler
-				@@MATCHER_ITEM = "CRAWLER_MATCHER"
+	module Crawlers
+		class CRAWLER_CLASS_NAME < Apollo::Crawler::Crawlers::Crawler
+			@@MATCHER_ITEM = "CRAWLER_MATCHER"
 
-				def name()
-					return "CRAWLER_NAME"
-				end
+			def name()
+				return "CRAWLER_NAME"
+			end
 
-				def url()
-					return "CRAWLER_URL"
-				end
+			def url()
+				return "CRAWLER_URL"
+			end
 
-				def extract_data(doc)
-					res = doc.xpath(@@MATCHER_ITEM).map { |i|
-						{ 
-							:text => i.text,
-							:link => URI.join(self.url, i['href'])
-						}
+			def extract_data(doc)
+				res = doc.xpath(@@MATCHER_ITEM).map { |i|
+					{ 
+						:text => i.text,
+						:link => URI.join(self.url, i['href'])
 					}
-				end
+				}
 			end
-		end # Crawlers
-	end # Crawler
+		end
+	end # Crawlers
 end # Apollo

data/lib/apollo_crawler/crawlers/alexa_com/alexa.rb

@@ -3,28 +3,26 @@ require 'iconv'
 require File.join(File.dirname(__FILE__), '..', '..', 'crawler') 
 
 module Apollo
-	module Crawler
-		module Crawlers
-			class Alexa < Apollo::Crawler::Crawlers::Crawler
-				@@MATCHER_ITEM = "//div[@id = 'alphabetically']/ul/li/a"
+	module Crawlers
+		class Alexa < Apollo::Crawlers::Crawler
+			@@MATCHER_ITEM = "//div[@id = 'alphabetically']/ul/li/a"
 
-				def name()
-					return "Alexa Rank"
-				end
+			def name()
+				return "Alexa Rank"
+			end
 
-				def url()
-					return "http://www.alexa.com/"
-				end
+			def url()
+				return "http://www.alexa.com/"
+			end
 
-				def extract_data(doc)
-					res = doc.xpath(@@MATCHER_ITEM).map { |i|
-						{ 
-							:text => i.text,
-							:link => URI.join(self.url, i['href'])
-						}
+			def extract_data(doc)
+				res = doc.xpath(@@MATCHER_ITEM).map { |i|
+					{ 
+						:text => i.text,
+						:link => URI.join(self.url, i['href'])
 					}
-				end
+				}
 			end
-		end # Crawlers
-	end # Crawler
+		end
+	end # Crawlers
 end # Apollo

data/lib/apollo_crawler/crawlers/firmy_cz/firmy.rb

@@ -3,28 +3,26 @@ require 'iconv'
 require File.join(File.dirname(__FILE__), '..', '..', 'crawler') 
 
 module Apollo
-	module Crawler
-		module Crawlers
-			class Firmy < Apollo::Crawler::Crawlers::Crawler
-				@@MATCHER_ITEM = "//div[@id = 'alphabetically']/ul/li/a"
+	module Crawlers
+		class Firmy < Apollo::Crawlers::Crawler
+			@@MATCHER_ITEM = "//div[@id = 'alphabetically']/ul/li/a"
 
-				def name()
-					return "Firmy.cz"
-				end
+			def name()
+				return "Firmy.cz"
+			end
 
-				def url()
-					return "http://www.firmy.cz/"
-				end
+			def url()
+				return "http://www.firmy.cz/"
+			end
 
-				def extract_data(doc)
-					res = doc.xpath(@@MATCHER_ITEM).map { |i|
-						{ 
-							:text => i.text,
-							:link => URI.join(self.url, i['href'])
-						}
+			def extract_data(doc)
+				res = doc.xpath(@@MATCHER_ITEM).map { |i|
+					{ 
+						:text => i.text,
+						:link => URI.join(self.url, i['href'])
 					}
-				end
+				}
 			end
-		end # Crawlers
-	end # Crawler
+		end
+	end # Crawlers
 end # Apollo

data/lib/apollo_crawler/crawlers/slashdot_org/slashdot.rb

@@ -3,28 +3,26 @@ require 'iconv'
 require File.join(File.dirname(__FILE__), '..', '..', 'crawler') 
 
 module Apollo
-	module Crawler
-		module Crawlers
-			class Slashdot < Apollo::Crawler::Crawlers::Crawler
-				@@MATCHER_ITEM = "//article/header/h2/span/a"
+	module Crawlers
+		class Slashdot < Apollo::Crawlers::Crawler
+			@@MATCHER_ITEM = "//article/header/h2/span/a"
 
-				def name
-					return "Slashdot"
-				end
+			def name
+				return "Slashdot"
+			end
 
-				def url()
-					return"http://slashdot.org/"
-				end
+			def url()
+				return"http://slashdot.org/"
+			end
 
-				def extract_data(doc)
-					res = doc.xpath(@@MATCHER_ITEM).map { |i|
-						{ 
-							:text => i.text,
-							:link => URI.join(self.url, i['href'])
-						}
+			def extract_data(doc)
+				res = doc.xpath(@@MATCHER_ITEM).map { |i|
+					{ 
+						:text => i.text,
+						:link => URI.join(self.url, i['href'])
 					}
-				end
+				}
 			end
-		end # Crawlers
-	end # Crawler
+		end
+	end # Crawlers
 end # Apollo

data/lib/apollo_crawler/crawlers/stackoverflow_com/stackoverflow.rb

@@ -4,27 +4,25 @@ require File.join(File.dirname(__FILE__), '..', '..', 'crawler')
 
 module Apollo
 	module Crawlers
-		module Crawler
-			class StackOverflow < Apollo::Crawler::Crawlers::Crawler
-				@@MATCHER_ITEM = "//div[@class = 'summary']/h3/a"
+		class StackOverflow < Apollo::Crawlers::Crawler
+			@@MATCHER_ITEM = "//div[@class = 'summary']/h3/a"
 
-				def name
-					return "StackOverflow"
-				end
+			def name
+				return "StackOverflow"
+			end
 
-				def url()
-					return "http://stackoverflow.com/"
-				end
+			def url()
+				return "http://stackoverflow.com/"
+			end
 
-				def extract_data(doc)
-					res = doc.xpath(@@MATCHER_ITEM).map { |i|
-						{ 
-							:text => i.text,
-							:link => URI.join(self.url, i['href'])
-						}
+			def extract_data(doc)
+				res = doc.xpath(@@MATCHER_ITEM).map { |i|
+					{ 
+						:text => i.text,
+						:link => URI.join(self.url, i['href'])
 					}
-				end
+				}
 			end
-		end # Crawlers
-	end # Crawler
+		end
+	end # Crawlers
 end # Apollo

data/lib/apollo_crawler/crawlers/xkcd_com/xkcd.rb

@@ -3,28 +3,26 @@ require 'iconv'
 require File.join(File.dirname(__FILE__), '..', '..', 'crawler') 
 
 module Apollo
-	module Crawler
-		module Crawlers
-			class Xkcd < Apollo::Crawler::Crawlers::Crawler
-				@@MATCHER_ITEM = "//div[@id = 'comic']/img"
+	module Crawlers
+		class Xkcd < Apollo::Crawlers::Crawler
+			@@MATCHER_ITEM = "//div[@id = 'comic']/img"
 
-				def name()
-					return "Xkcd"
-				end
+			def name()
+				return "Xkcd"
+			end
 
-				def url()
-					return "http://xkcd.com/"
-				end
+			def url()
+				return "http://xkcd.com/"
+			end
 
-				def extract_data(doc)
-					res = doc.xpath(@@MATCHER_ITEM).map { |node|
-						{ 
-							:text => node['title'],
-							:link => URI.join(self.url, node['src'])
-						}
+			def extract_data(doc)
+				res = doc.xpath(@@MATCHER_ITEM).map { |node|
+					{ 
+						:text => node['title'],
+						:link => URI.join(self.url, node['src'])
 					}
-				end
+				}
 			end
-		end # Crawlers
-	end # Crawler
+		end
+	end # Crawlers
 end # Apollo

data/lib/apollo_crawler/crawlers/ycombinator_com/hacker_news.rb

@@ -3,28 +3,26 @@ require 'iconv'
 require File.join(File.dirname(__FILE__), '..', '..', 'crawler') 
 
 module Apollo
-	module Crawler
-		module Crawlers
-			class HackerNews < Apollo::Crawler::Crawlers::Crawler
-				@@MATCHER_ITEM = "//td[@class = 'title']/a"
+	module Crawlers
+		class HackerNews < Apollo::Crawlers::Crawler
+			@@MATCHER_ITEM = "//td[@class = 'title']/a"
 
-				def name
-					return "Hacker News"
-				end
+			def name
+				return "Hacker News"
+			end
 
-				def url()
-					return "http://news.ycombinator.com/"
-				end
+			def url()
+				return "http://news.ycombinator.com/"
+			end
 
-				def extract_data(doc)
-					res = doc.xpath(@@MATCHER_ITEM).map { |i|
-						{ 
-							:text => i.text,
-							:link => URI.join(self.url, i['href'])
-						}
+			def extract_data(doc)
+				res = doc.xpath(@@MATCHER_ITEM).map { |i|
+					{ 
+						:text => i.text,
+						:link => URI.join(self.url, i['href'])
 					}
-				end
+				}
 			end
-		end # Crawlers
-	end # Crawler
+		end
+	end # Crawlers
 end # Apollo

data/lib/apollo_crawler/formatter.rb

@@ -1,8 +1,6 @@
 module Apollo
-	module Crawler
-		module Formatters
-			class Formatter
-			end
-		end # Formatters
-	end # Crawler
+	module Formatters
+		class Formatter
+		end
+	end # Formatters
 end # Apollo

data/lib/apollo_crawler/formatters/formatter_json.rb

@@ -3,17 +3,15 @@ require 'json'
 require File.join(File.dirname(__FILE__), '..', 'formatter') 
 
 module Apollo
-	module Crawler
-		module Formatters
-			class Json < Formatter
-				def format(obj)
-					return Json.format(obj)
-				end
+	module Formatters
+		class Json < Formatter
+			def format(obj)
+				return Json.format(obj)
+			end
 
-				def self.format(obj)
-					 return JSON.pretty_generate(obj)
-				end
+			def self.format(obj)
+				 return JSON.pretty_generate(obj)
 			end
-		end # Formatters
-	end # Crawler
+		end
+	end # Formatters
 end # Apollo

data/lib/apollo_crawler/formatters/formatter_plain.rb

@@ -3,17 +3,15 @@ require 'awesome_print'
 require File.join(File.dirname(__FILE__), '..', 'formatter') 
 
 module Apollo
-	module Crawler
-		module Formatters
-			class Plain < Formatter
-				def format(obj)
-					return Plain.format(obj)
-				end
+	module Formatters
+		class Plain < Formatter
+			def format(obj)
+				return Plain.format(obj)
+			end
 
-				def self.format(obj)
-					 return obj.inspect
-				end
+			def self.format(obj)
+				 return obj.inspect
 			end
-		end # Formatters
-	end # Crawler
+		end
+	end # Formatters
 end # Apollo

data/lib/apollo_crawler/formatters/formatter_table.rb

@@ -3,34 +3,32 @@ require 'terminal-table'
 require File.join(File.dirname(__FILE__), '..', 'formatter') 
 
 module Apollo
-	module Crawler
-		module Formatters
-			class Table < Formatter
-				def format(obj)
-					return Table.format(obj)
+	module Formatters
+		class Table < Formatter
+			def format(obj)
+				return Table.format(obj)
+			end
+
+			def self.format(obj)
+				headings = []
+				if(obj[:data].length > 0)
+					headings = obj[:data][0].keys
 				end
 
-				def self.format(obj)
-					headings = []
-					if(obj[:data].length > 0)
-						headings = obj[:data][0].keys
+				rows = []
+				obj[:data].each do |line|
+					data = []
+					headings.each do |column|
+						data << line[column]
 					end
 
-					rows = []
-					obj[:data].each do |line|
-						data = []
-						headings.each do |column|
-							data << line[column]
-						end
-
-						rows << data
-					end
+					rows << data
+				end
 
 
-					table = Terminal::Table.new :headings => headings, :rows => rows
-					return table
-				end
+				table = Terminal::Table.new :headings => headings, :rows => rows
+				return table
 			end
-		end # Formatters
-	end # Crawler
+		end
+	end # Formatters
 end # Apollo

data/lib/apollo_crawler/version.rb

@@ -1,5 +1,5 @@
 module Apollo
 	module Crawler
-		VERSION = '0.0.45'
+		VERSION = '0.0.46'
 	end # Crawler
 end # Apollo

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: apollo-crawler
 version: !ruby/object:Gem::Version
-  version: 0.0.45
+  version: 0.0.46
   prerelease: 
 platform: ruby
 authors: