apollo-crawler 0.1.4 → 0.1.5

data/lib/apollo_crawler/crawler_template.rb

@@ -1,24 +1,24 @@
-module Apollo
-	module Crawlers
-		class CRAWLER_CLASS_NAME < Crawler
-			@@MATCHER_ITEM = "CRAWLER_MATCHER"
-
-			def name()
-				return "CRAWLER_NAME"
-			end
-
-			def url()
-				return "CRAWLER_URL"
-			end
-
-			def extract_data(doc)
-				res = doc.xpath(@@MATCHER_ITEM).map { |i|
-					{ 
-						:text => i.text,
-						:link => URI.join(self.url, i['href'])
-					}
-				}
-			end
-		end # CRAWLER_CLASS_NAME
-	end # Crawlers
-end # Apollo
+module Apollo
+	module Crawlers
+		class CRAWLER_CLASS_NAME < Crawler
+			@@MATCHER_ITEM = "CRAWLER_MATCHER"
+
+			def name()
+				return "CRAWLER_NAME"
+			end
+
+			def url()
+				return "CRAWLER_URL"
+			end
+
+			def extract_data(doc)
+				res = doc.xpath(@@MATCHER_ITEM).map { |i|
+					{ 
+						:text => i.text,
+						:link => URI.join(self.url, i['href'])
+					}
+				}
+			end
+		end # CRAWLER_CLASS_NAME
+	end # Crawlers
+end # Apollo

data/lib/apollo_crawler/crawlers/google_com/google.rb

@@ -1,26 +1,40 @@
-require File.join(File.dirname(__FILE__), '..', '..', 'crawler') 
-
-module Apollo
-	module Crawlers
-		class Google < Crawler
-			@@MATCHER_ITEM = "//h3/a"
-
-			def name()
-				return "Google"
-			end
-
-			def url()
-				return "http://www.google.com/search?q=ruby"
-			end
-
-			def extract_data(doc)
-				res = doc.xpath(@@MATCHER_ITEM).map { |i|
-					{ 
-						:text => i.text,
-						:link => URI.join(self.url, i['href'])
-					}
-				}
-			end
-		end
-	end # Crawlers
-end # Apollo
+require File.join(File.dirname(__FILE__), '..', '..', 'crawler') 
+
+module Apollo
+	module Crawlers
+		class Google < Crawler
+			@@MATCHER_ITEM = "//h3/a"
+
+			def name()
+				return "Google"
+			end
+
+			def url()
+				return "http://www.google.com/search?q=ruby"
+			end
+
+			def extract_data(doc)
+				res = doc.xpath(@@MATCHER_ITEM).map { | node |
+					url = Crawler.try_get_url(self.url, node['href'])
+					next if url.nil?
+
+					{ 
+						:text => node.text,
+						:link => url
+					}
+				}
+			end
+
+			def extract_links(doc)
+				res = doc.xpath("(//td[@class = 'b']/a)[last()]").map { | node |
+					url = Crawler.try_get_url(self.url, node['href'])
+					next if url.nil?
+
+					{ 
+						:link => url
+					}
+				}
+			end
+		end
+	end # Crawlers
+end # Apollo

data/lib/apollo_crawler/crawlers/slashdot_org/slashdot.rb

@@ -1,26 +1,40 @@
-require File.join(File.dirname(__FILE__), '..', '..', 'crawler') 
-
-module Apollo
-	module Crawlers
-		class Slashdot < Crawler
-			@@MATCHER_ITEM = "//article/header/h2/span/a"
-
-			def name()
-				return "Slashdot"
-			end
-
-			def url()
-				return"http://slashdot.org/"
-			end
-
-			def extract_data(doc)
-				res = doc.xpath(@@MATCHER_ITEM).map { |i|
-					{ 
-						:text => i.text,
-						:link => URI.join(self.url, i['href'])
-					}
-				}
-			end
-		end
-	end # Crawlers
-end # Apollo
+require File.join(File.dirname(__FILE__), '..', '..', 'crawler') 
+
+module Apollo
+	module Crawlers
+		class Slashdot < Crawler
+			@@MATCHER_ITEM = "//article/header/h2/span/a"
+
+			def name()
+				return "Slashdot"
+			end
+
+			def url()
+				return"http://slashdot.org/"
+			end
+
+			def extract_data(doc)
+				res = doc.xpath(@@MATCHER_ITEM).map { | node |
+					url = Crawler.try_get_url(self.url, node['href'])
+					next if url.nil?
+
+					{ 
+						:text => node.text,
+						:link => url
+					}
+				}
+			end
+
+			def extract_links(doc)
+				res = doc.xpath(@@MATCHER_ITEM).map { | node |
+					url = Crawler.try_get_url(self.url, node['href'])
+					next if url.nil?
+
+					{ 
+						:link => url
+					}
+				}
+			end
+		end
+	end # Crawlers
+end # Apollo

data/lib/apollo_crawler/crawlers/stackoverflow_com/stackoverflow.rb

@@ -1,26 +1,44 @@
-require File.join(File.dirname(__FILE__), '..', '..', 'crawler') 
-
-module Apollo
-	module Crawlers
-		class StackOverflow < Crawler
-			@@MATCHER_ITEM = "//div[@class = 'summary']/h3/a"
-
-			def name()
-				return "Stackoverflow"
-			end
-
-			def url()
-				return "http://stackoverflow.com/"
-			end
-
-			def extract_data(doc)
-				res = doc.xpath(@@MATCHER_ITEM).map { |i|
-					{ 
-						:text => i.text,
-						:link => URI.join(self.url, i['href'])
-					}
-				}
-			end
-		end
-	end # Crawlers
-end # Apollo
+require File.join(File.dirname(__FILE__), '..', '..', 'crawler') 
+
+module Apollo
+	module Crawlers
+		class StackOverflow < Crawler
+			@@MATCHER_ITEM = "//div[@class = 'summary']/h3/a"
+
+			def name()
+				return "Stackoverflow"
+			end
+
+			def url()
+				return "http://stackoverflow.com/questions"
+			end
+
+			def extract_data(doc)
+				res = doc.xpath(@@MATCHER_ITEM).map { |node|
+					url = Crawler.try_get_url(self.url, node['href'])
+					next if url.nil?
+
+					{ 
+						:text => node.text,
+						:link => url
+					}
+				}
+
+				return res
+			end
+
+			def extract_links(doc)
+				res = doc.xpath("(//div[@class = 'pager fl']/a)[last()]").map { |node|
+					url = Crawler.try_get_url(self.url, node['href'])
+					next if url.nil?
+
+					{ 
+						:link => url
+					}
+				}
+				
+				return res.uniq
+			end
+		end
+	end # Crawlers
+end # Apollo

data/lib/apollo_crawler/crawlers/xkcd_com/xkcd.rb

@@ -1,35 +1,35 @@
-require File.join(File.dirname(__FILE__), '..', '..', 'crawler') 
-
-module Apollo
-	module Crawlers
-		class Xkcd < Crawler
-			@@MATCHER_ITEM = "//div[@id = 'comic']/img"
-
-			def name()
-				return "Xkcd"
-			end
-
-			def url()
-				return "http://xkcd.com/"
-			end
-
-			def extract_data(doc)
-				res = doc.xpath(@@MATCHER_ITEM).map { |node|
-					{ 
-						:text => node['title'],
-						:link => URI.join(self.url, node['src']),
-					}
-				}
-			end
-
-			def extract_links(doc)
-				res = doc.xpath("//ul[@class = 'comicNav']/li/a[@accesskey = 'p']").map { |node|
-					{ 
-						:link => URI.join(self.url, node['href']),
-					}
-				}
-				res.uniq
-			end
-		end
-	end # Crawlers
-end # Apollo
+require File.join(File.dirname(__FILE__), '..', '..', 'crawler') 
+
+module Apollo
+	module Crawlers
+		class Xkcd < Crawler
+			@@MATCHER_ITEM = "//div[@id = 'comic']/img"
+
+			def name()
+				return "Xkcd"
+			end
+
+			def url()
+				return "http://xkcd.com/"
+			end
+
+			def extract_data(doc)
+				res = doc.xpath(@@MATCHER_ITEM).map { |node|
+					{ 
+						:text => node['title'],
+						:link => URI.join(self.url, node['src']),
+					}
+				}
+			end
+
+			def extract_links(doc)
+				res = doc.xpath("//ul[@class = 'comicNav']/li/a[@accesskey = 'p']").map { |node|
+					{ 
+						:link => URI.join(self.url, node['href']),
+					}
+				}
+				res.uniq
+			end
+		end
+	end # Crawlers
+end # Apollo

data/lib/apollo_crawler/crawlers/ycombinator_com/hacker_news.rb

@@ -1,26 +1,44 @@
-require File.join(File.dirname(__FILE__), '..', '..', 'crawler') 
-
-module Apollo
-	module Crawlers
-		class HackerNews < Crawler
-			@@MATCHER_ITEM = "//td[@class = 'title']/a"
-
-			def name()
-				return "Hacker News"
-			end
-
-			def url()
-				return "http://news.ycombinator.com/"
-			end
-
-			def extract_data(doc)
-				res = doc.xpath(@@MATCHER_ITEM).map { |i|
-					{ 
-						:text => i.text,
-						:link => URI.join(self.url, i['href'])
-					}
-				}
-			end
-		end
-	end # Crawlers
-end # Apollo
+require File.join(File.dirname(__FILE__), '..', '..', 'crawler') 
+
+module Apollo
+	module Crawlers
+		class HackerNews < Crawler
+			@@MATCHER_ITEM = "(//td[@class = 'title']/a)[not(position() > last() -1)]"
+
+			def name()
+				return "Hacker News"
+			end
+
+			def url()
+				return "http://news.ycombinator.com/"
+			end
+
+			def extract_data(doc)
+				res = doc.xpath(@@MATCHER_ITEM).map { |node|
+					url = Crawler.try_get_url(self.url, node['href'])
+					next if url.nil?
+
+					{ 
+						:text => node.text,
+						:link => url
+					}
+				}
+
+				return res
+			end
+
+			def extract_links(doc)
+				res = doc.xpath("(//a[@class = 'prevnextbutact'])").map { |node|
+					url = Crawler.try_get_url(self.url, node['href'])
+					next if url.nil?
+
+					{ 
+						:link => url
+					}
+				}
+				
+				return res.uniq
+			end
+		end
+	end # Crawlers
+end # Apollo

data/lib/apollo_crawler/formatter.rb

@@ -1,6 +1,6 @@
-module Apollo
-	module Formatters
-		class Formatter
-		end # Formatter
-	end # Formatters
-end # Apollo
+module Apollo
+	module Formatters
+		class Formatter
+		end # Formatter
+	end # Formatters
+end # Apollo

data/lib/apollo_crawler/formatters/formatter_json.rb

@@ -1,17 +1,17 @@
-require 'json'
-
-require File.join(File.dirname(__FILE__), '..', 'formatter') 
-
-module Apollo
-	module Formatters
-		class Json < Formatter
-			def format(obj)
-				return Json.format(obj)
-			end
-
-			def self.format(obj)
-				 return JSON.pretty_generate(obj)
-			end
-		end
-	end # Formatters
-end # Apollo
+require 'json'
+
+require File.join(File.dirname(__FILE__), '..', 'formatter') 
+
+module Apollo
+	module Formatters
+		class Json < Formatter
+			def format(obj)
+				return Json.format(obj)
+			end
+
+			def self.format(obj)
+				 return JSON.pretty_generate(obj)
+			end
+		end
+	end # Formatters
+end # Apollo

data/lib/apollo_crawler/formatters/formatter_plain.rb

@@ -1,17 +1,17 @@
-require 'awesome_print'
-
-require File.join(File.dirname(__FILE__), '..', 'formatter') 
-
-module Apollo
-	module Formatters
-		class Plain < Formatter
-			def format(obj)
-				return Plain.format(obj)
-			end
-
-			def self.format(obj)
-				 return obj.inspect
-			end
-		end
-	end # Formatters
-end # Apollo
+require 'awesome_print'
+
+require File.join(File.dirname(__FILE__), '..', 'formatter') 
+
+module Apollo
+	module Formatters
+		class Plain < Formatter
+			def format(obj)
+				return Plain.format(obj)
+			end
+
+			def self.format(obj)
+				 return obj.inspect
+			end
+		end
+	end # Formatters
+end # Apollo