apollo-crawler 0.1.5 → 0.1.6

data/lib/apollo_crawler/{crawler_template.rb → crawler/crawler_template.rb}

@@ -1,24 +1,24 @@
-module Apollo
-	module Crawlers
-		class CRAWLER_CLASS_NAME < Crawler
-			@@MATCHER_ITEM = "CRAWLER_MATCHER"
-
-			def name()
-				return "CRAWLER_NAME"
-			end
-
-			def url()
-				return "CRAWLER_URL"
-			end
-
-			def extract_data(doc)
-				res = doc.xpath(@@MATCHER_ITEM).map { |i|
-					{ 
-						:text => i.text,
-						:link => URI.join(self.url, i['href'])
-					}
-				}
-			end
-		end # CRAWLER_CLASS_NAME
-	end # Crawlers
-end # Apollo
+module Apollo
+	module Crawler
+		class CRAWLER_CLASS_NAME < Crawler
+			@@MATCHER_ITEM = "CRAWLER_MATCHER"
+
+			def name()
+				return "CRAWLER_NAME"
+			end
+
+			def url()
+				return "CRAWLER_URL"
+			end
+
+			def extract_data(doc)
+				res = doc.xpath(@@MATCHER_ITEM).map { |i|
+					{ 
+						:text => i.text,
+						:link => URI.join(self.url, i['href'])
+					}
+				}
+			end
+		end # CRAWLER_CLASS_NAME
+	end # Crawler
+end # Apollo

data/lib/apollo_crawler/{crawlers → crawler}/google_com/google.rb

@@ -1,40 +1,40 @@
-require File.join(File.dirname(__FILE__), '..', '..', 'crawler') 
-
-module Apollo
-	module Crawlers
-		class Google < Crawler
-			@@MATCHER_ITEM = "//h3/a"
-
-			def name()
-				return "Google"
-			end
-
-			def url()
-				return "http://www.google.com/search?q=ruby"
-			end
-
-			def extract_data(doc)
-				res = doc.xpath(@@MATCHER_ITEM).map { | node |
-					url = Crawler.try_get_url(self.url, node['href'])
-					next if url.nil?
-
-					{ 
-						:text => node.text,
-						:link => url
-					}
-				}
-			end
-
-			def extract_links(doc)
-				res = doc.xpath("(//td[@class = 'b']/a)[last()]").map { | node |
-					url = Crawler.try_get_url(self.url, node['href'])
-					next if url.nil?
-
-					{ 
-						:link => url
-					}
-				}
-			end
-		end
-	end # Crawlers
-end # Apollo
+require File.join(File.dirname(__FILE__), '..', 'crawler_base') 
+
+module Apollo
+	module Crawler
+		class Google < CrawlerBase
+			@@MATCHER_ITEM = "//h3/a"
+
+			def name()
+				return "Google"
+			end
+
+			def url()
+				return "http://www.google.com/search?q=ruby"
+			end
+
+			def extract_data(doc)
+				res = doc.xpath(@@MATCHER_ITEM).map { | node |
+					url = Crawler.try_get_url(self.url, node['href'])
+					next if url.nil?
+
+					{ 
+						:text => node.text,
+						:link => url
+					}
+				}
+			end
+
+			def extract_links(doc)
+				res = doc.xpath("(//td[@class = 'b']/a)[last()]").map { | node |
+					url = Crawler.try_get_url(self.url, node['href'])
+					next if url.nil?
+
+					{ 
+						:link => url
+					}
+				}
+			end
+		end
+	end # Crawler
+end # Apollo

data/lib/apollo_crawler/{crawlers → crawler}/slashdot_org/slashdot.rb

@@ -1,40 +1,40 @@
-require File.join(File.dirname(__FILE__), '..', '..', 'crawler') 
-
-module Apollo
-	module Crawlers
-		class Slashdot < Crawler
-			@@MATCHER_ITEM = "//article/header/h2/span/a"
-
-			def name()
-				return "Slashdot"
-			end
-
-			def url()
-				return"http://slashdot.org/"
-			end
-
-			def extract_data(doc)
-				res = doc.xpath(@@MATCHER_ITEM).map { | node |
-					url = Crawler.try_get_url(self.url, node['href'])
-					next if url.nil?
-
-					{ 
-						:text => node.text,
-						:link => url
-					}
-				}
-			end
-
-			def extract_links(doc)
-				res = doc.xpath(@@MATCHER_ITEM).map { | node |
-					url = Crawler.try_get_url(self.url, node['href'])
-					next if url.nil?
-
-					{ 
-						:link => url
-					}
-				}
-			end
-		end
-	end # Crawlers
-end # Apollo
+require File.join(File.dirname(__FILE__), '..', 'crawler_base') 
+
+module Apollo
+	module Crawler
+		class Slashdot < CrawlerBase
+			@@MATCHER_ITEM = "//article/header/h2/span/a"
+
+			def name()
+				return "Slashdot"
+			end
+
+			def url()
+				return"http://slashdot.org/"
+			end
+
+			def extract_data(doc)
+				res = doc.xpath(@@MATCHER_ITEM).map { | node |
+					url = Crawler.try_get_url(self.url, node['href'])
+					next if url.nil?
+
+					{ 
+						:text => node.text,
+						:link => url
+					}
+				}
+			end
+
+			def extract_links(doc)
+				res = doc.xpath(@@MATCHER_ITEM).map { | node |
+					url = Crawler.try_get_url(self.url, node['href'])
+					next if url.nil?
+
+					{ 
+						:link => url
+					}
+				}
+			end
+		end
+	end # Crawler
+end # Apollo

data/lib/apollo_crawler/{crawlers → crawler}/stackoverflow_com/stackoverflow.rb

@@ -1,44 +1,44 @@
-require File.join(File.dirname(__FILE__), '..', '..', 'crawler') 
-
-module Apollo
-	module Crawlers
-		class StackOverflow < Crawler
-			@@MATCHER_ITEM = "//div[@class = 'summary']/h3/a"
-
-			def name()
-				return "Stackoverflow"
-			end
-
-			def url()
-				return "http://stackoverflow.com/questions"
-			end
-
-			def extract_data(doc)
-				res = doc.xpath(@@MATCHER_ITEM).map { |node|
-					url = Crawler.try_get_url(self.url, node['href'])
-					next if url.nil?
-
-					{ 
-						:text => node.text,
-						:link => url
-					}
-				}
-
-				return res
-			end
-
-			def extract_links(doc)
-				res = doc.xpath("(//div[@class = 'pager fl']/a)[last()]").map { |node|
-					url = Crawler.try_get_url(self.url, node['href'])
-					next if url.nil?
-
-					{ 
-						:link => url
-					}
-				}
-				
-				return res.uniq
-			end
-		end
-	end # Crawlers
-end # Apollo
+require File.join(File.dirname(__FILE__), '..', 'crawler_base') 
+
+module Apollo
+	module Crawler
+		class StackOverflow < CrawlerBase
+			@@MATCHER_ITEM = "//div[@class = 'summary']/h3/a"
+
+			def name()
+				return "Stackoverflow"
+			end
+
+			def url()
+				return "http://stackoverflow.com/questions"
+			end
+
+			def extract_data(doc)
+				res = doc.xpath(@@MATCHER_ITEM).map { |node|
+					url = Crawler.try_get_url(self.url, node['href'])
+					next if url.nil?
+
+					{ 
+						:text => node.text,
+						:link => url
+					}
+				}
+
+				return res
+			end
+
+			def extract_links(doc)
+				res = doc.xpath("(//div[@class = 'pager fl']/a)[last()]").map { |node|
+					url = Crawler.try_get_url(self.url, node['href'])
+					next if url.nil?
+
+					{ 
+						:link => url
+					}
+				}
+				
+				return res.uniq
+			end
+		end
+	end # Crawler
+end # Apollo

data/lib/apollo_crawler/{crawlers → crawler}/xkcd_com/xkcd.rb

@@ -1,35 +1,35 @@
-require File.join(File.dirname(__FILE__), '..', '..', 'crawler') 
-
-module Apollo
-	module Crawlers
-		class Xkcd < Crawler
-			@@MATCHER_ITEM = "//div[@id = 'comic']/img"
-
-			def name()
-				return "Xkcd"
-			end
-
-			def url()
-				return "http://xkcd.com/"
-			end
-
-			def extract_data(doc)
-				res = doc.xpath(@@MATCHER_ITEM).map { |node|
-					{ 
-						:text => node['title'],
-						:link => URI.join(self.url, node['src']),
-					}
-				}
-			end
-
-			def extract_links(doc)
-				res = doc.xpath("//ul[@class = 'comicNav']/li/a[@accesskey = 'p']").map { |node|
-					{ 
-						:link => URI.join(self.url, node['href']),
-					}
-				}
-				res.uniq
-			end
-		end
-	end # Crawlers
-end # Apollo
+require File.join(File.dirname(__FILE__), '..', 'crawler_base') 
+
+module Apollo
+	module Crawler
+		class Xkcd < CrawlerBase
+			@@MATCHER_ITEM = "//div[@id = 'comic']/img"
+
+			def name()
+				return "Xkcd"
+			end
+
+			def url()
+				return "http://xkcd.com/"
+			end
+
+			def extract_data(doc)
+				res = doc.xpath(@@MATCHER_ITEM).map { |node|
+					{ 
+						:text => node['title'],
+						:link => URI.join(self.url, node['src']),
+					}
+				}
+			end
+
+			def extract_links(doc)
+				res = doc.xpath("//ul[@class = 'comicNav']/li/a[@accesskey = 'p']").map { |node|
+					{ 
+						:link => URI.join(self.url, node['href']),
+					}
+				}
+				res.uniq
+			end
+		end
+	end # Crawler
+end # Apollo

data/lib/apollo_crawler/{crawlers → crawler}/ycombinator_com/hacker_news.rb

@@ -1,44 +1,44 @@
-require File.join(File.dirname(__FILE__), '..', '..', 'crawler') 
-
-module Apollo
-	module Crawlers
-		class HackerNews < Crawler
-			@@MATCHER_ITEM = "(//td[@class = 'title']/a)[not(position() > last() -1)]"
-
-			def name()
-				return "Hacker News"
-			end
-
-			def url()
-				return "http://news.ycombinator.com/"
-			end
-
-			def extract_data(doc)
-				res = doc.xpath(@@MATCHER_ITEM).map { |node|
-					url = Crawler.try_get_url(self.url, node['href'])
-					next if url.nil?
-
-					{ 
-						:text => node.text,
-						:link => url
-					}
-				}
-
-				return res
-			end
-
-			def extract_links(doc)
-				res = doc.xpath("(//a[@class = 'prevnextbutact'])").map { |node|
-					url = Crawler.try_get_url(self.url, node['href'])
-					next if url.nil?
-
-					{ 
-						:link => url
-					}
-				}
-				
-				return res.uniq
-			end
-		end
-	end # Crawlers
-end # Apollo
+require File.join(File.dirname(__FILE__), '..', 'crawler_base') 
+
+module Apollo
+	module Crawler
+		class HackerNews < CrawlerBase
+			@@MATCHER_ITEM = "(//td[@class = 'title']/a)[not(position() > last() -1)]"
+
+			def name()
+				return "Hacker News"
+			end
+
+			def url()
+				return "http://news.ycombinator.com/"
+			end
+
+			def extract_data(doc)
+				res = doc.xpath(@@MATCHER_ITEM).map { |node|
+					url = Crawler.try_get_url(self.url, node['href'])
+					next if url.nil?
+
+					{ 
+						:text => node.text,
+						:link => url
+					}
+				}
+
+				return res
+			end
+
+			def extract_links(doc)
+				res = doc.xpath("(//a[@class = 'prevnextbutact'])").map { |node|
+					url = Crawler.try_get_url(self.url, node['href'])
+					next if url.nil?
+
+					{ 
+						:link => url
+					}
+				}
+				
+				return res.uniq
+			end
+		end
+	end # Crawler
+end # Apollo