apollo-crawler 0.1.6 → 0.1.7

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ZjAyNjFlMDZkZTI3NjNjZjI0MjZjZmUwNjY5ZTIwM2MwMzBhNTA3NA==
+    N2FiYzUwM2Q5ZjdkMzJmZmFjMGRiOGRiZjhkMzdkZjAxNWZhZjczYg==
   data.tar.gz: !binary |-
-    ZWY0YzI5ZjMxZmNkNGI0Y2FlYWI2ODZmZGQzOWUwNzI0OTU3NjcyMg==
+    NGU2N2IwZWQzY2ExYjdjNjViZTNlNjljMWFlNmI0MGMwZjE1ODYwZQ==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    MmQ0YjM1NzZhZDk3NWM5ODBlMmNlMzVlYjE0MGRlZTM5NjQ0MWI0ZWJlZDI0
-    YzcwMGE3Zjc0NzBlMTAzZmY1MWFhNzhkMzdiNTdlZDYyM2I5Y2FhM2IzNjE5
-    MjAzZDE1ZjUyNWE3ZGU1YWYzZTJmYWYwZjAxZjI2YmRiYjY2ZWY=
+    ZjBmYTY2MzYwNGNmM2Y4NWUxZjU4MTZkYzEzZTU2Y2Q4YjIxZDViNzVhYjVm
+    NzBhNzQ5ZmQxZGM4YzQyYWQ0Zjg3ZTE4NDNhNzI3NzhhNjdhYWRhNDk3MWIz
+    YTY2M2NkZjk1MWM5NGFjNzZjNjc1MjYyN2IwNWM0NzhhMmYxZDM=
   data.tar.gz: !binary |-
-    OGNkYWRkNGNlMmI0ZDhmNjgzMWY4ZjUyNTBhZWZiNDlhYWUzZDRmZmFkYzU4
-    NTIwMmZjNzE0OGQ2Yzg3M2M3YjExMzg3YjhkNGVhMjg4MjAzY2MzZTg4N2Y2
-    MzEwM2UyMGZlZDRlOGIxMjFmOTA3YzA4NjgzZGYwNTVkODEzZmQ=
+    Y2Y2NGYwNTJlMTljOTgzYTA0MTc2MTk5OTc1NzBiNmJlZDQ0ZTBkYzRjMzA4
+    ZGUyYjA4MjhjMzBjMzBlOWJhMDc3NzNkOTgyZmU1YmRjMTIwMmRkMzA1YTU2
+    M2NiZWJmMTcxYTlkOWFiMTQ3ZWMyYjZjNTA5ZWI5YTI0MjkxNjY=

data/lib/apollo_crawler/crawler/crawler_base.rb

@@ -27,11 +27,21 @@ module Apollo
 				end
 			end
 
+			def self.try_get_doc(root, url)
+				doc = CrawlerBase.try_get_url(root, url)
+				
+				# TODO: Set experition header
+				return {
+					:doc => doc,
+					:url => url
+				}
+			end
+
 			# - (0) Figure out URL
 			# - (1) Extract Data
 			# - (2) Extract Links
 			# - (3) Go to (0) eventually
-			def etl(url=nil, &block)
+			def etl(url=nil, opts={}, &block)
 				# Look for passed URL use default instead and fail if it is not valid
 				if(url.nil? || url.empty?)
 					url = self.url
@@ -47,6 +57,8 @@ module Apollo
 					@backlog << url
 				end
 
+				docs_processed = 0
+
 				res = []
 				# TODO: Respect limit of documents/urls processed
 				while(@backlog.empty? == false)
@@ -72,6 +84,10 @@ module Apollo
 							@backlog << url
 						end
 					end
+
+					# Increase counter of processed documents
+					docs_processed = docs_processed + 1
+					break if opts[:doc_limit] && docs_processed >= opts[:doc_limit]
 				end
 				return res
 			end
@@ -88,7 +104,9 @@ module Apollo
 
 				# Try extract links for another documents 
 				links = self.extract_links(doc)
-				puts links.inspect
+				
+				# TODO: Make configurable if links extracted from doc should be printed
+				# puts links.inspect
 
 				# Format ETL result
 				res = { 

data/lib/apollo_crawler/crawler/google_com/google.rb

@@ -15,7 +15,7 @@ module Apollo
 
 			def extract_data(doc)
 				res = doc.xpath(@@MATCHER_ITEM).map { | node |
-					url = Crawler.try_get_url(self.url, node['href'])
+					url = CrawlerBase.try_get_url(self.url, node['href'])
 					next if url.nil?
 
 					{ 
@@ -27,7 +27,7 @@ module Apollo
 
 			def extract_links(doc)
 				res = doc.xpath("(//td[@class = 'b']/a)[last()]").map { | node |
-					url = Crawler.try_get_url(self.url, node['href'])
+					res_doc = CrawlerBase.try_get_url(self.url, node['href'])
 					next if url.nil?
 
 					{ 

data/lib/apollo_crawler/crawler/slashdot_org/slashdot.rb

@@ -15,7 +15,7 @@ module Apollo
 
 			def extract_data(doc)
 				res = doc.xpath(@@MATCHER_ITEM).map { | node |
-					url = Crawler.try_get_url(self.url, node['href'])
+					url = CrawlerBase.try_get_url(self.url, node['href'])
 					next if url.nil?
 
 					{ 
@@ -27,7 +27,7 @@ module Apollo
 
 			def extract_links(doc)
 				res = doc.xpath(@@MATCHER_ITEM).map { | node |
-					url = Crawler.try_get_url(self.url, node['href'])
+					url = CrawlerBase.try_get_url(self.url, node['href'])
 					next if url.nil?
 
 					{ 

data/lib/apollo_crawler/crawler/stackoverflow_com/stackoverflow.rb

@@ -15,7 +15,7 @@ module Apollo
 
 			def extract_data(doc)
 				res = doc.xpath(@@MATCHER_ITEM).map { |node|
-					url = Crawler.try_get_url(self.url, node['href'])
+					url = CrawlerBase.try_get_url(self.url, node['href'])
 					next if url.nil?
 
 					{ 
@@ -29,7 +29,7 @@ module Apollo
 
 			def extract_links(doc)
 				res = doc.xpath("(//div[@class = 'pager fl']/a)[last()]").map { |node|
-					url = Crawler.try_get_url(self.url, node['href'])
+					url = CrawlerBase.try_get_url(self.url, node['href'])
 					next if url.nil?
 
 					{ 

data/lib/apollo_crawler/crawler/ycombinator_com/hacker_news.rb

@@ -15,7 +15,7 @@ module Apollo
 
 			def extract_data(doc)
 				res = doc.xpath(@@MATCHER_ITEM).map { |node|
-					url = Crawler.try_get_url(self.url, node['href'])
+					url = CrawlerBase.try_get_url(self.url, node['href'])
 					next if url.nil?
 
 					{ 
@@ -29,7 +29,7 @@ module Apollo
 
 			def extract_links(doc)
 				res = doc.xpath("(//a[@class = 'prevnextbutact'])").map { |node|
-					url = Crawler.try_get_url(self.url, node['href'])
+					url = CrawlerBase.try_get_url(self.url, node['href'])
 					next if url.nil?
 
 					{ 

data/lib/apollo_crawler/program.rb

@@ -45,6 +45,7 @@ module Apollo
 		# Initialize command-line options
 		def init_options
 			@options = {}
+			@options[:doc_limit] = nil
 			@options[:verbose] = false
 			@options[:version] = false
 			@options[:cache_dirs] = [
@@ -86,6 +87,10 @@ module Apollo
 					@options[:crawler_dirs] << path
 				end
 
+				opts.on('-n', '--doc-limit [NUM]', 'Limit count of documents to be processed') do |count|
+					@options[:doc_limit] = count.to_i
+				end
+
 				opts.on('-v', '--verbose', 'Enable verbose output') do
 					@options[:verbose] = true
 				end
@@ -236,6 +241,23 @@ module Apollo
 			end
 		end
 
+		def register_modules()
+			# Register caches which can be used
+			@options[:cache_dirs].each do |dir|
+				register_cache(dir)
+			end
+
+			# Register sites which can be crawled
+			@options[:crawler_dirs].each do |dir|
+				register_crawlers(dir)
+			end
+
+			# Register sites which can be crawled
+			@options[:formatter_dirs].each do |dir|
+				register_formatters(dir)
+			end
+		end
+
 		def generate_crawler(name, url = nil, matcher = nil)
 			name = name.titleize.gsub(" ", "")
 
@@ -281,8 +303,23 @@ module Apollo
 			end  
 		end
 
+		def self.console_table(headings, rows)
+			table = Terminal::Table.new :headings => headings, :rows => rows
+			puts table
+		end
+
+		def list_crawlers()
+			CrawlerProgram.console_table(['name', 'class'], @crawlers)
+			return
+		end
+
+		def list_formatters()
+			CrawlerProgram.console_table(['name', 'class'], @formatters)
+			return
+		end
+
 		def run(args = ARGV)
-			puts "#{ARGV.inspect}"
+			# puts "#{ARGV.inspect}"
 
 			init_options()
 
@@ -309,20 +346,7 @@ module Apollo
 				return 0
 			end
 
-			# Register caches which can be used
-			@options[:cache_dirs].each do |dir|
-				register_cache(dir)
-			end
-
-			# Register sites which can be crawled
-			@options[:crawler_dirs].each do |dir|
-				register_crawlers(dir)
-			end
-
-			# Register sites which can be crawled
-			@options[:formatter_dirs].each do |dir|
-				register_formatters(dir)
-			end
+			register_modules()
 
 			# Set default formatter here
 			formatter_name = "json"
@@ -340,22 +364,12 @@ module Apollo
 			end
 
 			if(@options[:list_formatters])
-				headings = ['name', 'class']
-				rows = @formatters
-
-				table = Terminal::Table.new :headings => headings, :rows => rows
-				
-				puts table
+				list_formatters()
 				return 0
 			end
 
 			if(@options[:list_crawlers])
-				headings = ['name', 'class']
-				rows = @crawlers
-
-				table = Terminal::Table.new :headings => headings, :rows => rows
-				
-				puts table
+				list_crawlers()
 				return 0
 			end
 
@@ -385,7 +399,11 @@ module Apollo
 					puts "Running '#{crawler}'"
 				end
 
-				res = p.new.etl(args) { | docs |
+				opts = {
+					:doc_limit => @options[:doc_limit]
+				}
+
+				res = p.new.etl(args, opts) { | docs |
 					if(docs.nil?)
 						next
 					end

data/lib/apollo_crawler/version.rb

@@ -1,3 +1,3 @@
 module Apollo
-	VERSION = '0.1.6'
+	VERSION = '0.1.7'
 end # Apollo

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: apollo-crawler
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors:
 - Tomas Korcak