apollo-crawler 0.0.36 → 0.0.37

data/bin/apollo-crawler

@@ -18,6 +18,7 @@ require "optparse"
 require 'active_support'
 require 'active_support/inflector'
 
+require 'terminal-table'
 
 require File.join(File.dirname(__FILE__), '..', 'lib', 'apollo_crawler', 'version') 
 
@@ -219,16 +220,12 @@ module Crawler
 			end
 
 			if(@options[:list_plugins])
-				puts "Listing plugins"
-				puts "----------------------------------------"
-				i = 0
-				@plugins.sort.each do |plugin, klass|
-					instance = klass.new
-					# puts klass.class_eval("@@NAME")
-					puts "(#{i}) #{plugin} - #{instance.name}"
-					i += 1
-				end
-				puts "----------------------------------------"
+				headings = ['name', 'class']
+				rows = @plugins
+
+				table = Terminal::Table.new :headings => headings, :rows => rows
+				
+				puts table
 				return
 			end
 
@@ -250,7 +247,10 @@ module Crawler
 					next
 				end
 
-				# puts "Running '#{plugin}'"
+				if(@options[:verbose])
+					puts "Running '#{plugin}'"
+				end
+
 				res = p.new.etl
 				if(res.nil?)
 					next

data/lib/apollo_crawler/plugin.rb

@@ -15,38 +15,31 @@ module Apollo
 					return nil
 				end
 
-				def etl()
-					#return run()
-					res = []
-
-					if(self.url.nil?)
+				def etl(url=nil)
+					# Look for passed URL use default instead and fail if it is not valid
+					url = url ? url : self.url
+					if(url.nil?)
 						return nil
 					end
 
-					doc = self.fetch_document(self.url)
+					# Try fetch document
+					doc = self.fetch_document(url)
 					if(doc.nil?)
 						return nil
 					end
 
-					res = self.extract_data(doc)
+					# Try extract data from document
+					data = self.extract_data(doc)
+
+					# Try extract links for another documents 
+					links = self.extract_links(doc)
 
+					# Return ETL result
 					return { 
 						:plugin => self.class.name,
 						:title => doc.title,
-						:data => res
-					}
-				end
-
-				# - Fetch default URL (and transform it to document)
-				# - Extract and Load (Store) important data
-				# - Look for another documents
-				#    Examples:
-				#     - "next page"
-				#     - "people you may know on Linked in"
-				#     - "will attend on FB")
-				def run
-					return { 
-						:plugin => self.class.name
+						:data => data,
+						:links => links
 					}
 				end
 

data/lib/apollo_crawler/version.rb

@@ -1,5 +1,5 @@
 module Apollo
 	module Crawler
-		VERSION = '0.0.36'
+		VERSION = '0.0.37'
 	end # Crawler
 end # Apollo

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: apollo-crawler
 version: !ruby/object:Gem::Version
-  version: 0.0.36
+  version: 0.0.37
   prerelease: 
 platform: ruby
 authors:
@@ -123,6 +123,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: terminal-table
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement