apollo-crawler 0.1.11 → 0.1.12

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    M2EwYTQzNjFlZWY1YWFhOTNkYjAyNzYzODM1ZWRiYWRkOWQzYzk5MQ==
+    MjkxYjlmY2NjMDYwODcxN2JmMzA1MDM3NzM5NzQ1ZWVhNDNiYWQ0MQ==
   data.tar.gz: !binary |-
-    YTU3NzEwMzQ1OGYyNGNkMDM5YjdmMTA0ODA3OGI0ZTQ4YjMxNTQwZA==
+    N2ViMTdhNmQ1OGM3ZTczYjIxZWU1Y2NlY2NlYWMxMDM1MDkwZjBjYg==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    OTYxMWI5ZmFmYTg3NWU5ZGI2YjcwY2UzN2UxNzIwN2E0OTU3YmEzNGNlMmQ0
-    YmE5NmNmY2MxNjMyNmY5MzgxNjc3ZWIxMzY1ZjY5ZTVkNjFmMzE5ZWQxZGI0
-    YTY1YzcyMDc4Y2MxN2FlYmZiZTY1ZGI2YjQxOTViN2ZiY2M3MjQ=
+    NmNiNjMzZGFjY2ZmOTNjOTU4NTAxMGQzODZlNGYyOWI2MGQzY2YwM2Q4ZmMw
+    NDMzYzM2OTNkYmU2MzJiYzNhYzMwNGNmZDI0OWZiM2ZiZjJiYjFkMWExY2Rh
+    NTA3MDkxNTA1OTk1NWE5ZWMyNGFiZjY5ODhiMDMxZDU5NjgwZDU=
   data.tar.gz: !binary |-
-    YzYwZTdmMTUxMWZiY2ZiZTBhYzU4OTUxZmQxNTE2NTYyY2Q0YThmNTA1OWU4
-    NDc4YTMzNTBhNzU4ODIwNzVlZDcyYTlmNjBlODE0MDM2YTI4ZWY4ZWY3NzMw
-    ODM1MjExYjE1YmYzNDAwY2I0NmIzZDVhMDQyYTY4ODk5NzVjZjI=
+    MWI1YWI3NjcwYzQ1NWVkZDI2ZjM3NjY5MGRjMzZkMDQxMThlMWE2MGU3MzAx
+    ZWE5ZWIyYWExNjdjMjYyYjUxNTU5MWZlNGI5MWUzOWYwZGI2NjQ2YTNkMTIy
+    YmRiNzIzNGY5ZThlNTdkMzIwODJkNjc2ZWUyNzQ5MWNlOWZlM2I=

data/lib/apollo_crawler.rb

@@ -29,6 +29,7 @@ require File.join(File.dirname(__FILE__), 'apollo_crawler/cache/base_cache')
 require File.join(File.dirname(__FILE__), 'apollo_crawler/cache/factory')
 require File.join(File.dirname(__FILE__), 'apollo_crawler/cache/memcached_cache')
 require File.join(File.dirname(__FILE__), 'apollo_crawler/cache/memory_cache')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/cache/mongo_cache')
 require File.join(File.dirname(__FILE__), 'apollo_crawler/cache/null_cache')
 
 # Crawlers

data/lib/apollo_crawler/cache/mongo_cache.rb

@@ -0,0 +1,54 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require File.join(File.dirname(__FILE__), 'base_cache') 
+
+require 'mongo'
+
+module Apollo
+	module Cache
+		class MongoCache < BaseCache
+			def initialize
+				@mongo_client = Mongo::MongoClient.new('localhost', 27017, :pool_size => 5, :pool_timeout => 5)
+				@db = @mongo_client['apollo-crawler']
+			end
+
+			# Get value associated with key from cache
+			def get(key, *args)
+				res = @db['docs'].find({:url => key})
+
+				# Not found, Create, cache and return
+				if res.nil? || res.count < 1 && block_given?
+					res = yield args
+					return self.set(key, res)
+				end
+
+				return res.to_a[0]
+			end
+
+			# Set value associated with key 
+			# Return cached value
+			def set(key, value)
+				@db['docs'].insert(value)
+				return value
+			end
+		end # MongoCache
+	end # Cache
+end # Apollo

data/lib/apollo_crawler/crawler/base_crawler.rb

@@ -150,7 +150,6 @@ module Apollo
 				# Format ETL result
 				res = { 
 					:crawler => self.class.name,
-					:title => doc.title,
 					:data => data,
 					:links => links
 				}
@@ -158,6 +157,17 @@ module Apollo
 				return res
 			end
 
+			def self.create_metadoc(url, doc)
+				return {
+					'url' => url,
+					'doc' => doc.encode('utf-8'),
+					'hash' => Digest::SHA256.new.update(doc).hexdigest,
+					'created_at' => Time.now.utc,
+					'expires_at' => nil,
+					'version' => 0
+				}
+			end
+
 			# Fetch document
 			def fetch_document(url)
 				# TODO: Refactor following idiom
@@ -171,15 +181,15 @@ module Apollo
 
 				# TODO: Use some (custom-made) low-level HTTTP Protocol cache - just for sure
 				cache = Apollo::Cache::Factory.instance.construct
-				raw = cache.get(url) do
+				metadoc = cache.get(url) do
 					max_attempts = 3
 					attempt_no = 0
 					success = false
 					
-					res = nil
+					doc = nil
 					while(attempt_no < max_attempts && success == false) do
 						begin
-							res = BaseCrawler.fetch(url)
+							doc = BaseCrawler.fetch(url)
 							success = true
 						rescue Exception => e
 							puts "EXCEPTION: Unable to fetch '#{url}', reason: '#{e.to_s}'"
@@ -190,12 +200,12 @@ module Apollo
 						end
 					end
 
-					res
+					# Create metadata
+					BaseCrawler.create_metadoc(url, doc)
 				end
 
 				# TODO: Encapsulate and make more robust => invalid hostname, timeouts and so
-				doc = Nokogiri::HTML(raw)
-				return doc
+				return Nokogiri::HTML(metadoc['doc'])
 			end
 
 			# Extracts data from document

data/lib/apollo_crawler/fetcher/base_fetcher.rb

@@ -18,14 +18,16 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+require "open-uri"
+require "nokogiri"
+
 module Apollo
 	module Fetcher
 		class BaseFetcher
 			def self.fetch(url)
 				# TODO: Throw exception ???
-				return nil
-			end
-			
+				return open(url).read
+			end			
 		end # class BaseFetcher
 	end # module Fetcher
 end # module Apollo

data/lib/apollo_crawler/fetcher/simple_fetcher.rb

@@ -28,7 +28,7 @@ module Apollo
 		class SimpleFetcher < BaseFetcher
 			def self.fetch(url)
 				# TODO: Throw exception ???
-				return open(url).read
+				return BaseFetcher::fetch(url)
 			end
 		end # class SimpleFetcher
 	end # module Fetcher

data/lib/apollo_crawler/fetcher/smart_fetcher.rb

@@ -38,11 +38,11 @@ module Apollo
 					sleep(diff)
 				end
 
-				res = open(url).read
+				res = BaseFetcher::fetch(url)
 
 				@@LAST_FETCH = DateTime.now
 				return res
 			end
 		end # class SimpleFetcher
 	end # module SmartFetcher
-end # module Apollo
+end # module Apollo

data/lib/apollo_crawler/lib.rb

@@ -26,6 +26,7 @@ require File.join(File.dirname(__FILE__), 'cache/base_cache')
 require File.join(File.dirname(__FILE__), 'cache/factory')
 require File.join(File.dirname(__FILE__), 'cache/memcached_cache')
 require File.join(File.dirname(__FILE__), 'cache/memory_cache')
+require File.join(File.dirname(__FILE__), 'cache/mongo_cache')
 require File.join(File.dirname(__FILE__), 'cache/null_cache')
 
 # Crawlers

data/lib/apollo_crawler/program.rb

@@ -65,18 +65,23 @@ module Apollo
 		# Initialize command-line options
 		def init_options
 			@options = {}
+			
 			@options[:doc_limit] = nil
 			@options[:verbose] = false
 			@options[:version] = false
+			
 			@options[:cache_dirs] = [
 				RbConfig::CACHES_DIR
 			]
+			
 			@options[:crawler_dirs] = [
 				RbConfig::CRAWLERS_DIR
 			]
+			
 			@options[:formatter_dirs] = [
 				RbConfig::FORMATTERS_DIR
 			]
+
 			@options[:generate_crawler] = nil
 
 			@optparser = OptionParser.new do | opts |
@@ -139,6 +144,56 @@ module Apollo
 			@optparser.parse!(args)
 		end
 
+		def process_options(args)
+			if(@options[:version])
+				puts Apollo::VERSION
+				return 0
+			end
+
+			if(@options[:show_help])
+				puts @optparser
+				return 0
+			end
+
+			if(@options[:generate_crawler])
+				name = @options[:generate_crawler]
+				url = args.length > 0 ? args[0] : nil
+				matcher = args.length > 1 ? args[1] : nil
+				
+				return self.generate_crawler(name, url, matcher)
+			end
+
+			if(@options[:list_formatters])
+				list_formatters()
+				return 0
+			end
+
+			if(@options[:list_crawlers])
+				list_crawlers()
+				return 0
+			end
+
+			return nil
+		end
+
+		def init_formatter()
+			# Set default formatter here
+			formatter_name = "json"
+			if(@options[:formatter])
+				formatter_name = @options[:formatter]
+			end
+
+			# Look for specified formatter
+			f = @formatters.select { |k, v|
+				name = formatter_name.gsub(Apollo::Formatter::BaseFormatter::name_re, "")
+				k.downcase == name
+			}
+
+			if(f)
+				@formatter = f[f.keys[0]] 
+			end
+		end
+
 		# Load global options first
 		# Merge it with local options (if they exists)
 		def load_config_file()
@@ -328,96 +383,31 @@ module Apollo
 			return 0
 		end
 
+		# Show tabular data in form of CLI table
 		def self.console_table(headings, rows)
 			table = Terminal::Table.new :headings => headings, :rows => rows
 			puts table
 		end
 
+		# List available crawlers
 		def list_crawlers()
 			CrawlerProgram.console_table(['name', 'class'], @crawlers)
 			return
 		end
 
+		# List available formatters
 		def list_formatters()
 			CrawlerProgram.console_table(['name', 'class'], @formatters)
 			return
 		end
 
-		def run(args = ARGV)
-			# puts "#{ARGV.inspect}"
-
-			init_options()
-
-			parse_options(args)
-
-			if(@options[:version])
-				puts Apollo::VERSION
-				return 0
-			end
-
-			if(@options[:show_help])
-				puts @optparser
-				return 0
-			end
-
-			load_config_file()
-
-			if(@options[:generate_crawler])
-				name = @options[:generate_crawler]
-				url = args.length > 0 ? args[0] : nil
-				matcher = args.length > 1 ? args[1] : nil
-				
-				return self.generate_crawler(name, url, matcher)
-			end
-
-			register_modules()
-
-			# Set default formatter here
-			formatter_name = "json"
-			if(@options[:formatter])
-				formatter_name = @options[:formatter]
-			end
-
-			# Look for specified formatter
-			f = @formatters.select { |k, v|
-				name = formatter_name.gsub(Apollo::Formatter::BaseFormatter::name_re, "")
-				k.downcase == name
-			}
-
-			if(f)
-				@formatter = f[f.keys[0]] 
-			end
-
-			if(@options[:list_formatters])
-				list_formatters()
-				return 0
-			end
-
-			if(@options[:list_crawlers])
-				list_crawlers()
-				return 0
-			end
-
-			crawlers = []
-			if(args.length > 0)
-				crawlers << args.shift
-			end
-
-			if(@options[:run_all])
-				crawlers = @crawlers.keys
-			end
-
-			if(crawlers.empty?)
-				puts @optparser
-				return 0
-			end
-
-			crawlers.each do |crawler|
-				crawler_name = crawler.downcase.gsub(Apollo::Crawler::BaseCrawler.name_re, "")
+		def run_crawlers(crawlers, args)
+			crawlers.each do |name|
+				crawler_name = name.downcase.gsub(Apollo::Crawler::BaseCrawler.name_re, "")
 
-				p = @crawlers[crawler_name]
-				if(p == nil)
-					puts "Invalid crawler name - '#{crawler}'"
+				crawler = @crawlers[crawler_name]
+				if(crawler == nil)
+					puts "Invalid crawler name - '#{name}'"
 					puts "See program help"
 					return 0
 				end
@@ -430,7 +420,7 @@ module Apollo
 					:doc_limit => @options[:doc_limit]
 				}
 
-				res = p.new.etl(args, opts) { | docs |
+				res = crawler.new.etl(args, opts) { | docs |
 					if(docs.nil?)
 						next
 					end
@@ -447,5 +437,50 @@ module Apollo
 
 			return 0
 		end
+
+		# Get crawlers passd to cmd-line
+		def get_crawlers(args)
+			crawlers = []
+			if(args.length > 0)
+				crawlers << args.shift
+			end
+
+			if(@options[:run_all])
+				crawlers = @crawlers.keys
+			end
+
+			return crawlers
+		end
+
+		# Init program
+		def init_program(args)
+			init_options()
+
+			parse_options(args)
+
+			load_config_file()
+
+			register_modules()
+
+			res = process_options(args)
+			if res != nil
+				return res
+			end
+
+			init_formatter()
+		end
+
+		# Run Program
+		def run(args = ARGV)
+			init_program(args)
+
+			crawlers = get_crawlers(args)
+			if(crawlers.empty?)
+				puts @optparser
+				return 0
+			end
+
+			return run_crawlers(crawlers, args)
+		end
 	end
 end

data/lib/apollo_crawler/version.rb

@@ -19,5 +19,5 @@
 # THE SOFTWARE.
 
 module Apollo
-	VERSION = '0.1.11'
+	VERSION = '0.1.12'
 end # Apollo

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: apollo-crawler
 version: !ruby/object:Gem::Version
-  version: 0.1.11
+  version: 0.1.12
 platform: ruby
 authors:
 - Tomas Korcak
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-28 00:00:00.000000000 Z
+date: 2013-03-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: amqp
@@ -80,6 +80,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 1.5.5
+- !ruby/object:Gem::Dependency
+  name: ffi
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: eventmachine
   requirement: !ruby/object:Gem::Requirement
@@ -136,6 +150,34 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: mongo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mongoid
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: mime-types
   requirement: !ruby/object:Gem::Requirement
@@ -220,6 +262,48 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Gem for crawling data from external sources
 email: korczis@gmail.com
 executables:
@@ -240,6 +324,7 @@ files:
 - ./lib/apollo_crawler/cache/null_cache.rb
 - ./lib/apollo_crawler/cache/memory_cache.rb
 - ./lib/apollo_crawler/cache/base_cache.rb
+- ./lib/apollo_crawler/cache/mongo_cache.rb
 - ./lib/apollo_crawler/cache/memcached_cache.rb
 - ./lib/apollo_crawler/crawler/xkcd_crawler.rb
 - ./lib/apollo_crawler/crawler/google_crawler.rb