apollo-crawler 0.1.24 → 0.1.25

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 71fb379b6ae32ceb79e40cce451c8a3646278d32
-  data.tar.gz: 08fdec629298945a86993b91be3a743b880ad4a0
+  metadata.gz: 79e9ecdfed577a1ce13b74b24d6d5bc26bf75843
+  data.tar.gz: 6d93c6da6316d4666ddc5e434bab1caadc213ba3
 SHA512:
-  metadata.gz: d0789d2ef99358144c90d148c378d9bf53e3084b326ede425ed7ff171ab450a4ed9de7a86494dbf49992ed235807f92e6a795cc52676bd61280a006408fc4e90
-  data.tar.gz: 16ac99fc7e192fe137348c6d364e730c9c0071f274f200b395746c4247bb3958c7238b315b571a730014b5f8286602bbc67eb9f26db30c217440e15401d3ac95
+  metadata.gz: 863d10a255722bd53c9ee998e2886fd86d04cf7808284323d33f7da1fe77fc99ac0874f1e2a61f20c8656e039bc3bac2d2a9cef911e9b6e6d12266e91636b3bc
+  data.tar.gz: db77a2d4606dcecbec1ae2e0d872cc41f6fea64012280e179b1147d8bbabfe7eef5446bf364978ee930f338cb367624e30a86e11fe0613e4499e03dcbc670e4b

data/lib/apollo_crawler.rb

@@ -57,5 +57,8 @@ require File.join(File.dirname(__FILE__), 'apollo_crawler/planner/planners')
 # Program
 require File.join(File.dirname(__FILE__), 'apollo_crawler/program/programs')
 
+# Scheduler
+require File.join(File.dirname(__FILE__), 'apollo_crawler/scheduler/schedulers')
+
 # Stores
 require File.join(File.dirname(__FILE__), 'apollo_crawler/store/stores')

data/lib/apollo_crawler/agent/agents.rb

@@ -18,5 +18,8 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+require File.join(File.dirname(__FILE__), 'exchanges')
 require File.join(File.dirname(__FILE__), 'base_agent')
+require File.join(File.dirname(__FILE__), 'domainer_agent')
+require File.join(File.dirname(__FILE__), 'crawler_agent')
 require File.join(File.dirname(__FILE__), 'fetcher_agent')

data/lib/apollo_crawler/agent/base_agent.rb

@@ -18,6 +18,8 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+require File.join(File.dirname(__FILE__), 'exchanges')
+
 module Apollo
 	module Agent
 		class BaseAgent

data/lib/apollo_crawler/agent/crawler_agent.rb

@@ -0,0 +1,77 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require File.join(File.dirname(__FILE__), 'exchanges')
+require File.join(File.dirname(__FILE__), 'base_agent')
+
+require File.join(File.dirname(__FILE__), '../crawler/crawlers')
+
+require 'nokogiri'
+
+module Apollo
+	module Agent
+		class CrawlerAgent < BaseAgent
+			attr_accessor :declarations
+
+			def initialize(amqp, opts={})
+				if(opts[:verbose])
+					puts "Initializing crawler agent..."
+				end
+
+				# Declarations
+				channel = amqp.create_channel
+				declarations = Apollo::Agent.declare_exchanges(channel, opts)
+
+				# Binding
+				declarations[:queues]["crawler.queue"].bind(declarations[:exchanges]["crawler"]).subscribe do |delivery_info, metadata, payload|
+					puts "CrawlerAgent: Received, metadata #{metadata.inspect}" if opts[:verbose]
+
+					msg = JSON.parse(payload)
+
+					request = msg['request']
+					response = msg['response']
+
+					# puts "PLANEEEER: #{msg.inspect}"
+
+					doc = Nokogiri::HTML(response['body'])
+					crawler = request['crawler_name'].constantize.new
+					data = crawler.extract_data(doc)
+					links = crawler.extract_links(doc)
+
+					# puts crawler.to_s
+					# puts res.inspect
+
+					if(metadata[:reply_to] != nil)
+						x = declarations[:exchanges][metadata[:reply_to]]
+
+						msg = {
+							:request => request,
+							:response => response,
+							:data => data,
+							:links => links
+						}
+
+						x.publish(msg.to_json)
+					end
+				end
+			end
+		end # class CrawlerAgent
+	end # module Agent
+end # module Apollo

data/lib/apollo_crawler/agent/domainer_agent.rb

@@ -0,0 +1,51 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require File.join(File.dirname(__FILE__), 'exchanges')
+require File.join(File.dirname(__FILE__), 'base_agent')
+
+require File.join(File.dirname(__FILE__), '../crawler/crawlers')
+
+require 'nokogiri'
+
+module Apollo
+	module Agent
+		class DomainerAgent < BaseAgent
+			attr_accessor :declarations
+
+			def initialize(amqp, opts={})
+				if(opts[:verbose])
+					puts "Initializing crawler agent..."
+				end
+
+				# Declarations
+				channel = amqp.create_channel
+				declarations = Apollo::Agent.declare_exchanges(channel, opts)
+
+				# Binding
+				declarations[:queues]["domainer.queue"].bind(declarations[:exchanges]["domainer"]).subscribe do |delivery_info, metadata, payload|
+					puts "DomainerAgent: Received, metadata #{metadata.inspect}" if opts[:verbose]
+
+					msg = JSON.parse(payload)
+				end
+			end
+		end # class DomainerAgent
+	end # module Agent
+end # module Apollo

data/lib/apollo_crawler/agent/exchanges.rb

@@ -0,0 +1,55 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+module Apollo
+	module Agent
+		def self.declare_exchanges(channel, opts={})
+			if(opts[:verbose])
+				puts "Declaring AMQP Exchanges"
+			end
+			
+			# Exchanges
+			exchanges = {}
+			exchanges["crawler"] = channel.direct("crawler", :auto_delete => false, :durable => true)
+			exchanges["domainer"] = channel.direct("domainer", :auto_delete => false, :durable => true)
+			exchanges["fetcher"] = channel.direct("fetcher", :auto_delete => false, :durable => true)
+			exchanges["planner.crawled"] = channel.direct("planner.crawled", :auto_delete => false, :durable => true)
+			exchanges["planner.domained"] = channel.direct("planner.domained", :auto_delete => false, :durable => true)
+			exchanges["planner.fetched"] = channel.direct("planner.fetched", :auto_delete => false, :durable => true)
+			
+			# Queues
+			queues = {}
+			queues["crawler.queue"] = channel.queue("crawler.queue", :auto_delete => false, :durable => true)
+			queues["domainer.queue"] = channel.queue("domainer.queue", :auto_delete => false, :durable => true)
+			queues["fetcher.queue"] = channel.queue("fetcher.queue", :auto_delete => false, :durable => true)
+			queues["planner.crawled.queue"] = channel.queue("planner.crawled.queue", :auto_delete => false, :durable => true)
+			queues["planner.domained.queue"] = channel.queue("planner.domained.queue", :auto_delete => false, :durable => true)
+			queues["planner.fetched.queue"] = channel.queue("planner.fetched.queue", :auto_delete => false, :durable => true)
+
+			# Compose res
+			res = {
+				:exchanges => exchanges,
+				:queues => queues				
+			}
+
+			return res
+		end
+	end
+end

data/lib/apollo_crawler/agent/fetcher_agent.rb

@@ -18,13 +18,22 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+require File.join(File.dirname(__FILE__), 'exchanges')
 require File.join(File.dirname(__FILE__), 'base_agent')
+
 require File.join(File.dirname(__FILE__), '../fetcher/fetchers')
 
+require 'digest/sha1'
+require 'thread/pool'
+
 module Apollo
 	module Agent
 		class FetcherAgent < BaseAgent
+			THREAD_POOL_SIZE = 10
+
 			attr_accessor :fetcher
+			attr_accessor :declarations
+			attr_accessor :thread_pool
 
 			def initialize(amqp, opts={})
 				self.fetcher = Apollo::Fetcher::SmartFetcher.new
@@ -33,23 +42,64 @@ module Apollo
 					puts "Initializing fetcher agent..."
 				end
 
-				ch = amqp.create_channel
-				q  = ch.queue("fetcher", :auto_delete => false, :durable => true)
-				x  = ch.default_exchange
+				thread_pool = Thread::Pool.new(THREAD_POOL_SIZE)
 
-				q.subscribe do |delivery_info, metadata, payload|
-					res = nil
+				# Declarations
+				channel = amqp.create_channel
+				declarations = Apollo::Agent.declare_exchanges(channel, opts)
 
-					puts "Received #{payload}" if opts[:verbose]
-
-					Thread.new do |t|
+				# Binding
+				declarations[:queues]["fetcher.queue"].bind(declarations[:exchanges]["fetcher"]).subscribe do |delivery_info, metadata, payload|
+					thread_pool.process {
 						queued_url = JSON.parse(payload)
-						# puts queued_url["url"]
-						# res = Apollo::Fetcher::SmartFetcher::fetch(queued_url["url"])
-						# puts "#{queued_url['url']} - " + res.inspect
-					end
+						url = queued_url["url"]
+
+						puts "FetcherAgent: Received - '#{url}', metadata #{metadata.inspect}" if opts[:verbose]
+
+						doc = nil
+						begin
+							doc = Apollo::Fetcher::SmartFetcher::fetch(url)
+						rescue Exception => e
+							puts "EXCEPTION: FetcherAgent::initialize() - Unable to fetch '#{url}', reason: '#{e.to_s}'"
+						end
+
+						doc = get_fetched_doc(queued_url, doc, metadata, opts)
+
+						# send_response_msg(metadata[:reply_to], get_response_msg(queued_url, doc))
+
+						if(metadata && metadata[:reply_to])
+							exchange_name = metadata[:reply_to]
+
+							if(exchange_name != nil)							
+								msg = get_response_msg(queued_url, doc)
+
+								x = declarations[:exchanges][exchange_name]
+								x.publish(msg.to_json)
+							end
+						end						
+					}
 				end
 			end
+
+			def get_fetched_doc(queued_url, doc, metadata, opts={})
+				url = queued_url["url"]
+
+				res = Apollo::Model::RawDocument.new
+				res.headers = doc.headers
+				res.body = doc.body
+				res.sha_hash = Digest::SHA1.hexdigest(doc.body)
+				res.status = doc.status
+				res.url = url
+
+				return res
+			end
+
+			def get_response_msg(queued_url, doc)
+				return {
+					:request => queued_url,
+					:response => doc
+				}
+			end
 		end # class FetcherAgent
 	end # module Agent
 end # module Apollo

data/lib/apollo_crawler/crawler/base_crawler.rb

@@ -172,8 +172,8 @@ module Apollo
 			def self.create_metadoc(url, doc)
 				return {
 					'url' => url,
-					'doc' => doc.encode('UTF-8', {:invalid => :replace, :undef => :replace, :replace => '?'}),
-					'hash' => Digest::SHA256.new.update(doc).hexdigest,
+					'doc' => doc.body.encode('UTF-8', {:invalid => :replace, :undef => :replace, :replace => '?'}),
+					'hash' => Digest::SHA256.new.update(doc.body).hexdigest,
 					'created_at' => Time.now.utc,
 					'expires_at' => nil,
 					'version' => 0

data/lib/apollo_crawler/crawler/slashdot_crawler.rb

@@ -46,14 +46,7 @@ module Apollo
 			end
 
 			def extract_links(doc)
-				res = doc.xpath(@@MATCHER_ITEM).map { | node |
-					url = BaseCrawler.try_get_url(self.url, node['href']).to_s
-					next if url.nil?
-
-					{ 
-						:link => url
-					}
-				}
+				return []
 			end
 		end # class SlashdotCrawler
 	end # module Crawler

data/lib/apollo_crawler/fetcher/base_fetcher.rb

@@ -24,6 +24,9 @@ require "open-uri"
 require 'faraday'
 require 'ipaddr'
 
+# require 'resolv'
+# require 'resolv-replace'
+
 module Apollo
 	module Fetcher
 		class BaseFetcher
@@ -36,7 +39,12 @@ module Apollo
 			end
 
 			def self.fetch(url, options = {})
-				uri = URI.parse(url.to_s)				
+				begin
+					uri = URI.parse(url.to_s)
+				rescue Exception => e
+					puts "EXCEPTION: BaseFetcher::fetch() - Unable to fetch: '#{e.to_s}'"
+					return nil
+				end
 
 				# See https://github.com/lostisland/faraday
 				conn = Faraday.new(:url => url) do |faraday|
@@ -46,12 +54,17 @@ module Apollo
 				end
 
 				# Make request
-				res = conn.get(uri) do |request|
-					request.headers = BaseFetcher.get_fake_headers(uri)
+				begin
+					res = conn.get(uri) do |request|
+						request.headers = BaseFetcher.get_fake_headers(uri)
+					end
+				rescue Exception => e
+					puts "EXCEPTION: BaseFetcher::fetch() - Unable to fetch: '#{e.to_s}'"
+					return nil
 				end
 
 				# Return result
-				return res.body
+				return res
 			end			
 		end # class BaseFetcher
 	end # module Fetcher

data/lib/apollo_crawler/lib.rb

@@ -54,5 +54,8 @@ require File.join(File.dirname(__FILE__), 'planner/planners')
 # Programs
 require File.join(File.dirname(__FILE__), 'program/programs')
 
+# Programs
+require File.join(File.dirname(__FILE__), 'scheduler/schedulers')
+
 # Stores
 require File.join(File.dirname(__FILE__), 'store/stores')

data/lib/apollo_crawler/model/data_source.rb

@@ -0,0 +1,37 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require File.join(File.dirname(__FILE__), 'base_model') 
+
+module Apollo
+	module Model
+		class DataSource < BaseModel
+			include Mongoid::Document
+			include Mongoid::Timestamps
+			
+			store_in collection: "data_sources"
+
+			field :url
+
+			# Indexes
+			index({ created_at: 1, updated_at: 1, url: 1 })
+		end # class DataSource
+	end # module Model
+end # module Apollo

data/lib/apollo_crawler/model/domain.rb

@@ -0,0 +1,37 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require File.join(File.dirname(__FILE__), 'base_model') 
+
+module Apollo
+	module Model
+		class Domain < BaseModel
+			include Mongoid::Document
+			include Mongoid::Timestamps
+			
+			store_in collection: "domains"
+
+			field :name
+
+			# Indexes
+			index({ created_at: 1, updated_at: 1, name: 1 })
+		end # class Domain
+	end # module Model
+end # module Apollo

data/lib/apollo_crawler/model/models.rb

@@ -20,5 +20,7 @@
 
 require File.join(File.dirname(__FILE__), 'base_model')
 require File.join(File.dirname(__FILE__), 'crawler')
+require File.join(File.dirname(__FILE__), 'data_source')
+require File.join(File.dirname(__FILE__), 'domain')
 require File.join(File.dirname(__FILE__), 'queued_url')
-require File.join(File.dirname(__FILE__), 'raw_document')
+require File.join(File.dirname(__FILE__), 'raw_document')

data/lib/apollo_crawler/model/queued_url.rb

@@ -30,9 +30,10 @@ module Apollo
 
 			field :url
 			field :state
+			field :crawler_name
 
 			# Indexes
-			index({ created_at: 1, updated_at: 1 })
+			index({ created_at: 1, updated_at: 1, crawler_name: 1 })
 		end # class QueuedUrl
 	end # module Model
 end # module Apollo

data/lib/apollo_crawler/model/raw_document.rb

@@ -28,10 +28,14 @@ module Apollo
 
 			store_in collection: "raw_docs"
 
+			field :url
+			field :headers
 			field :body
+			field :status
+			field :sha_hash
 
 			# Indexes
-			index({ created_at: 1, updated_at: 1 })
+			index({ created_at: 1, updated_at: 1, sha_hash: 1 })
 		end # class RawDocument
 	end # module Model
 end # module Apollo

data/lib/apollo_crawler/planner/smart_planner.rb

@@ -22,43 +22,101 @@ require File.join(File.dirname(__FILE__),'base_planner')
 
 require File.join(File.dirname(__FILE__),'../model/models.rb') 
 
+require File.join(File.dirname(__FILE__),'../agent/exchanges.rb') 
+
+require File.join(File.dirname(__FILE__),'../scheduler/schedulers')
+
+require 'nokogiri'
+
 module Apollo
 	module Planner
 		class SmartPlanner < BasePlanner
 			attr_accessor :amqp
 			attr_accessor :mongo
+			attr_accessor :declarations
 
-			def initialize(amqp=nil, mongo=nil)
+			def initialize(amqp=nil, mongo=nil, opts={})
 				self.amqp = amqp
 				self.mongo = mongo
-			end
 
-			def fetch_url(url, opts={})
-				puts "AMQP fetching '#{url.inspect}'"
+				# Declarations
+				channel = amqp.create_channel
+				self.declarations = Apollo::Agent.declare_exchanges(channel, opts)
 
-				ch = amqp.create_channel
-				x  = ch.default_exchange
-				x.publish(url.to_json, :routing_key => "fetcher")
+				# Bindings
+				declarations[:queues]["planner.fetched.queue"].bind(declarations[:exchanges]["planner.fetched"]).subscribe do |delivery_info, metadata, payload|
+					msg = JSON.parse(payload)
 
-			end
+					request = msg['request']
+					response = msg['response']
 
-			def fetch_queued_urls(opts={})
-				urls = Apollo::Model::QueuedUrl.where({:state => :queued})
-				return if urls.count < 1
+					doc = Apollo::Model::QueuedUrl.find(request["_id"])
+					doc.update_attributes(msg['request'])
+					doc.state = "fetched"
+					doc.save
+
+					doc = Apollo::Model::RawDocument.where(:url => request['url']).first
+					if doc
+						if doc.sha_hash != response['sha_hash']
+							puts "Removing old cached version of '#{request['url']}'" if opts[:verbose]
+							
+							doc.destroy
+							doc = nil
+						else
+							puts "Using cached version of '#{request['url']}'" if opts[:verbose]
+						end
+					else
+						doc = Apollo::Model::RawDocument.where(:sha_hash => response['sha_hash']).first
+					end
+
+					if(doc.nil?)
+						doc = Apollo::Model::RawDocument.new(response).save
+					end
+
+					# Publish
+					declarations[:exchanges]["crawler"].publish(msg.to_json, :reply_to => "planner.crawled")
+				end
+
+				declarations[:queues]["planner.domained.queue"].bind(declarations[:exchanges]["planner.domained"]).subscribe do |delivery_info, metadata, payload|
+					msg = JSON.parse(payload)
 
+					puts "DOMAINED !!!"
+				end
+
+				declarations[:queues]["planner.crawled.queue"].bind(declarations[:exchanges]["planner.crawled"]).subscribe do |delivery_info, metadata, payload|
+					msg = JSON.parse(payload)
+
+					request = msg['request']
+					response = msg['response']
+					data = msg['data']
+					links = msg['links']
+					links = [] if links.nil?
+
+					links.each do |url|
+						link = url['link']
+						
+						Apollo::Scheduler::BaseScheduler::schedule(link, request['crawler_name'])
+					end
+
+					# puts JSON.pretty_generate(data)
+					# puts JSON.pretty_generate(links)
+				end
+			end
+
+			def fetch_url(url, opts={})
 				if(opts[:verbose])
-					puts "Fetching Queued URLS"
+					puts "AMQP fetching '#{url.inspect}'"
 				end
 
-				puts "Count of URLs in Queue: #{urls.count}" if opts[:verbose]
+				# Publish
+				declarations[:exchanges]["fetcher"].publish(url.to_json, :reply_to => "planner.fetched")
+			end
 
-				urls.each do |url|
-					url.state = :fetching
-					url.save
+			def fetch_queued_urls(opts={})
+				while url = Apollo::Model::QueuedUrl.where({:state => :queued}).find_and_modify({ "$set" => { state: :fetching }}, new: true)
+					# puts "Count of URLs in Queue: #{url.count}" if opts[:verbose]
 
 					fetch_url(url, opts)
-					
-					# puts "Removing URL from Queue '#{url.inspect}'" if opts[:verbose]
 				end
 			end
 

data/lib/apollo_crawler/program/platform_program.rb

@@ -37,6 +37,7 @@ require 'eventmachine'
 require 'em-http'
 
 require 'fileutils'
+require 'csv'
 
 require 'mongoid'
 
@@ -44,6 +45,8 @@ require File.join(File.dirname(__FILE__), '..', 'version')
 
 require File.join(File.dirname(__FILE__),'base_program')
 
+require File.join(File.dirname(__FILE__),'../scheduler/schedulers')
+
 module Apollo
 	# Apollo Crawler Base Directory
 	APOLLO_PLATFORM_BASE_DIR = File.join(File.dirname(__FILE__), "..")
@@ -97,29 +100,56 @@ module Apollo
 		def enqueue_crawlers_urls(amqp, crawlers=Apollo::Crawler::BaseCrawler.subclasses, opts={})
 			crawlers.each do |crawler|
 				i = crawler.new
-				puts "Queuying Crawler base URL: '#{i.url}'" if opts[:verbose]
-				qu = Apollo::Model::QueuedUrl.new(:url => i.url, :state => :queued)
-				qu.save
+				Apollo::Scheduler::BaseScheduler::schedule(i.url, crawler)
 			end	
 		end
 
+		def init_crawlers(amqp, opts={})
+			crawlers = []
+			crawlers << Apollo::Agent::CrawlerAgent.new(amqp, self.options)
+		end
+
+		def init_domainers(amqp, opts={})
+			domainers = []
+			domainers << Apollo::Agent::DomainerAgent.new(amqp, self.options)
+		end
+
 		def init_fetchers(amqp, opts={})
 			fetchers = []
 			fetchers << Apollo::Agent::FetcherAgent.new(amqp, self.options)
 
+			# TODO: This should not be here!
 			enqueue_crawlers_urls(amqp, Apollo::Crawler::BaseCrawler.subclasses, opts)
-			
-			# ch = self.amqp.create_channel
-			# x  = ch.default_exchange
-			# x.publish("Hello!", :routing_key => "fetcher")
 		end
 
 		def init_agents(amqp, opts={})
 			puts "Initializing agents"
 
+			init_crawlers(amqp, opts)
+			init_domainers(amqp, opts)
 			init_fetchers(amqp, opts)	
 		end
 
+		def init_domains(opts={})
+			path = File.join(File.dirname(__FILE__), "../../../tmp/top-1m.csv")
+			puts "#{path}"
+			if(File.exists?(path) == false)
+				return 0
+			end
+
+			Thread::new {
+				CSV.foreach(path) do |row|
+					name = row[1]
+					domain = Apollo::Model::Domain.where({:name => name}).first()
+					if(domain.nil?)
+						domain = Apollo::Model::Domain.new({:name => name})
+						domain.save
+						print "."
+					end
+				end
+			}
+		end
+
 		def init_program(args)
 			res = super(args)
 			return res unless res.nil?
@@ -148,6 +178,8 @@ module Apollo
 			res = super(args)
 			return res unless res.nil?
 
+			init_domains()
+
 			# Here we start
 			# if(ARGV.length < 1)
 			# 	puts optparser
@@ -156,7 +188,7 @@ module Apollo
 
 			res_code = 0
 			if(self.options[:daemon])
-				planner = Apollo::Planner::SmartPlanner.new(self.amqp, self.mongo)
+				planner = Apollo::Planner::SmartPlanner.new(self.amqp, self.mongo, self.options)
 				res_code = planner.run(self.options)
 			end
 

data/lib/apollo_crawler/scheduler/base_scheduler.rb

@@ -0,0 +1,39 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require File.join(File.dirname(__FILE__), "../model/models")
+
+module Apollo
+	module Scheduler
+		class BaseScheduler
+			def self.schedule(url, crawler=nil, opts={})
+				queued_url = Apollo::Model::QueuedUrl.where(:url => url).first
+
+				if(queued_url.nil?)
+					qu = Apollo::Model::QueuedUrl.new(:url => url, :state => :queued, :crawler_name => crawler.to_s)
+					qu.save
+					return qu
+				end
+
+				return nil
+			end
+		end # class BaseScheduler
+	end # module Scheduler
+end # module Apollo

data/lib/apollo_crawler/scheduler/schedulers.rb

@@ -0,0 +1,21 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require File.join(File.dirname(__FILE__), 'base_scheduler')

data/lib/apollo_crawler/version.rb

@@ -19,5 +19,5 @@
 # THE SOFTWARE.
 
 module Apollo
-	VERSION = '0.1.24'
+	VERSION = '0.1.25'
 end # Apollo

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: apollo-crawler
 version: !ruby/object:Gem::Version
-  version: 0.1.24
+  version: 0.1.25
 platform: ruby
 authors:
 - Tomas Korcak
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-23 00:00:00.000000000 Z
+date: 2013-03-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: awesome_print
@@ -414,6 +414,9 @@ files:
 - ./lib/apollo_crawler/adapter/mongo_adapter.rb
 - ./lib/apollo_crawler/agent/agents.rb
 - ./lib/apollo_crawler/agent/base_agent.rb
+- ./lib/apollo_crawler/agent/crawler_agent.rb
+- ./lib/apollo_crawler/agent/domainer_agent.rb
+- ./lib/apollo_crawler/agent/exchanges.rb
 - ./lib/apollo_crawler/agent/fetcher_agent.rb
 - ./lib/apollo_crawler/cache/base_cache.rb
 - ./lib/apollo_crawler/cache/caches.rb
@@ -452,6 +455,8 @@ files:
 - ./lib/apollo_crawler/logger/loggers.rb
 - ./lib/apollo_crawler/model/base_model.rb
 - ./lib/apollo_crawler/model/crawler.rb
+- ./lib/apollo_crawler/model/data_source.rb
+- ./lib/apollo_crawler/model/domain.rb
 - ./lib/apollo_crawler/model/models.rb
 - ./lib/apollo_crawler/model/queued_url.rb
 - ./lib/apollo_crawler/model/raw_document.rb
@@ -463,6 +468,8 @@ files:
 - ./lib/apollo_crawler/program/crawler_program.rb
 - ./lib/apollo_crawler/program/platform_program.rb
 - ./lib/apollo_crawler/program/programs.rb
+- ./lib/apollo_crawler/scheduler/base_scheduler.rb
+- ./lib/apollo_crawler/scheduler/schedulers.rb
 - ./lib/apollo_crawler/store/base_store.rb
 - ./lib/apollo_crawler/store/stores.rb
 - ./lib/apollo_crawler/version.rb