apollo-crawler 0.1.28 → 0.1.30

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 22e840b1ad9ecabcb011c4b02383c060d325a61f
-  data.tar.gz: faf47f1ae1b7451262037d7c3e28a83bb05a2678
+  metadata.gz: 4f7597400d1ab12d2ab2d1d52330ca29e4f2192f
+  data.tar.gz: 297216744ee4ea8a9af6595b1361dd3bfcf8d0b1
 SHA512:
-  metadata.gz: f916c2da81657168389ddf998407d338bbdafd323dfb5b7f0e3b4dc9ef2e9d001d00974fee48d2918a12762a90e5172146820553b09d798720f33c319db052a7
-  data.tar.gz: a820bf980c655156eefcad5487a79805f121f7e59d2e95285415d0229583b3df4440a4c0e8021186fc963837239cd0b1a9c1e8ca18718dc5db67bc54ff6ce9a7
+  metadata.gz: 71bf482ad465efa67976547c5e0a99d8a728478c5c346f987e675da8574b45e4dba44375fd501e8699c083e42eaf7e19ac1db6d2d9962f9736d408c52f896844
+  data.tar.gz: 01651a1aa4d3408ce63142a46c6e3727ceee35995520cfcc2df4b6451495b09cf5cebe1cc1244dca6ebef2138bb06b6ede97ac1720b75b8de866b3f381c2e25f

data/lib/apollo_crawler/agent/fetcher_agent.rb

@@ -96,10 +96,10 @@ module Apollo
 				url = queued_url["url"]
 
 				res = Apollo::Model::RawDocument.new
-				res.headers = doc.headers
-				res.body = doc.body
-				res.sha_hash = Digest::SHA1.hexdigest(doc.body)
-				res.status = doc.status
+				res.headers = doc[:headers]
+				res.body = doc[:body]
+				res.sha_hash = Digest::SHA1.hexdigest(doc[:body])
+				res.status = doc[:status]
 				res.url = url
 
 				return res

data/lib/apollo_crawler/crawler/base_crawler.rb

@@ -170,10 +170,12 @@ module Apollo
 			end
 
 			def self.create_metadoc(url, doc)
+				body = doc[:body].encode('UTF-8', {:invalid => :replace, :undef => :replace, :replace => '?'})
+				
 				return {
 					'url' => url,
-					'doc' => doc.body.encode('UTF-8', {:invalid => :replace, :undef => :replace, :replace => '?'}),
-					'hash' => Digest::SHA256.new.update(doc.body).hexdigest,
+					'doc' => body,
+					'hash' => Digest::SHA256.new.update(body).hexdigest,
 					'created_at' => Time.now.utc,
 					'expires_at' => nil,
 					'version' => 0

data/lib/apollo_crawler/fetcher/base_fetcher.rb

@@ -19,9 +19,15 @@
 # THE SOFTWARE.
 
 require 'cgi'
-require "net/http"
-require "open-uri"
+
+require 'open-uri'
+require 'net/http'
+
 require 'faraday'
+require 'faraday_middleware'
+
+require 'mechanize'
+
 require 'ipaddr'
 
 # require 'resolv'
@@ -38,7 +44,7 @@ module Apollo
 				}
 			end
 
-			def self.fetch(url, options = {})
+			def self.fetch_old(url, options = {})
 				begin
 					uri = URI.parse(url.to_s)
 				rescue Exception => e
@@ -64,6 +70,29 @@ module Apollo
 				end
 
 				# Return result
+				return res
+			end	
+
+			def self.fetch(url, options = {})
+				begin
+					uri = URI.parse(url.to_s)
+				rescue Exception => e
+					puts "EXCEPTION: BaseFetcher::fetch() - Unable to fetch: '#{e.to_s}'"
+					return nil
+				end
+
+				agent = Mechanize.new do |agent|
+  					agent.user_agent = 'Apollo Crawler'
+  				end
+
+				page = agent.get(uri)
+
+				res = {
+					:status => page.code,
+					:headers => page.header.to_hash,
+					:body => page.content
+				}
+
 				return res
 			end			
 		end # class BaseFetcher

data/lib/apollo_crawler/planner/smart_planner.rb

@@ -90,7 +90,7 @@ module Apollo
 				declarations[:queues]["planner.crawled.queue"].bind(declarations[:exchanges]["planner.crawled"]).subscribe do |delivery_info, metadata, payload|
 					msg = JSON.parse(payload)
 
-					puts "Crawled - msg.inspect"
+					# puts "Crawled - #{msg.inspect}"
 
 					request = msg['request']
 					response = msg['response']

data/lib/apollo_crawler/scheduler/base_scheduler.rb

@@ -18,6 +18,9 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+require "net/http"
+require "uri"
+
 require File.join(File.dirname(__FILE__), "../model/models.rb")
 
 module Apollo
@@ -29,6 +32,13 @@ module Apollo
 				if queued_url.nil? == false
 					return queued_url
 				end
+
+				uri = URI.parse(url)
+				domain = Apollo::Model::Domain.where(:name => uri.hostname).first
+				if(domain.nil?)
+					domain = Apollo::Model::Domain.new(:name => uri.hostname)
+					domain.save
+				end
 					
 				res = Apollo::Model::QueuedUrl.new(:url => url, :state => :queued, :crawler_name => crawler.to_s)
 				res.save

data/lib/apollo_crawler/version.rb

@@ -19,5 +19,5 @@
 # THE SOFTWARE.
 
 module Apollo
-	VERSION = '0.1.28'
+	VERSION = '0.1.30'
 end # Apollo

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: apollo-crawler
 version: !ruby/object:Gem::Version
-  version: 0.1.28
+  version: 0.1.30
 platform: ruby
 authors:
 - Tomas Korcak