apollo-crawler 0.1.25 → 0.1.26

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 79e9ecdfed577a1ce13b74b24d6d5bc26bf75843
-  data.tar.gz: 6d93c6da6316d4666ddc5e434bab1caadc213ba3
+  metadata.gz: 5e6fa7213e0e7f81364c5dbda6a8f53def1fda6e
+  data.tar.gz: ecd79cd04f4a4331124b3910ef5e85da5f590692
 SHA512:
-  metadata.gz: 863d10a255722bd53c9ee998e2886fd86d04cf7808284323d33f7da1fe77fc99ac0874f1e2a61f20c8656e039bc3bac2d2a9cef911e9b6e6d12266e91636b3bc
-  data.tar.gz: db77a2d4606dcecbec1ae2e0d872cc41f6fea64012280e179b1147d8bbabfe7eef5446bf364978ee930f338cb367624e30a86e11fe0613e4499e03dcbc670e4b
+  metadata.gz: ae28d3adaca4125abdeee5ffbf73615a1efc6388e97882b67359846e81b6b58acca74fc08f59034c13f132892cca92b324876c20ab6680222aefc08b5e0d9ea1
+  data.tar.gz: f452e71e138696125affbaa3ae646775857bf7b99a06a0656a85e08881e66fdbe357cd31eac4bc5f3030aadbf0075b4f602f280c669d65b139b14bc20ab974ec

data/lib/apollo_crawler/planner/smart_planner.rb

@@ -52,7 +52,7 @@ module Apollo
 
 					doc = Apollo::Model::QueuedUrl.find(request["_id"])
 					doc.update_attributes(msg['request'])
-					doc.state = "fetched"
+					doc.state = :fetched
 					doc.save
 
 					doc = Apollo::Model::RawDocument.where(:url => request['url']).first
@@ -112,11 +112,20 @@ module Apollo
 				declarations[:exchanges]["fetcher"].publish(url.to_json, :reply_to => "planner.fetched")
 			end
 
+			def get_next_url(opts={})
+				Apollo::Model::QueuedUrl.where({:state => :queued}).find_and_modify({ "$set" => { state: :fetching }}, new: true)
+			end
+
 			def fetch_queued_urls(opts={})
-				while url = Apollo::Model::QueuedUrl.where({:state => :queued}).find_and_modify({ "$set" => { state: :fetching }}, new: true)
+				url = get_next_url(opts)
+				
+				while url
+					puts url.inspect 
 					# puts "Count of URLs in Queue: #{url.count}" if opts[:verbose]
 
 					fetch_url(url, opts)
+
+					url = get_next_url()
 				end
 			end
 

data/lib/apollo_crawler/program/platform_program.rb

@@ -173,6 +173,16 @@ module Apollo
 			return nil
 		end
 
+		def requeue_fetching_urls(opts={})
+			urls = Apollo::Model::QueuedUrl.where(:state => :fetching)
+			urls.each do |url|
+				puts "Requeing '#{url.inspect}'" if opts[:verbose]
+
+				url.state = :queued
+				url.save
+			end
+		end
+
 		# Run Program
 		def run(args = ARGV)
 			res = super(args)
@@ -180,6 +190,8 @@ module Apollo
 
 			init_domains()
 
+			requeue_fetching_urls(self.options)
+
 			# Here we start
 			# if(ARGV.length < 1)
 			# 	puts optparser

data/lib/apollo_crawler/version.rb

@@ -19,5 +19,5 @@
 # THE SOFTWARE.
 
 module Apollo
-	VERSION = '0.1.25'
+	VERSION = '0.1.26'
 end # Apollo

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: apollo-crawler
 version: !ruby/object:Gem::Version
-  version: 0.1.25
+  version: 0.1.26
 platform: ruby
 authors:
 - Tomas Korcak