rservicebus 0.0.9 → 0.0.10

data/README.md

@@ -1,10 +1,6 @@
 #RServiceBus
 
-A Ruby implementation of NServiceBus
-
-Where I started this project to increase my knowledge of NServiceBus, it is now
-proving to be a workable framework.
-
+A Ruby interpretation of NServiceBus
 
 ##Principles
 	*Why are you doing it and what can go wrong

data/bin/ReturnErroredMessagesToSourceQueue

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+
+require "amqp"
+require "yaml"
+
+require "rservicebus"
+
+host = 'localhost:11300'
+errorQueueName = "error"
+beanstalk = Beanstalk::Pool.new([host])
+
+tubes = beanstalk.list_tubes[host]
+if !tubes.include?(errorQueueName) then
+	abort( "Nothing waiting on the error queue" )
+end
+
+tubeStats = beanstalk.stats_tube(errorQueueName)
+number_of_messages = tubeStats["current-jobs-ready"]
+puts
+puts "Attempting to return #{number_of_messages} to their source queue"
+puts
+
+begin
+	beanstalk.watch(errorQueueName)
+	1.upto(number_of_messages) do |request_nbr|
+		job = beanstalk.reserve 1
+		payload = job.body
+
+		puts "#" + request_nbr.to_s + ": " + payload
+		msg = YAML::load(payload)
+		queueName = msg.getLastErrorMsg.sourceQueue
+
+		beanstalk.use( queueName )
+		beanstalk.put( payload )
+
+		job.delete
+	end
+rescue Exception => e
+	if e.message == "TIMED_OUT" then
+	else
+		raise
+	end
+end

data/lib/rservicebus/AppResource/Redis.rb

@@ -1,3 +1,5 @@
+module RServiceBus
+
 require "redis"
 
 #Implementation of an AppResource - Redis
@@ -15,3 +17,5 @@ class AppResource_Redis<AppResource
 	end
 
 end
+
+end

data/lib/rservicebus/AppResource.rb

@@ -1,3 +1,5 @@
+module RServiceBus
+
 require "uri"
 
 # Wrapper base class for resources used by applications, allowing rservicebus to configure the resource
@@ -20,3 +22,5 @@ class AppResource
 		raise "Method, getResource, needs to be implemented for resource"
 	end
 end
+
+end

data/lib/rservicebus/Config.rb

@@ -2,7 +2,7 @@ module RServiceBus
 
 #Marshals configuration information for an rservicebus host
 class Config
-	attr_reader :appName, :messageEndpointMappings, :handlerPathList, :localQueueName, :errorQueueName, :maxRetries, :forwardReceivedMessagesTo, :verbose, :beanstalkHost
+	attr_reader :appName, :messageEndpointMappings, :handlerPathList, :localQueueName, :errorQueueName, :maxRetries, :forwardReceivedMessagesTo, :verbose, :beanstalkHost, :queueTimeout
 
 	@appName
 	@messageEndpointMappings
@@ -17,6 +17,8 @@ class Config
 	
 	@beanstalkHost
 
+	@queueTimeout
+
 	def initialize()
 		puts "Cannot instantiate config directly."
 		puts "For production, use ConfigFromEnv."
@@ -85,6 +87,7 @@ class Config
 		@errorQueueName = self.getValue( "ERROR_QUEUE_NAME", "error" )
 		@maxRetries = self.getValue( "MAX_RETRIES", "5" ).to_i
 		@forwardReceivedMessagesTo = self.getValue( "FORWARD_RECEIVED_MESSAGES_TO" )
+		@queueTimeout = self.getValue( "QUEUE_TIMEOUT", "5" ).to_i
 
 		return self
 	end

data/lib/rservicebus/ConfigureAppResource.rb

@@ -1,3 +1,5 @@
+module RServiceBus
+
 require "uri"
 
 #Configure AppResources for an rservicebus host
@@ -24,3 +26,5 @@ class ConfigureAppResource
 	end
 
 end
+
+end

data/lib/rservicebus/ErrorMessage.rb

@@ -2,9 +2,11 @@ module RServiceBus
 
 class ErrorMessage
 
-	attr_reader :sourceQueue, :errorMsg
+	attr_reader :occurredAt, :sourceQueue, :errorMsg
 
 	def initialize( sourceQueue, errorMsg )
+		@occurredAt = DateTime.now
+
 		@sourceQueue=sourceQueue
 		@errorMsg=errorMsg
 	end

data/lib/rservicebus/Host.rb

@@ -3,7 +3,6 @@ module RServiceBus
 #Host process for rservicebus
 class Host
 
-
 	@handlerList
 
 	@forwardReceivedMessagesToQueue
@@ -15,6 +14,10 @@ class Host
 	@appResources	
 	
 	@config
+	
+	@subscriptionManager
+	
+	@stats
 
 	def log(string, ver=false)
 		type = ver ? "VERB" : "INFO"
@@ -77,46 +80,20 @@ class Host
 		return self
 	end
 
-#Load an existing subscription - startup function
-	def loadSubscriptions
-		log "Load subscriptions"
-		@subscriptions = Hash.new
+	def configureSubscriptions
+		subscriptionStorage = SubscriptionStorage_Redis.new( @config.appName, "uri" )
+		@subscriptionManager = SubscriptionManager.new( subscriptionStorage )
 		
-		begin
-			redis = Redis.new
-
-			prefix = @config.appName + ".Subscriptions."
-			subscriptions = redis.keys prefix + "*Event"
-		rescue Exception => e
-			puts "Error connecting to redis"
-#			puts "Host string, #{@config.beanstalkHost}"
-			if e.message == "Redis::CannotConnectError" ||
-					e.message == "Redis::ECONNREFUSED" then
-				puts "***Most likely, redis is not running. Start redis, and try running this again."
-#				puts "***If you still get this error, check redis is running at, " + beanstalkHost
-			else
-				puts e.message
-				puts e.backtrace
-			end
-			abort()
-		end
-
-		subscriptions.each do |subscriptionName|
-			log "Loading subscription: " + subscriptionName, true
-			eventName = subscriptionName.sub( prefix, "" )
-			@subscriptions[eventName] = Array.new			
+		return self
+	end
 
-			log "Loading for event: " + eventName, true
-			subscription = redis.smembers subscriptionName
-			subscription.each do |subscriber|
-				log "Loading subscriber, " + subscriber + " for event, " + eventName, true
-				@subscriptions[eventName] << subscriber
-			end
-		end
+	def configureStatistics
+		@stats = Stats.new
 		
 		return self
 	end
 
+
 	def initialize()
 
 		@config = ConfigFromEnv.new
@@ -127,28 +104,16 @@ class Host
 			.loadMessageEndpointMappings()
 			.loadHandlerPathList();
 
-		self.configureAppResource()
+		self.configureStatistics()
+			.configureAppResource()
 			.connectToBeanstalk()
 			.loadHandlers()
-			.loadSubscriptions()
+			.configureSubscriptions()
 			.sendSubscriptions()
 
 		return self
 	end
 
-#Process a subscription request from a subscriber
-	def addSubscrption( eventName, queueName )
-		log "Adding subscrption for, " + eventName + ", to, " + queueName
-		redis = Redis.new
-		key = @config.appName + ".Subscriptions." + eventName
-		redis.sadd key, queueName
-
-		if @subscriptions[eventName].nil? then
-			@subscriptions[eventName] = Array.new
-		end
-		@subscriptions[eventName] << queueName
-	end
-
 	def run
 		log "Starting the Host"
 
@@ -161,59 +126,75 @@ class Host
 		self.StartListeningToEndpoints
 	end
 
-
+#Receive a msg, prep it, and handle any errors that may occur
+# - Most of this should be queue independant
 	def StartListeningToEndpoints
 		log "Waiting for messages. To exit press CTRL+C"
 
 		loop do
 			retries = @config.maxRetries
+			#Popping a msg off the queue should not be in the message handler, as it affects retry
 			begin
-				job = @beanstalk.reserve
-				body = job.body
-
-				@msg = YAML::load(body)
-				if @msg.msg.class.name == "RServiceBus::Subscription" then
-					self.addSubscrption( @msg.msg.eventName, @msg.returnAddress )
-				else
-					self.HandleMessage()
-					if !@config.forwardReceivedMessagesTo.nil? then
-						self._SendAlreadyWrappedAndSerialised(body,@config.forwardReceivedMessagesTo)
+    			log @stats.getForReporting
+				job = @beanstalk.reserve @config.queueTimeout
+				begin
+					body = job.body
+					@stats.incTotalProcessed
+					@msg = YAML::load(body)
+					if @msg.msg.class.name == "RServiceBus::Message_Subscription" then
+						@subscriptionManager.add( @msg.msg.eventName, @msg.returnAddress )
+
+					else
+						self.HandleMessage()
+						if !@config.forwardReceivedMessagesTo.nil? then
+							self._SendAlreadyWrappedAndSerialised(body,@config.forwardReceivedMessagesTo)
+						end
+					end
+					job.delete
+		    	rescue Exception => e
+					sleep 0.5
+			    	retry if (retries -= 1) > 0		    	
+
+					@stats.incTotalErrored
+					if e.class.name == "Beanstalk::NotConnected" then
+						puts "Lost connection to beanstalkd."
+						puts "*** Start or Restart beanstalkd and try again."
+						abort();
 					end
-				end
-				job.delete
-	    	rescue Exception => e
-			sleep 0.5
-		    	retry if (retries -= 1) > 0		    	
-
-				if e.class.name == "Beanstalk::NotConnected" then
-					puts "Lost connection to beanstalkd."
-					puts "*** Start or Restart beanstalkd and try again."
-					abort();
-				end
 				
-				if e.class.name == "Redis::CannotConnectError" then
-					puts "Lost connection to redis."
-					puts "*** Start or Restart redis and try again."
-					abort();
-				end
+					if e.class.name == "Redis::CannotConnectError" then
+						puts "Lost connection to redis."
+						puts "*** Start or Restart redis and try again."
+						abort();
+					end
 
-				errorString = e.message + ". " + e.backtrace[0]
+					errorString = e.message + ". " + e.backtrace[0]
 					if e.backtrace.length > 1 then
 						errorString = errorString + ". " + e.backtrace[1]
 					end
 					if e.backtrace.length > 2 then
 						errorString = errorString + ". " + e.backtrace[2]
 					end
-				log errorString
+					log errorString
 
-				@msg.addErrorMsg( @config.localQueueName, errorString )
-				serialized_object = YAML::dump(@msg)
-				self._SendAlreadyWrappedAndSerialised(serialized_object, @config.errorQueueName)
-				job.delete
-    		end
+					@msg.addErrorMsg( @config.localQueueName, errorString )
+					serialized_object = YAML::dump(@msg)
+					self._SendAlreadyWrappedAndSerialised(serialized_object, @config.errorQueueName)
+					job.delete
+	    		end
+	    	rescue Exception => e
+	    		if e.message == "TIMED_OUT" then
+	    		else
+					puts "*** This is really unexpected."
+			    	puts e.message
+			    	puts e.backtrace
+			    	abort()
+		    	end
+		    end
 		end
 	end
 
+#Send the current msg to the appropriate handlers
 	def HandleMessage()
 		msgName = @msg.msg.class.name
 		handlerList = @handlerList[msgName]
@@ -228,6 +209,7 @@ class Host
 			   			handler.Handle( @msg.msg )
 	   				rescue Exception => e
 						log "An error occured in Handler: " + handler.class.name
+						log e.message + ". " + e.backtrace[0]
 						raise e
 			   		end
 		   		end
@@ -265,6 +247,7 @@ class Host
 # @param [RServiceBus::Message] msg msg to be sent
 	def Reply( msg )
 		log "Reply with: " + msg.class.name + " To: " + @msg.returnAddress, true
+		@stats.incTotalReply
 
 		self._SendNeedsWrapping( msg, @msg.returnAddress )
 	end
@@ -276,6 +259,7 @@ class Host
 # @param [RServiceBus::Message] msg msg to be sent
 	def Send( msg )
 		log "Bus.Send", true
+		@stats.incTotalSent
 
 		msgName = msg.class.name
 		if !@config.messageEndpointMappings.has_key?( msgName ) then
@@ -294,14 +278,10 @@ class Host
 # @param [RServiceBus::Message] msg msg to be sent
 	def Publish( msg )
 		log "Bus.Publish", true
+		@stats.incTotalPublished
 
-		subscription = @subscriptions[msg.class.name]
-		if subscription.nil? then
-			log "No subscribers for event, " + msg.class.name
-			return
-		end
-
-		subscription.each do |subscriber|
+		subscriptions = @subscriptionManager.get(msg.class.name)
+		subscriptions.each do |subscriber|
 			self._SendNeedsWrapping( msg, subscriber )
 		end
 
@@ -315,7 +295,7 @@ class Host
 
 
 		queueName = @config.messageEndpointMappings[eventName]
-		subscription = Subscription.new( eventName )
+		subscription = Message_Subscription.new( eventName )
 
 
 		self._SendNeedsWrapping( subscription, queueName )

data/lib/rservicebus/Message/Subscription.rb

@@ -0,0 +1,13 @@
+module RServiceBus
+
+
+class Message_Subscription
+	attr_reader :eventName
+
+	def initialize( eventName )
+		@eventName=eventName
+	end
+	
+end
+
+end

data/lib/rservicebus/Message.rb

@@ -13,6 +13,8 @@ class Message
 		@_msg=YAML::dump(msg)
 		@returnAddress=returnAddress
 		
+		@createdAt = DateTime.now
+		
 		@msgId=UUIDTools::UUID.random_create
 		@errorList = Array.new
 	end

data/lib/rservicebus/Stats.rb

@@ -0,0 +1,69 @@
+module RServiceBus
+
+#Used to collect various run time stats for runtime reporting
+class Stats
+
+	def initialize
+		@hash = Hash.new
+
+		@totalProcessed = 0
+		@totalErrored = 0
+		@totalSent = 0
+		@totalPublished = 0
+		@totalReply = 0
+
+		@totalByMessageType = Hash.new
+	end
+
+	def incTotalProcessed
+		@totalProcessed = @totalProcessed + 1
+	end
+
+	def incTotalErrored
+		@totalErrored = @totalErrored + 1
+	end
+
+	def incTotalSent
+		@totalSent = @totalSent + 1
+	end
+
+	def incTotalPublished
+		@totalPublished = @totalPublished + 1
+	end
+
+	def incTotalReply
+		@totalReply = @totalReply + 1
+	end
+
+	def inc( key )
+		if @hash[key].nil? then
+			@hash[key] = 0
+		end
+		@hash[key] = @hash[key] + 1
+	end
+
+	def incMessageType( className )
+		if @totalByMessageType[className].nil? then
+			@totalByMessageType[className] = 1
+		else
+			@totalByMessageType[className] = @totalByMessageType[className] + 1
+		end
+
+	end
+
+	def getForReporting
+		string = "T:#{@totalProcessed};E:#{@totalErrored};S:#{@totalSent};P:#{@totalPublished};R:#{@totalReply}"
+
+		if @hash.length > 0 then
+			@hash.each do |k,v|
+				string = "#{string};#{k}:#{v}"
+			end
+		end
+		
+		return string
+	end
+
+end
+
+
+end

data/lib/rservicebus/SubscriptionManager.rb

@@ -0,0 +1,34 @@
+module RServiceBus
+
+class SubscriptionManager
+
+	@subscriptionStorage
+	@subscriptions
+
+	def initialize( subscriptionStorage )
+		@subscriptionStorage = subscriptionStorage
+		@subscriptions = @subscriptionStorage.getAll
+	end
+
+	#Get subscriptions for given eventName
+	def get( eventName )
+		subscriptions = @subscriptions[eventName]
+		if subscriptions.nil? then
+			RServiceBus.log "No subscribers for event, " + eventName
+			return Array.new
+		end
+
+		return subscriptions
+	end
+
+	def add( eventName, queueName )
+		RServiceBus.log "Adding subscrption for, " + eventName + ", to, " + queueName
+		@subscriptions = @subscriptionStorage.add( eventName, queueName )
+	end
+
+	def remove( eventName, queueName )
+		raise "Method, remove, needs to be implemented for this subscription storage"
+	end
+end
+
+end

data/lib/rservicebus/SubscriptionStorage/Redis.rb

@@ -0,0 +1,65 @@
+module RServiceBus
+
+require "redis"
+
+#Implementation of Subscription Storage to Redis
+class SubscriptionStorage_Redis<SubscriptionStorage
+
+	@redis
+
+	def initialize( appName, uri )
+		super(appName, uri)
+		@redis = Redis.new
+	end
+
+	def getAll
+		RServiceBus.log "Load subscriptions"
+		begin
+			content = @redis.get( @appName + ".Subscriptions" )
+			if content.nil? then
+				subscriptions = Hash.new
+			else
+				subscriptions = YAML::load(content)
+			end
+			return subscriptions
+		rescue Exception => e
+			puts "Error connecting to redis"
+			if e.message == "Redis::CannotConnectError" ||
+					e.message == "Redis::ECONNREFUSED" then
+				puts "***Most likely, redis is not running. Start redis, and try running this again."
+			else
+				puts e.message
+				puts e.backtrace
+			end
+			abort()
+		end
+	end
+
+	def add( eventName, queueName )
+#		RServiceBus.log "Storing subscrption for, " + eventName + ", to, " + queueName
+		content = @redis.get( @appName + ".Subscriptions" )
+		if content.nil? then
+			subscriptions = Hash.new
+		else
+			subscriptions = YAML::load(content)
+		end
+		
+		if subscriptions[eventName].nil? then
+			subscriptions[eventName] = Array.new
+		end
+		
+		subscriptions[eventName] << queueName
+		subscriptions[eventName] = subscriptions[eventName].uniq
+
+		@redis.set( @appName + ".Subscriptions", YAML::dump(subscriptions ) )
+
+		return subscriptions
+	end
+
+	def remove( eventName, queueName )
+		raise "Method, remove, needs to be implemented for this subscription storage"
+	end
+
+end
+
+end

data/lib/rservicebus/SubscriptionStorage.rb

@@ -0,0 +1,30 @@
+module RServiceBus
+
+require "uri"
+
+# Base class for subscription storage
+#
+class SubscriptionStorage
+	@appName
+
+# Specified using URI.
+#
+# @param [String] uri a location for the resource to which we will attach, eg redis://127.0.0.1/foo
+	def initialize(appName, uri)
+		@appName = appName
+	end
+
+	def getAll
+		raise "Method, getResource, needs to be implemented for resource"
+	end
+
+	def add( eventName, queueName )
+		raise "Method, add, needs to be implemented for this subscription storage"
+	end
+
+	def remove( eventName, queueName )
+		raise "Method, remove, needs to be implemented for this subscription storage"
+	end
+end
+
+end

data/lib/rservicebus/helper_functions.rb

@@ -13,4 +13,12 @@ module RServiceBus
 		return hash.to_json
 	end
 
+	def RServiceBus.log(string, ver=false)
+		type = ver ? "VERB" : "INFO"
+#		if @config.verbose || !ver then
+			timestamp = Time.new.strftime( "%Y-%m-%d %H:%M:%S" )
+			puts "[#{type}] #{timestamp} :: #{string}"
+#		end
+	end
+
 end

data/lib/rservicebus.rb

@@ -8,15 +8,22 @@ require "json"
 require "rservicebus/helper_functions"
 require "rservicebus/Agent"
 require "rservicebus/ErrorMessage"
-require "rservicebus/Message"
 require "rservicebus/HandlerLoader"
 require "rservicebus/ConfigureAppResource"
 require "rservicebus/Host"
 require "rservicebus/Config"
+require "rservicebus/Stats"
+
+require "rservicebus/Message"
+require "rservicebus/Message/Subscription"
 
 require "rservicebus/AppResource"
 require "rservicebus/AppResource/Redis"
 
+require "rservicebus/SubscriptionManager"
+require "rservicebus/SubscriptionStorage"
+require "rservicebus/SubscriptionStorage/Redis"
+
 
 module RServiceBus
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rservicebus
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.0.10
   prerelease: 
 platform: ruby
 authors:
@@ -11,10 +11,11 @@ bindir: bin
 cert_chain: []
 date: 2012-06-25 00:00:00.000000000 Z
 dependencies: []
-description: A ruby implementation of NServiceBus
+description: A Ruby interpretation of NServiceBus
 email: guy@guyirvine.com
 executables:
 - rservicebus
+- ReturnErroredMessagesToSourceQueue
 extensions: []
 extra_rdoc_files: []
 files:
@@ -27,11 +28,17 @@ files:
 - lib/rservicebus/HandlerLoader.rb
 - lib/rservicebus/helper_functions.rb
 - lib/rservicebus/Host.rb
+- lib/rservicebus/Message/Subscription.rb
 - lib/rservicebus/Message.rb
+- lib/rservicebus/Stats.rb
+- lib/rservicebus/SubscriptionManager.rb
+- lib/rservicebus/SubscriptionStorage/Redis.rb
+- lib/rservicebus/SubscriptionStorage.rb
 - lib/rservicebus/Test/Bus.rb
 - lib/rservicebus/Test/Redis.rb
 - lib/rservicebus/Test.rb
 - lib/rservicebus.rb
+- bin/ReturnErroredMessagesToSourceQueue
 - bin/rservicebus
 - LICENSE
 - README.md