eventhub-processor 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 134f2e6d20850ddc0901916ee045823c68e10e2c
-  data.tar.gz: f65baa4cad21c4d98fdf4a75ffb5bcf085b18e1d
+  metadata.gz: dea7fe8b1b08a77c68d736c1c2e2e72325df218a
+  data.tar.gz: bc476eac97e284575a684ed0cfe61850d4ea9b8f
 SHA512:
-  metadata.gz: 34501530c76eb1150b4291e9ea4eff0f874e25965447678ead5ec629d0117cb32f2de45e45310b59238392dd7cf82dbe9619e0cd4edae884db89a39723d2d2ec
-  data.tar.gz: b7a2958916947dfa8ef261a0d862e7783ebb7832f43ca6d24f0d78d6139a523fedebd40616efc5e6dc05edcb5ea025ea63f67cccb282b0b0a1e492b0a75ca9c6
+  metadata.gz: dcce6e241aefab6b054f79d942969722fa04ea9162e86fbcefce05e396d39c69062785ae0a2eca6409dd4e0864d604bbf84092f977f80d7a7068b531cc0c044b
+  data.tar.gz: be828faa80aaa01901a29918413f265956835686b5120402726eec913eded92203fe57760f1b4f744da63ebbc9f307efe22175c4382662ab6a4322c0d9ee5d09

data/lib/eventhub-processor.rb

@@ -4,9 +4,11 @@ require 'rest-client'
 require 'json'
 require 'singleton'
 require 'uuidtools'
+require 'base64'
 
 require_relative 'eventhub/version'
 require_relative 'eventhub/constant'
+require_relative 'eventhub/helper'
 require_relative 'eventhub/multi_logger'
 
 require_relative 'eventhub/configuration'

data/lib/eventhub/configuration.rb

@@ -2,6 +2,7 @@ module EventHub
 
 	class Configuration
 		include Singleton
+    include Helper
 
 		attr_accessor :data
   	
@@ -12,7 +13,11 @@ module EventHub
   	def load_file(input, env='development')
     	tmp = JSON.parse( IO.read(input))
   		@data = tmp[env]
-  	end
+      true
+    rescue => e
+      EventHub.logger.info("Unexpected exception while loading configuration [#{input}]: #{format_raw_string(e.message)}")
+  	  false
+    end
 
 	end
 

data/lib/eventhub/helper.rb

@@ -0,0 +1,19 @@
+module EventHub
+
+	module Helper
+
+		# converts a base class name, Whatever::MyClassName => my_class_name
+		def class_to_string(class_name)
+			class_name.to_s.split("::")[-1].gsub(/[A-Z]/) { |c| "_#{c}"}.gsub(/^_/,"").downcase
+		end
+
+		def format_raw_string(message,max_characters=80) 
+		  max_characters = 5 if max_characters < 5
+		  m = message.gsub(/\r\n|\n|\r/m,";")
+		  return (m[0..max_characters-3] + "...") if m.size > max_characters
+		  return m
+	  end
+
+	end	
+
+end

data/lib/eventhub/message.rb

@@ -35,15 +35,15 @@ module EventHub
       end
     end
 
-    def self.from_json(json)
-      data = JSON.parse(json)
-      Message.new(data.get('header'), data.get('body'),json)
+    def self.from_json(raw)
+      data = JSON.parse(raw)
+      Message.new(data.get('header'), data.get('body'),raw)
     rescue => e
-      Message.new({ "status" =>  { "code" => STATUS_INVALID, "message" => "JSON parse error: #{e}" }} ,{},json)
+      Message.new({ "status" =>  { "code" => STATUS_INVALID, "message" => "JSON parse error: #{e}" }} ,{ "original_message_base64_encoded" => Base64.encode64(raw)},raw)
     end
 
     # process_step_position should be
-    def initialize(header, body,raw=nil)
+    def initialize(header=nil, body=nil,raw=nil)
 
       @header = header || {}
       @body   = body || {}

data/lib/eventhub/multi_logger.rb

@@ -15,8 +15,16 @@ end
 module EventHub
 
   class MultiLogger
-    def initialize(*targets)
-        @targets = targets
+
+    attr_accessor :folder, :devices
+
+    def initialize(folder=nil)
+        @folder = folder || Dir.pwd
+        @devices = []
+    end
+
+    def add_device(device)
+      @devices << device
     end
 
     def save_detailed_error(feedback,message=nil)
@@ -24,9 +32,9 @@ module EventHub
       stamp = "#{time.strftime("%Y%m%d_%H%M%S")}_#{"%03d" % (time.usec/1000)}"
       filename = "#{stamp}.log"
 
-    	FileUtils.makedirs("exceptions")
+    	FileUtils.makedirs("#{folder}/exceptions")
 
-      File.open("exceptions/#{filename}","w") do |output|
+      File.open("#{@folder}/exceptions/#{filename}","w") do |output|
         output.write("#{feedback}\n\n")
         output.write("Exception: #{feedback.class.to_s}\n\n")
         output.write("Call Stack:\n")
@@ -37,7 +45,7 @@ module EventHub
 
       # save message if provided
       if message
-  	    File.open("exceptions/#{stamp}.msg.raw","wb") do |output|
+  	    File.open("#{@folder}/exceptions/#{stamp}.msg.raw","wb") do |output|
   	    	output.write(message)
   	    end
     	end	
@@ -47,7 +55,7 @@ module EventHub
 
     %w(log debug info warn error).each do |m|
       define_method(m) do |*args|
-          @targets.map { |t| t.send(m, *args) }
+          @devices.map { |d| d.send(m, *args) }
       end
     end
 

data/lib/eventhub/processor.rb

@@ -1,6 +1,26 @@
 module EventHub
 	class Processor
 
+		attr_accessor :name, :folder
+
+		include Helper
+
+		def initialize(name=nil)
+			@name = name || class_to_string(self.class)
+			@folder = Dir.pwd
+
+			@started = Time.now
+
+			@messages_successful 		= 0
+			@messages_unsuccessful 	= 0
+
+			@restart = true
+		end
+
+		def configuration
+			EventHub::Configuration.instance.data
+		end
+
 		def host
 			configuration.get('server.host') || 'localhost'
 		end
@@ -17,27 +37,30 @@ module EventHub
 			configuration.get('server.management_port') || 15672
 		end
 
-		def queue_name
-			configuration.get('processor.queue') || 'inbound'
+		def listener_queue
+			configuration.get('processor.listener_queue') || 'inbound'
 		end
 
 		def vhost
 			configuration.get('server.vhost') || nil
 		end
 
-		def watchdog_cycle
-			configuration.get('processor.watchdog_cycle') || 5
+		def watchdog_cycle_in_s
+			configuration.get('processor.watchdog_cycle_is_s') || 15
 		end
 
-		def configuration
-			EventHub::Configuration.instance.data
+		def restart_in_s
+			configuration.get('processor.restart_in_s') || 15
 		end
 
+		def heartbeat_cycle_in_s
+			configuration.get('processor.heartbeat_cycle_in_s') || 300
+		end
 
+		def start(detached=false)
+			daemonize if detached
 
-		def start
-			restart = true
-			while restart
+			while @restart
 
 				begin 
 					AMQP.start(configuration.get('server')) do |connection, open_ok|
@@ -46,9 +69,8 @@ module EventHub
 						
 						# deal with tcp connection issues
 						@connection.on_tcp_connection_loss do |conn, settings|
-				  		EventHub.logger.warn("Processor lost tcp connection. Trying to restart in 5 seconds...")
-				  		sleep 5
-				    	EventMachine.stop
+				  		EventHub.logger.warn("Processor lost tcp connection. Trying to restart in #{@restart_in_s} seconds...")
+				  		stop_processor(true)
 				  	end
 
 						# create channel
@@ -56,75 +78,108 @@ module EventHub
 				  	@channel.auto_recovery = true
 				  
 				  	# connect to queue
-				  	@queue = @channel.queue(queue_name, durable: true, auto_delete: false)
+				  	@queue = @channel.queue(self.listener_queue, durable: true, auto_delete: false)
 
 				  	# subscribe to queue
 					  @queue.subscribe(:ack => true) do |metadata, payload|
-					  	handle_heartbeat(payload)
-					  	if handle_message(metadata,payload)
-					  		metadata.ack
+					  	begin
+						  	if handle_message(metadata,payload)
+						  		raise
+						  		metadata.ack
+						  	else
+						  		metadata.nack
+						  	end
+					  	rescue => e
+					  		EventHub.logger.error("Unexpected exception in handle_message method: #{e}")
+								EventHub.logger.save_detailed_error(e)
 					  	end
 					  end
 
-						# Features to stop main event loop
-						stop_main_loop = Proc.new {
-					    @connection.disconnect { 
-					    	EventHub.logger.info("Processor is stopping main event loop")
-					    	EventMachine.stop
-					    	restart = false
-					    }
-				  	}
+					  EventHub.logger.info("Processor [#{@name}] is listening to queue [#{self.listener_queue}], base folder [#{@folder}]")
 
-					  Signal.trap "TERM", stop_main_loop
-					  Signal.trap "INT",  stop_main_loop
+					  # Singnal Listening
+					  Signal.trap("TERM") {stop_processor}
+					  Signal.trap("INT")  {stop_processor}
 
-					  EventMachine.add_timer(self.watchdog_cycle) { watchdog }
+					  # Various timers
+					  EventMachine.add_timer(@watchdog_cycle_in_s) { watchdog }
 
-					  EventHub.logger.info("Processor is listening to queue [#{[configuration.get('server.vhost'),configuration.get('processor.queue')].compact.join(".")}]")
+					  heartbeat
 					end
 				rescue => e
-					EventHub.logger.error("Unexpected exception: #{e}")
-					EventHub.logger.save_detailed_error(e)
+					Signal.trap("TERM") { stop_processor }
+					Signal.trap("INT")  { stop_processor }
+
+					id = EventHub.logger.save_detailed_error(e)
+					EventHub.logger.error("Unexpected exception: #{e}, see => #{id}")
+					
+					sleep_break self.restart_in_s
 				end
 
 			end # while
 
-			EventHub.logger.info("Processor has been stopped")
+			EventHub.logger.info("Processor [#{@name}] has been stopped")
+		ensure
+			# remove pid file
+			begin
+				File.delete("#{@folder}/#{@name}.pid")	
+			rescue
+				# ignore exceptions here
+			end
 		end
 
 		def handle_message(metadata,payload)
 			raise "Please implement method in derived class"
 		end
 
-		def handle_heartbeat(message)
-			# sends a standard message back to dispatcher
-		end
-
 		def watchdog
 			begin
-				response = RestClient.get "http://#{self.user}:#{self.password}@#{host}:#{management_port}/api/queues/#{self.vhost}/#{self.queue_name}/bindings", { :content_type => :json}
+				response = RestClient.get "http://#{self.user}:#{self.password}@#{self.host}:#{self.management_port}/api/queues/#{self.vhost}/#{self.listener_queue}/bindings", { :content_type => :json}
   			data = JSON.parse(response.body)
   	
   			if response.code != 200
-  				EventHub.logger.warn("Watchdog: Server did not answered properly. Trying to restart in 5 seconds...")
-  				sleep 5
-  				EventMachine.stop
+  				EventHub.logger.warn("Watchdog: Server did not answered properly. Trying to restart in #{self.restart_in_s} seconds...")
+  				stop_processor
   			elsif data.size == 0
-  				EventHub.logger.warn("Watchdog: Something is wrong with the vhost, queue, and/or bindings. Trying to restart in 5 seconds...")
-  				sleep 5
-  				EventMachine.stop
+  				EventHub.logger.warn("Watchdog: Something is wrong with the vhost, queue, and/or bindings. Trying to restart in #{self.restart_in_s} seconds...")
+  				stop_processor
   			else
   				# Watchdog is happy :-)
-  			end
-
+					# add timer for next check
+					EventMachine.add_timer(self.watchdog_cycle_in_s) { watchdog }
+				end	
+					
 			rescue => e
-				EventHub.logger.error("Watchdog: Unexpected exception: #{e}. Trying to restart in 5 seconds...")
-				sleep 5
-				EventMachine.stop
+				EventHub.logger.error("Watchdog: Unexpected exception: #{e}. Trying to restart in #{self.restart_in_s} seconds...")
+				stop_processor
 			end
+		end
 
-			# place next time
-			EventMachine.add_timer(watchdog_cycle) { watchdog }
+		def heartbeat
+			message = Message.new
+			message.origin_module_id 	= @name
+			message.origin_type 			= "processor"
+			message.origin_site_id 		= 'chbs'
+
+			message.process_name    	= 'event_hub.heartbeat'
+
+			message.body = {
+				 heartbeat: {
+				 started: 							@started,
+				 stamp_last_beat:       Time.now, 
+				 heartbeat_cycle_in_s: 	self.heartbeat_cycle_in_s,
+				 served_queues: 				[self.listener_queue],
+				 messages: {
+				 	total: 								@messages_successful+@messages_unsuccessful,
+				 	successful: 					@messages_successful,
+				 	unsuccessful: 				@messages_unsuccessful
+				 	}
+				}
+			}
+
+			send_to_dispatcher(message.to_json)
+
+			EventMachine.add_timer(self.heartbeat_cycle_in_s) { heartbeat }
 		end
 
 		def send_to_dispatcher(payload)
@@ -136,12 +191,15 @@ module EventHub
 			channel = connection.create_channel
       channel.confirm_select
 
-    	channel.default_exchange.publish(payload,routing_key: 'inbound', persistent: true)
+    	channel.default_exchange.publish(payload,routing_key: EVENT_HUB_QUEUE_INBOUND, persistent: true)
     	success = channel.wait_for_confirms
 
     	if !success
       	EventHub.logger.error("Message has not been confirmed by the server to be received !!!")
     		confirmed = false
+    		@messages_unsuccessful += 1
+    	else
+    		@messages_successful += 1	
     	end
 
       channel.close   
@@ -150,12 +208,35 @@ module EventHub
       confirmed
 		end
 
-		def format_raw_string(message,max_characters=80) 
-	    max_characters = 5 if max_characters < 5
-	    m = message.gsub(/\r\n/m,";")
-	    return (m[0..max_characters-3] + "...") if m.size > max_characters
-	    return m
-  	end
+  	def sleep_break( seconds ) # breaks after n seconds or after interrupt
+  		while (seconds > 0)
+    		sleep(1)
+    		seconds -= 1
+    		break unless @restart
+  		end
+		end
+
+		private
+
+		def stop_processor(restart=false)
+			@restart = restart
+
+			# stop event loop
+			@connection.disconnect { 
+				EventHub.logger.info("Processor [#{@name}] is stopping main event loop")
+				EventMachine.stop
+			}
+		end
+
+		def daemonize
+			EventHub.logger.info("Processor [#{@name}] is going to start as daemon")
+		
+			# daemonize
+			Process.daemon
+
+		  # write daemon pid
+  		IO.write("#{@folder}/#{@name}.pid",Process.pid.to_s)
+		end
 
 	end
 end

data/lib/eventhub/version.rb

@@ -1,3 +1,3 @@
 module EventHub
-  VERSION = "0.0.5"
+  VERSION = "0.0.6"
 end

metadata

@@ -1,97 +1,97 @@
 --- !ruby/object:Gem::Specification
 name: eventhub-processor
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - Thomas Steiner
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-01 00:00:00.000000000 Z
+date: 2014-04-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rest-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: amqp
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: bunny
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: uuidtools
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Gem to build Event Hub processors
@@ -101,16 +101,17 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- LICENSE.txt
+- Rakefile
+- lib/eventhub-processor.rb
 - lib/eventhub/configuration.rb
 - lib/eventhub/constant.rb
 - lib/eventhub/hash.rb
+- lib/eventhub/helper.rb
 - lib/eventhub/message.rb
 - lib/eventhub/multi_logger.rb
 - lib/eventhub/processor.rb
 - lib/eventhub/version.rb
-- lib/eventhub-processor.rb
-- LICENSE.txt
-- Rakefile
 homepage: http://github.com/thomis/eventhub-processor
 licenses:
 - MIT
@@ -121,17 +122,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Gem to build Event Hub processors