eventhub-processor 0.0.6 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dea7fe8b1b08a77c68d736c1c2e2e72325df218a
-  data.tar.gz: bc476eac97e284575a684ed0cfe61850d4ea9b8f
+  metadata.gz: d6896414dd331b83418d8bbf1e7ed6f6efc78f35
+  data.tar.gz: c7b8614fc58ba0cdef80a3532d94a645564d08ee
 SHA512:
-  metadata.gz: dcce6e241aefab6b054f79d942969722fa04ea9162e86fbcefce05e396d39c69062785ae0a2eca6409dd4e0864d604bbf84092f977f80d7a7068b531cc0c044b
-  data.tar.gz: be828faa80aaa01901a29918413f265956835686b5120402726eec913eded92203fe57760f1b4f744da63ebbc9f307efe22175c4382662ab6a4322c0d9ee5d09
+  metadata.gz: 46bc0b39fb711bab9835d507bd6bb1410a154d262c0a25fc9239eda0f7bdf79e70480adba332b5e6f4201b9e31ac293ac7861525ea4689807dc8b991203d3916
+  data.tar.gz: b67fe54f620f45b063721dd6510258b520ea5ec37c7edc83f8e4bbf2ed343588c864726a0f3bb2f6052b3b3a936840b4fb377856a1ae7fa31d4e3bd9ec480f26

data/lib/eventhub/configuration.rb

@@ -4,18 +4,21 @@ module EventHub
 		include Singleton
     include Helper
 
-		attr_accessor :data
+		attr_accessor :data, :folder, :environment
   	
   	def initialize
     	@data = nil
+      @folder = Dir.pwd
+      @environment = 'development'
   	end
 
   	def load_file(input, env='development')
     	tmp = JSON.parse( IO.read(input))
   		@data = tmp[env]
+      @environment = env
       true
     rescue => e
-      EventHub.logger.info("Unexpected exception while loading configuration [#{input}]: #{format_raw_string(e.message)}")
+      EventHub.logger.info("Unexpected exception while loading configuration [#{input}]: #{format_string(e.message)}")
   	  false
     end
 

data/lib/eventhub/constant.rb

@@ -1,11 +1,11 @@
 module EventHub
 
-	EVENT_HUB_QUEUE_INBOUND = 'inbound'
+	EH_X_INBOUND 						= 'event_hub.inbound'
 						
 	STATUS_INITIAL 					= 0         # initial status code
 	STATUS_SUCCESS 					= 200       # compare with HTTP Status Code: Success
 	STATUS_RETRIED 					= 300       # compare with HTTP Status Code: Multiple Choices
 	STATUS_INVALID					= 400       # compare with HTTP Status Code: Bad request
 	STATUS_UNDELIVERABLE 		= 500  			# comapre with HTTP Status Code: Server Error
-
+	STATUS_ERROR						= 600				# Business Process Error
 end

data/lib/eventhub/hash_extensions.rb

@@ -0,0 +1,48 @@
+module HashExtensions
+
+	module ClassMethods	
+	end
+
+	module InstanceMethods
+		# get value from provided key path, e.g. hash.get(%w(event_hub plate.queue1 retry_s))
+	  # "a" => { "b" => { "c" => { "value"}}}
+	  def get(arg)
+	    path = arg.is_a?(String) ? arg.split('.') : arg
+	    path.inject(self,:[])
+	  rescue NoMethodError 
+	    return nil
+	  end
+
+	  # set value from provided key path, e.h. hash.set('a.b.c','new value')
+	  # if overwrite is false, value will be set if it was nil previously
+	  def set(arg,value,overwrite=true)
+	    *key_path, last = arg.is_a?(String) ? arg.split(".") : arg
+	    if overwrite
+	      key_path.inject(self) { |h,key| h.has_key?(key) ? h[key] :  h[key]={}} [last] = value 
+	    else
+	      key_path.inject(self) { |h,key| h.has_key?(key) ? h[key] :  h[key]={}} [last] ||= value
+	    end  
+	  end
+
+	  # get all keys path, { 'a' => 'value1', 'b' => { 'c' => 'value2'}}.all_keys_with_path => ['a','b.c']
+	  def all_keys_with_path(parent=nil)
+	  	a = []
+	  	each do |k,v|
+	  		if v.is_a?(Hash)
+	  			a << v.all_keys_with_path([parent,k].compact.join('.'))
+	  		else
+	  			a << "#{[parent,k].compact.join(".")}"
+	  		end
+	  	end
+	  	a.flatten
+	  end
+
+	end
+		
+	def self.included(receiver)
+		receiver.extend         ClassMethods
+		receiver.send :include, InstanceMethods
+	end
+end
+
+HashExtensions.included(Hash)

data/lib/eventhub/helper.rb

@@ -2,18 +2,54 @@ module EventHub
 
 	module Helper
 
-		# converts a base class name, Whatever::MyClassName => my_class_name
-		def class_to_string(class_name)
-			class_name.to_s.split("::")[-1].gsub(/[A-Z]/) { |c| "_#{c}"}.gsub(/^_/,"").downcase
+		# converts a class like EventHub::PlateStore::MyClassName to an array ['event_hub','plate_store','my_class_name']
+		def class_to_array(class_name)
+			class_name.to_s.split("::").map{ |m| m.gsub(/[A-Z]/) { |c| "_#{c}"}.gsub(/^_/,"").downcase }
 		end
 
-		def format_raw_string(message,max_characters=80) 
+		# replaces CR, LF, CRLF with ";" and cut's string to requied length by adding "..." if string would be longer
+		def format_string(message,max_characters=80) 
 		  max_characters = 5 if max_characters < 5
 		  m = message.gsub(/\r\n|\n|\r/m,";")
-		  return (m[0..max_characters-3] + "...") if m.size > max_characters
+		  return (m[0..max_characters-4] + "...") if m.size > max_characters
 		  return m
 	  end
 
+	  def get_host
+	  	Socket.gethostname
+	  end
+
+	  def get_ip_adresses
+	  	list = Socket.ip_address_list.map { |i| i.ip_address unless i.ipv4_loopback? || i.ipv6_loopback? }.compact
+	  	return list.size == 0 ? ["no ip address found (loopback excluded)"] : list
+	  end
+
+	  def now_stamp(now=nil)
+	  	now ||= Time.now
+      now.utc.strftime("%Y-%m-%dT%H:%M:%S.#{now.usec}Z")
+	  end
+
+	  def duration(difference)
+			rest, secs = difference.divmod( 60 )  # self is the time difference t2 - t1
+			rest, mins = rest.divmod( 60 )
+			days, hours = rest.divmod( 24 )
+			secs = secs.truncate
+			milliseconds = ((difference - difference.truncate)*1000).round
+
+			result = []
+			result << "#{days} days" if days > 1
+			result << "#{days} day" if days == 1
+			result << "#{hours} hours" if hours > 1
+			result << "#{hours} hour" if hours == 1
+			result << "#{mins} minutes" if mins > 1
+			result << "#{mins} minute" if mins == 1
+			result << "#{secs} seconds" if secs > 1
+			result << "#{secs} second" if secs == 1
+			result << "#{milliseconds} milliseconds" if milliseconds > 1
+			result << "#{milliseconds} millisecond" if milliseconds == 1
+			return result.join(' ') 
+    end
+
 	end	
 
 end

data/lib/eventhub/message.rb

@@ -1,6 +1,7 @@
 module EventHub
 
   class Message
+    include Helper
 
     VERSION = '1.0.0'
 
@@ -50,23 +51,27 @@ module EventHub
       @raw    = raw
 
       # set message defaults, that we have required headers
-      now = Time.now
       @header.set('message_id',UUIDTools::UUID.timestamp_create.to_s,false)
       @header.set('version',VERSION,false)
-      @header.set('created_at',now.utc.strftime("%Y-%m-%dT%H:%M:%S.#{now.usec/1000}Z"),false)
+      @header.set('created_at',now_stamp,false)
 
-      @header.set('process.name',nil,false)
+      @header.set('origin.module_id','undefined',false)
+      @header.set('origin.type','undefined',false)
+      @header.set('origin.site_id','undefined',false)
+
+      @header.set('process.name','undefined',false)
       @header.set('process.execution_id',UUIDTools::UUID.timestamp_create.to_s,false)
       @header.set('process.step_position',0,false)
 
       @header.set('status.retried_count',0,false)
       @header.set('status.code',STATUS_INITIAL,false)
-      @header.set('status.message',nil,false)
+      @header.set('status.message','',false)
 
     end
 
     def valid?
-      REQUIRED_HEADERS.all? { |key| @header.all_keys_with_path.include?(key) }
+      # check for existence and defined value
+      REQUIRED_HEADERS.all? { |key| @header.all_keys_with_path.include?(key) && !!self.send(key.gsub(/\./,"_").to_sym)}
     end
 
     def success?
@@ -90,14 +95,19 @@ module EventHub
     end
 
     def to_s
-      "Message: message_id [#{self.message_id}], status.code [#{status_code}], status.message [#{status_message}], status.retried_count [#{status_retried_count}] "
+      "Msg: process [#{self.process_name},#{self.process_step_position},#{self.process_execution_id}], status [#{self.status_code},#{self.status_message},#{self.status_retried_count}]"
     end
 
-    def copy(args={})
-      copied_header = self.header.dup
+    # copies the message and set's provided status code (default: success), actual stamp, and a new message id
+    def copy(status_code=STATUS_SUCCESS)
+      
+      copied_header = self.header.dup 
       copied_body   = self.body.dup
 
-      args.each { |key,value| copied_header.set(key,value) } if args.is_a?(Hash)
+      copied_header.set("message_id",UUIDTools::UUID.timestamp_create.to_s)
+      copied_header.set("created_at",now_stamp)
+      copied_header.set("status.code",status_code)
+
       Message.new(copied_header, copied_body)
     end  
 

data/lib/eventhub/processor.rb

@@ -6,7 +6,7 @@ module EventHub
 		include Helper
 
 		def initialize(name=nil)
-			@name = name || class_to_string(self.class)
+			@name = name || class_to_array(self.class)[1..-1].join(".")
 			@folder = Dir.pwd
 
 			@started = Time.now
@@ -21,28 +21,32 @@ module EventHub
 			EventHub::Configuration.instance.data
 		end
 
-		def host
+		def server_host
 			configuration.get('server.host') || 'localhost'
 		end
 
-		def user
+		def server_user
 			configuration.get('server.user') || 'admin'
 		end
 
-		def password
+		def server_password
 			configuration.get('server.password') || 'admin'
 		end
 
-		def management_port
+		def server_management_port
 			configuration.get('server.management_port') || 15672
 		end
 
-		def listener_queue
-			configuration.get('processor.listener_queue') || 'inbound'
+		def server_vhost
+			configuration.get('server.vhost') || 'event_hub'
 		end
 
-		def vhost
-			configuration.get('server.vhost') || nil
+		def connection_settings
+			{ user: server_user, password: server_password, host: server_host, vhost: server_vhost }
+		end
+
+		def listener_queue
+			configuration.get('processor.listener_queue') || 'undefined_listener_queue'
 		end
 
 		def watchdog_cycle_in_s
@@ -60,42 +64,64 @@ module EventHub
 		def start(detached=false)
 			daemonize if detached
 
+			EventHub.logger.info("Processor [#{@name}] base folder [#{@folder}]")
+
+			channel = nil
+
 			while @restart
 
 				begin 
-					AMQP.start(configuration.get('server')) do |connection, open_ok|
+					AMQP.start(self.connection_settings) do |connection, open_ok|
 
 						@connection = connection
 						
 						# deal with tcp connection issues
 						@connection.on_tcp_connection_loss do |conn, settings|
-				  		EventHub.logger.warn("Processor lost tcp connection. Trying to restart in #{@restart_in_s} seconds...")
+				  		EventHub.logger.warn("Processor lost tcp connection. Trying to restart in #{self.restart_in_s} seconds...")
 				  		stop_processor(true)
 				  	end
 
 						# create channel
-						@channel = AMQP::Channel.new(@connection)
-				  	@channel.auto_recovery = true
+						channel = AMQP::Channel.new(@connection)
 				  
 				  	# connect to queue
-				  	@queue = @channel.queue(self.listener_queue, durable: true, auto_delete: false)
+				  	@queue = channel.queue(self.listener_queue, durable: true, auto_delete: false)
 
 				  	# subscribe to queue
-					  @queue.subscribe(:ack => true) do |metadata, payload|
+					  @queue.subscribe(:ack => true) do |metadata, payload|	  	
+
+					  	
 					  	begin
-						  	if handle_message(metadata,payload)
-						  		raise
-						  		metadata.ack
-						  	else
-						  		metadata.nack
+					  		messages_to_send = []
+
+					  		# try to convert to Evenhub message
+					  		message = Message.from_json(payload)
+					  		EventHub.logger.info("-> #{message.to_s}")
+
+					  		if message.status_code == STATUS_INVALID
+					  			messages_to_send << message
+					  			EventHub.logger.info("-> #{message.to_s} => Put to queue [#{EH_X_INBOUND}].")
+					  		else
+						  		# pass received message to handler or dervied handler
+						  		messages_to_send = Array(handle_message(message))
+						  	end
+
+					  		# forward invalid or returned messages to dispatcher
+						  	messages_to_send.each do |message|
+						  		send_message(message)
 						  	end
+						  	channel.acknowledge(metadata.delivery_tag)
+						  	@messages_successful += 1
+						  	
 					  	rescue => e
-					  		EventHub.logger.error("Unexpected exception in handle_message method: #{e}")
+					  		channel.reject(metadata.delivery_tag,false)
+					  		@messages_unsuccessful += 1
+					  		EventHub.logger.error("Unexpected exception in handle_message method: #{e}. Message dead lettered.")
 								EventHub.logger.save_detailed_error(e)
 					  	end
 					  end
 
-					  EventHub.logger.info("Processor [#{@name}] is listening to queue [#{self.listener_queue}], base folder [#{@folder}]")
+					  EventHub.logger.info("Processor [#{@name}] is listening to vhost [#{self.server_vhost}], queue [#{self.listener_queue}]")
 
 					  # Singnal Listening
 					  Signal.trap("TERM") {stop_processor}
@@ -111,7 +137,7 @@ module EventHub
 					Signal.trap("INT")  { stop_processor }
 
 					id = EventHub.logger.save_detailed_error(e)
-					EventHub.logger.error("Unexpected exception: #{e}, see => #{id}")
+					EventHub.logger.error("Unexpected exception: #{e}, see => #{id}. Trying to restart in #{self.restart_in_s} seconds...")
 					
 					sleep_break self.restart_in_s
 				end
@@ -134,15 +160,16 @@ module EventHub
 
 		def watchdog
 			begin
-				response = RestClient.get "http://#{self.user}:#{self.password}@#{self.host}:#{self.management_port}/api/queues/#{self.vhost}/#{self.listener_queue}/bindings", { :content_type => :json}
+				response = RestClient.get "http://#{self.server_user}:#{self.server_password}@#{self.server_host}:#{self.server_management_port}/api/queues/#{self.server_vhost}/#{self.listener_queue}/bindings", { :content_type => :json}
   			data = JSON.parse(response.body)
   	
   			if response.code != 200
   				EventHub.logger.warn("Watchdog: Server did not answered properly. Trying to restart in #{self.restart_in_s} seconds...")
-  				stop_processor
+  				EventMachine.add_timer(self.restart_in_s) { stop_processor(true) }
   			elsif data.size == 0
   				EventHub.logger.warn("Watchdog: Something is wrong with the vhost, queue, and/or bindings. Trying to restart in #{self.restart_in_s} seconds...")
-  				stop_processor
+  				EventMachine.add_timer(self.restart_in_s) { stop_processor(true) }
+  				# does it make sence ? Needs maybe more checks in future
   			else
   				# Watchdog is happy :-)
 					# add timer for next check
@@ -159,16 +186,20 @@ module EventHub
 			message = Message.new
 			message.origin_module_id 	= @name
 			message.origin_type 			= "processor"
-			message.origin_site_id 		= 'chbs'
+			message.origin_site_id 		= 'global'
 
 			message.process_name    	= 'event_hub.heartbeat'
 
+			now = Time.now
 			message.body = {
 				 heartbeat: {
-				 started: 							@started,
-				 stamp_last_beat:       Time.now, 
+				 started: 							now_stamp(@started),
+				 stamp_last_beat:       now_stamp(now), 
+				 uptime:                duration(now-@started),
 				 heartbeat_cycle_in_s: 	self.heartbeat_cycle_in_s,
 				 served_queues: 				[self.listener_queue],
+				 host: 									get_host,
+				 ip_adresses:           get_ip_adresses,
 				 messages: {
 				 	total: 								@messages_successful+@messages_unsuccessful,
 				 	successful: 					@messages_successful,
@@ -177,35 +208,21 @@ module EventHub
 				}
 			}
 
-			send_to_dispatcher(message.to_json)
+			# send heartbeat message
+			send_message(message)
 
 			EventMachine.add_timer(self.heartbeat_cycle_in_s) { heartbeat }
 		end
 
-		def send_to_dispatcher(payload)
-			confirmed = true
-
-			connection = Bunny.new({hostname: self.host, user: self.user, password: self.password, vhost: "event_hub"})
-			connection.start
-
-			channel = connection.create_channel
-      channel.confirm_select
-
-    	channel.default_exchange.publish(payload,routing_key: EVENT_HUB_QUEUE_INBOUND, persistent: true)
-    	success = channel.wait_for_confirms
-
-    	if !success
-      	EventHub.logger.error("Message has not been confirmed by the server to be received !!!")
-    		confirmed = false
-    		@messages_unsuccessful += 1
-    	else
-    		@messages_successful += 1	
-    	end
-
-      channel.close   
-      connection.close
-
-      confirmed
+		# send message
+		def send_message(message,exchange_name=EH_X_INBOUND)
+			AMQP::Channel.new(@connection) do |channel|
+				channel.direct(exchange_name, :durable => true, :auto_delete => false) do |exchange, declare_ok|
+      		exchange.publish(message.to_json, :persistent => true)
+      	end
+			end
+		rescue => e
+				EventHub.logger.error("Unexpected exception while sending message to [#{exchange_name}]: #{e}")
 		end
 
   	def sleep_break( seconds ) # breaks after n seconds or after interrupt
@@ -221,11 +238,11 @@ module EventHub
 		def stop_processor(restart=false)
 			@restart = restart
 
-			# stop event loop
-			@connection.disconnect { 
-				EventHub.logger.info("Processor [#{@name}] is stopping main event loop")
-				EventMachine.stop
-			}
+			# stop connection and event loop
+			if @connection
+				@connection.disconnect if @connection.connected?
+				EventMachine.stop if EventMachine.reactor_running?
+			end
 		end
 
 		def daemonize

data/lib/eventhub/version.rb

@@ -1,3 +1,3 @@
 module EventHub
-  VERSION = "0.0.6"
+  VERSION = "0.0.7"
 end

data/lib/eventhub-processor.rb

@@ -1,10 +1,10 @@
 require 'amqp'
-require 'bunny'
 require 'rest-client'
 require 'json'
 require 'singleton'
 require 'uuidtools'
 require 'base64'
+require 'socket'
 
 require_relative 'eventhub/version'
 require_relative 'eventhub/constant'
@@ -12,7 +12,7 @@ require_relative 'eventhub/helper'
 require_relative 'eventhub/multi_logger'
 
 require_relative 'eventhub/configuration'
-require_relative 'eventhub/hash'
+require_relative 'eventhub/hash_extensions'
 require_relative 'eventhub/processor'
 require_relative 'eventhub/message'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: eventhub-processor
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors:
 - Thomas Steiner
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-03 00:00:00.000000000 Z
+date: 2014-04-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -39,13 +39,13 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rest-client
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -53,7 +53,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: amqp
+  name: rest-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -67,7 +67,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: bunny
+  name: amqp
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -102,11 +102,10 @@ extensions: []
 extra_rdoc_files: []
 files:
 - LICENSE.txt
-- Rakefile
 - lib/eventhub-processor.rb
 - lib/eventhub/configuration.rb
 - lib/eventhub/constant.rb
-- lib/eventhub/hash.rb
+- lib/eventhub/hash_extensions.rb
 - lib/eventhub/helper.rb
 - lib/eventhub/message.rb
 - lib/eventhub/multi_logger.rb

data/Rakefile

@@ -1 +0,0 @@
-require "bundler/gem_tasks"

data/lib/eventhub/hash.rb

@@ -1,34 +0,0 @@
-class ::Hash
-
-	# get value from provided key path, e.g. hash.get('a.b.c')
-  def get(key_path)
-    key_path.split('.').inject(self,:[])
-  rescue NoMethodError 
-    return nil
-  end
-
-  # set value from provided key path, e.h. hash.set('a.b.c','new value')
-  # if overwrite is false, value will be set if it was nil previously
-  def set(key_path,value,overwrite=true)
-    *key_path, last = key_path.split(".")
-    if overwrite
-      key_path.inject(self) { |h,key| h.has_key?(key) ? h[key] :  h[key]={}} [last] = value 
-    else
-      key_path.inject(self) { |h,key| h.has_key?(key) ? h[key] :  h[key]={}} [last] ||= value
-    end  
-  end
-
-  # get all keys path, { 'a' => 'value1', 'b' => { 'c' => 'value2'}}.all_keys_with_path => ['a','b.c']
-  def all_keys_with_path(parent=nil)
-  	a = []
-  	each do |k,v|
-  		if v.is_a?(Hash)
-  			a << v.all_keys_with_path([parent,k].compact.join('.'))
-  		else
-  			a << "#{[parent,k].compact.join(".")}"
-  		end
-  	end
-  	a.flatten
-  end
-
-end