eventhub-processor 0.2.3 → 0.3.0

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    YjAxYjAwMjEyMzZiODlhNTIzNjI2MTQ5OWI2ZDNkYzJlM2I4MTE4MQ==
-  data.tar.gz: !binary |-
-    M2Q2NjM4YTg2YTBmMWViMzcyYTU0ODQ3NDAzZjgyYzhlNjlkYjQwYQ==
+SHA1:
+  metadata.gz: ea2ee52ffdb9af4cd5133eb9febcf4e51e1ca363
+  data.tar.gz: bafee6fc1bb89684fd29ac0b4d3119bd6c82af28
 SHA512:
-  metadata.gz: !binary |-
-    ZGNlNGQ1MzE4NTRiYmFkMDE0MjY0ZTFlNjZiMGZlNGQzZjYxZTNkMWFhYzJl
-    MzUyNGVlYTkzNDYyYzc4YTE3YmQ2N2NmM2M0NjdjYjVhNGE3MmEwZTI2M2Ux
-    NDVmOWNiOGM2OGExYjQwN2U0NWIyZGMwYmYwZjE5ZmQ4MzQ0NjI=
-  data.tar.gz: !binary |-
-    ZGExOGJlMTVkNDA4NzVlOWRlNjRjMDVmMmUzY2M0MDcwYTNlMjRjNDBiMDFk
-    ZDg4OGY0NmE0YWU4OTllMjllYWQ4YjNkN2ZiMmJjNTYyNjAwZTgwMjM3NGEw
-    OWY4ZTBiMmYzYjFlMzQ0MzMyMWZiZTFkYWU2NmM2NzE2M2RhYjI=
+  metadata.gz: aad91acb51fff2d8078a85f762c5a7b43bb65d2b187a75760d01f3697cc67c445aa0e276675239506d6c35807b10f67cca9026f0e92463799f712d9339395023
+  data.tar.gz: ac51f8ba1761f9e833fc621df03f41826afbdfb2985a538d8eb5a4cd3861c2858ad504bb260fd1c546ef2c0bc35a9c82dbae1fa56bfe6f475799d752c8490a96

data/lib/eventhub/argument_parser.rb

@@ -1,6 +1,7 @@
 module EventHub
 
   class ArgumentParser
+
     def self.parse(args)
       # The options specified on the command line will be collected in *options*.
       # We set default values here.
@@ -10,6 +11,7 @@ module EventHub
 
       opt_parser = OptionParser.new do |opts|
         opts.banner = "Usage: #{args[0]}.rb [options]"
+        yield(opts, options) if block_given? # allow to add more options
 
         opts.on("-e", "--environment ENVIRONMENT","Environment the processor is running") do |environment|
           options.environment = environment

data/lib/eventhub/base_exception.rb

@@ -0,0 +1,2 @@
+class EventHub::BaseException < RuntimeError
+end

data/lib/eventhub/configuration.rb

@@ -1,27 +1,26 @@
-module EventHub
-
-  class Configuration
-    include Singleton
-    include Helper
-
-    attr_accessor :data, :folder, :environment
-
-    def initialize
-      @data = nil
-      @folder = Dir.pwd
-      @environment = 'development'
-    end
-
-    def load_file(input, env = 'development')
-      tmp = JSON.parse( IO.read(input))
-      @data = tmp[env]
-      @environment = env
-      true
-    rescue => e
-      EventHub.logger.info("Unexpected exception while loading configuration [#{input}]: #{format_string(e.message)}")
-      false
-    end
-
-  end
-
+module EventHub
+
+  class Configuration
+    include Singleton
+    include Helper
+
+    attr_accessor :data, :folder, :environment
+
+    def initialize
+      @data = nil
+      @environment = 'development'
+    end
+
+    def load_file(input, env = 'development')
+      json = JSON.parse(IO.read(input))
+      @data = json[env]
+      @environment = env
+      true
+    rescue => e
+      EventHub.logger.info("Unexpected exception while loading configuration [#{input}]: #{format_string(e.message)}")
+      false
+    end
+
+  end
+
 end

data/lib/eventhub/constant.rb

@@ -1,16 +1,16 @@
 module EventHub
 
-	EH_X_INBOUND 						= 'event_hub.inbound'
-						
-	STATUS_INITIAL 					= 0         # To be set when dispatcher needs to dispatch to first process step.
-	STATUS_SUCCESS 					= 200       # To be set to indicate successful processed message. Dispatcher will routes message to the next step.
-	STATUS_RETRY 						= 300       # To be set to trigger retry cycle controlled by the dispatcher
-	STATUS_RETRY_PENDING		= 301			  # Set and used by the dispatcher only. 
-																			# Set before putting the message into a retry queue. 
-																			# Once message has been retried it will sent do the same step with status.code = STATUS_SUCCESS 
-	STATUS_INVALID					= 400       # To be set to indicate invalid message (not json, invalid Event Hub Message). 
-																			# Dispatcher will publish message to the invalid queue.
-	STATUS_DEADLETTER 			= 500  			# To be set by dispatcher, processor or channel adapters to indicate 
-																			# that message needs to be dead-lettered. Rejected messages could miss the 
-																			# status.code = STATUS_DEADLETTER due to the RabbitMQ deadletter exchange mechanism.
+  EH_X_INBOUND            = 'event_hub.inbound'
+            
+  STATUS_INITIAL          = 0         # To be set when dispatcher needs to dispatch to first process step.
+  STATUS_SUCCESS          = 200       # To be set to indicate successful processed message. Dispatcher will routes message to the next step.
+  STATUS_RETRY            = 300       # To be set to trigger retry cycle controlled by the dispatcher
+  STATUS_RETRY_PENDING    = 301       # Set and used by the dispatcher only. 
+                                      # Set before putting the message into a retry queue. 
+                                      # Once message has been retried it will sent do the same step with status.code = STATUS_SUCCESS 
+  STATUS_INVALID          = 400       # To be set to indicate invalid message (not json, invalid Event Hub Message). 
+                                      # Dispatcher will publish message to the invalid queue.
+  STATUS_DEADLETTER       = 500       # To be set by dispatcher, processor or channel adapters to indicate 
+                                      # that message needs to be dead-lettered. Rejected messages could miss the 
+                                      # status.code = STATUS_DEADLETTER due to the RabbitMQ deadletter exchange mechanism.
 end

data/lib/eventhub/heartbeat.rb

@@ -0,0 +1,77 @@
+module EventHub
+
+  class Heartbeat
+    include Helper
+
+    attr_reader :processor, :statistics, :started_at
+
+    def initialize(processor)
+      @started_at = Time.now
+      @processor = processor
+      @statistics = @processor.statistics
+    end
+
+
+    def build_message(action = "running")
+      message = ::EventHub::Message.new
+      message.origin_module_id  = processor.name
+      message.origin_type       = "processor"
+      message.origin_site_id    = 'global'
+
+      message.process_name      = 'event_hub.heartbeat'
+
+      now = Time.now
+
+      # message structure needs more changes
+      message.body = {
+        version: processor.version,
+        action:  action,
+        pid:     Process.pid,
+        process_name: 'event_hub.heartbeat',
+
+        heartbeat: {
+          started:                      now_stamp(started_at),
+          stamp_last_beat:              now_stamp(now),
+          uptime_in_ms:                 (now - started_at)*1000,
+          heartbeat_cycle_in_ms:        processor.heartbeat_cycle_in_s * 1000,
+          queues_consuming_from:        processor.listener_queues,
+          queues_publishing_to:         ['event_hub.inbound'], # needs more dynamic in the future
+          host:                         Socket.gethostname,
+          addresses:                    addresses,
+          messages: {
+            total:                      statistics.messages_total,
+            successful:                 statistics.messages_successful,
+            unsuccessful:               statistics.messages_unsuccessful,
+            average_size:               statistics.messages_average_size,
+            average_process_time_in_ms: statistics.messages_average_process_time*1000,
+            total_prozess_time_in_ms:   statistics.messages_total_process_time*1000
+          }
+        }
+      }
+      message
+    end
+
+    private
+
+    def addresses
+      interfaces = Socket.getifaddrs.select do |interface|
+        !interface.addr.ipv4_loopback? && !interface.addr.ipv6_loopback?
+      end
+
+      interfaces.map do |interface|
+        begin
+          {
+            :interface => interface.name,
+            :host_name => Socket.gethostname,
+            :ip_address => interface.addr.ip_address
+          }
+        rescue
+          nil # will be ignored
+        end
+      end.compact
+
+    end
+
+  end
+
+end

data/lib/eventhub/helper.rb

@@ -1,55 +1,49 @@
-module EventHub
-
-	module Helper
-
-		# converts a class like EventHub::PlateStore::MyClassName to an array ['event_hub','plate_store','my_class_name']
-		def class_to_array(class_name)
-			class_name.to_s.split("::").map{ |m| m.gsub(/[A-Z]/) { |c| "_#{c}"}.gsub(/^_/,"").downcase }
-		end
-
-		# replaces CR, LF, CRLF with ";" and cut's string to requied length by adding "..." if string would be longer
-		def format_string(message,max_characters=80) 
-		  max_characters = 5 if max_characters < 5
-		  m = message.gsub(/\r\n|\n|\r/m,";")
-		  return (m[0..max_characters-4] + "...") if m.size > max_characters
-		  return m
-	  end
-
-	  def get_host
-	  	Socket.gethostname
-	  end
-
-	  def get_ip_adresses
-	  	list = Socket.ip_address_list.map { |i| i.ip_address unless i.ipv4_loopback? || i.ipv6_loopback? }.compact
-	  	return list.size == 0 ? ["no ip address found (loopback excluded)"] : list
-	  end
-
-	  def now_stamp(now=nil)
-	  	now ||= Time.now
-      now.utc.strftime("%Y-%m-%dT%H:%M:%S.#{now.usec}Z")
-	  end
-
-	  def duration(difference)
-			rest, secs = difference.divmod( 60 )  # self is the time difference t2 - t1
-			rest, mins = rest.divmod( 60 )
-			days, hours = rest.divmod( 24 )
-			secs = secs.truncate
-			milliseconds = ((difference - difference.truncate)*1000).round
-
-			result = []
-			result << "#{days} days" if days > 1
-			result << "#{days} day" if days == 1
-			result << "#{hours} hours" if hours > 1
-			result << "#{hours} hour" if hours == 1
-			result << "#{mins} minutes" if mins > 1
-			result << "#{mins} minute" if mins == 1
-			result << "#{secs} seconds" if secs > 1
-			result << "#{secs} second" if secs == 1
-			result << "#{milliseconds} milliseconds" if milliseconds > 1
-			result << "#{milliseconds} millisecond" if milliseconds == 1
-			return result.join(' ') 
-    end
-
-	end	
-
-end
+module EventHub
+
+	module Helper
+
+		# converts a class like EventHub::PlateStore::MyClassName to an array ['event_hub','plate_store','my_class_name']
+		def class_to_array(class_name)
+			class_name.to_s.split("::").map{ |m| m.gsub(/[A-Z]/) { |c| "_#{c}"}.gsub(/^_/,"").downcase }
+		end
+
+		# replaces CR, LF, CRLF with ";" and cut's string to requied length by adding "..." if string would be longer
+		def format_string(message,max_characters=80)
+		  max_characters = 5 if max_characters < 5
+		  m = message.gsub(/\r\n|\n|\r/m,";")
+		  return (m[0..max_characters-4] + "...") if m.size > max_characters
+		  return m
+	  end
+
+	  def now_stamp(now=nil)
+	  	now ||= Time.now
+      now.utc.strftime("%Y-%m-%dT%H:%M:%S.#{now.usec}Z")
+	  end
+
+	  def duration(difference)
+	  	negative = difference < 0
+	  	difference = difference.abs
+
+			rest, secs = difference.divmod( 60 )  # self is the time difference t2 - t1
+			rest, mins = rest.divmod( 60 )
+			days, hours = rest.divmod( 24 )
+			secs = secs.truncate
+			milliseconds = ((difference - difference.truncate)*1000).round
+
+			result = []
+			result << "#{days} days" if days > 1
+			result << "#{days} day" if days == 1
+			result << "#{hours} hours" if hours > 1
+			result << "#{hours} hour" if hours == 1
+			result << "#{mins} minutes" if mins > 1
+			result << "#{mins} minute" if mins == 1
+			result << "#{secs} seconds" if secs > 1
+			result << "#{secs} second" if secs == 1
+			result << "#{milliseconds} milliseconds" if milliseconds > 1
+			result << "#{milliseconds} millisecond" if milliseconds == 1
+			return (negative ? "-" : "") + result.join(' ')
+    end
+
+	end
+
+end

data/lib/eventhub/message.rb

@@ -1,128 +1,138 @@
-module EventHub
-
-  class Message
-    include Helper
-
-    VERSION = '1.0.0'
-
-    # Headers that are required (value can be nil) in order to pass valid?
-    REQUIRED_HEADERS = [
-      'message_id',
-      'version',
-      'created_at',
-      'origin.module_id',
-      'origin.type',
-      'origin.site_id',
-      'process.name',
-      'process.step_position',
-      'process.execution_id',
-      'status.retried_count',
-      'status.code',
-      'status.message'
-    ]
-
-    attr_accessor :header, :body, :raw, :vhost, :routing_key
-
-    # Build accessors for all required headers
-    REQUIRED_HEADERS.each do |header|
-      name = header.gsub(/\./,"_")
-
-      define_method(name) do
-        self.header.get(header)
-      end
-
-      define_method("#{name}=") do |value|
-        self.header.set(header,value)
-      end
-    end
-
-    def self.from_json(raw)
-      data = JSON.parse(raw)
-      Message.new(data.get('header'), data.get('body'),raw)
-    rescue => e
-      Message.new({ "status" =>  { "code" => STATUS_INVALID, "message" => "JSON parse error: #{e}" }} ,{ "original_message_base64_encoded" => Base64.encode64(raw)},raw)
-    end
-
-    # process_step_position should be
-    def initialize(header=nil, body=nil,raw=nil)
-
-      @header = header || {}
-      @body   = body || {}
-      @raw    = raw
-
-      # set message defaults, that we have required headers
-      @header.set('message_id',UUIDTools::UUID.timestamp_create.to_s,false)
-      @header.set('version',VERSION,false)
-      @header.set('created_at',now_stamp,false)
-
-      @header.set('origin.module_id','undefined',false)
-      @header.set('origin.type','undefined',false)
-      @header.set('origin.site_id','undefined',false)
-
-      @header.set('process.name','undefined',false)
-      @header.set('process.execution_id',UUIDTools::UUID.timestamp_create.to_s,false)
-      @header.set('process.step_position',0,false)
-
-      @header.set('status.retried_count',0,false)
-      @header.set('status.code',STATUS_INITIAL,false)
-      @header.set('status.message','',false)
-
-    end
-
-    def valid?
-      # check for existence and defined value
-      REQUIRED_HEADERS.all? { |key| @header.all_keys_with_path.include?(key) && !!self.send(key.gsub(/\./,"_").to_sym)}
-    end
-
-    def success?
-      self.status_code == STATUS_SUCCESS
-    end
-
-    def retry?
-      !success?
-    end
-
-    def initial?
-      self.status_code == STATUS_INITIAL
-    end
-
-    def retry_pending?
-      self.status_code == STATUS_RETRY_PENDING
-    end
-
-    def to_json
-      {'header' => self.header, 'body' => self.body}.to_json
-    end
-
-    def to_s
-      "Msg: process [#{self.process_name},#{self.process_step_position},#{self.process_execution_id}], status [#{self.status_code},#{self.status_message},#{self.status_retried_count}]"
-    end
-
-    # copies the message and set's provided status code (default: success), actual stamp, and a new message id
-    def copy(status_code=STATUS_SUCCESS)
-      
-      # use Marshal dump and load to make a deep object copy
-      copied_header = Marshal.load( Marshal.dump(header)) 
-      copied_body   = Marshal.load( Marshal.dump(body))
-
-      copied_header.set("message_id",UUIDTools::UUID.timestamp_create.to_s)
-      copied_header.set("created_at",now_stamp)
-      copied_header.set("status.code",status_code)
-
-      Message.new(copied_header, copied_body)
-    end  
-
-    def self.translate_status_code(code)
-      case code
-        when EventHub::STATUS_INITIAL       then return 'STATUS_INITIAL'
-        when EventHub::STATUS_SUCCESS       then return 'STATUS_SUCCESS'
-        when EventHub::STATUS_RETRY         then return 'STATUS_RETRY'
-        when EventHub::STATUS_RETRY_PENDING then return 'STATUS_RETRY_PENDING'
-        when EventHub::STATUS_INVALID       then return 'STATUS_INVALID'
-        when EventHub::STATUS_DEADLETTER    then return 'STATUS_DEADLETTER'
-      end
-    end
-
-  end
-
-end
+module EventHub
+
+  class Message
+    include Helper
+
+    VERSION = '1.0.0'
+
+    # Headers that are required (value can be nil) in order to pass valid?
+    REQUIRED_HEADERS = [
+      'message_id',
+      'version',
+      'created_at',
+      'origin.module_id',
+      'origin.type',
+      'origin.site_id',
+      'process.name',
+      'process.step_position',
+      'process.execution_id',
+      'status.retried_count',
+      'status.code',
+      'status.message'
+    ]
+
+    attr_accessor :header, :body, :raw, :vhost, :routing_key
+
+    # Build accessors for all required headers
+    REQUIRED_HEADERS.each do |header|
+      name = header.gsub(/\./,"_")
+
+      define_method(name) do
+        self.header.get(header)
+      end
+
+      define_method("#{name}=") do |value|
+        self.header.set(header,value)
+      end
+    end
+
+    def self.from_json(raw)
+      data = JSON.parse(raw)
+      Message.new(data.get('header'), data.get('body'),raw)
+    rescue => e
+      Message.new({ "status" =>  { "code" => STATUS_INVALID, "message" => "JSON parse error: #{e}" }} ,{ "original_message_base64_encoded" => Base64.encode64(raw)},raw)
+    end
+
+    def initialize(header = nil, body = nil, raw = nil)
+
+      @header = header || {}
+      @body   = body || {}
+      @raw    = raw
+
+      # set message defaults, that we have required headers
+      @header.set('message_id', UUIDTools::UUID.timestamp_create.to_s, false)
+      @header.set('version', VERSION, false)
+      @header.set('created_at', now_stamp, false)
+
+      @header.set('origin.module_id', 'undefined', false)
+      @header.set('origin.type', 'undefined', false)
+      @header.set('origin.site_id', 'undefined', false)
+
+      @header.set('process.name', 'undefined', false)
+      @header.set('process.execution_id', UUIDTools::UUID.timestamp_create.to_s, false)
+      @header.set('process.step_position', 0, false)
+
+      @header.set('status.retried_count', 0, false)
+      @header.set('status.code', STATUS_INITIAL, false)
+      @header.set('status.message', '', false)
+
+    end
+
+    def valid?
+      # check for existence and defined value
+      REQUIRED_HEADERS.all? { |key| @header.all_keys_with_path.include?(key) && !!self.send(key.gsub(/\./,"_").to_sym)}
+    end
+
+    def success?
+      self.status_code == STATUS_SUCCESS
+    end
+
+    def retry?
+      !success?
+    end
+
+    def initial?
+      self.status_code == STATUS_INITIAL
+    end
+
+    def retry_pending?
+      self.status_code == STATUS_RETRY_PENDING
+    end
+
+    def invalid?
+      self.status_code == STATUS_INVALID
+    end
+
+    def to_json
+      {'header' => self.header, 'body' => self.body}.to_json
+    end
+
+    def to_s
+      "Msg: process [#{self.process_name},#{self.process_step_position},#{self.process_execution_id}], status [#{self.status_code},#{self.status_message},#{self.status_retried_count}]"
+    end
+
+    # copies the message and set's provided status code (default: success), actual stamp, and a new message id
+    def copy(status_code = STATUS_SUCCESS)
+
+      # use Marshal dump and load to make a deep object copy
+      copied_header = Marshal.load( Marshal.dump(header))
+      copied_body   = Marshal.load( Marshal.dump(body))
+
+      copied_header.set("message_id",UUIDTools::UUID.timestamp_create.to_s)
+      copied_header.set("created_at",now_stamp)
+      copied_header.set("status.code",status_code)
+
+      Message.new(copied_header, copied_body)
+    end
+
+    def append_to_execution_history(processor_name)
+      unless header.get('execution_history')
+        header.set('execution_history', [])
+      end
+      header.get('execution_history') << {'processor' => processor_name, 'timestamp' => now_stamp}
+    end
+
+    def self.translate_status_code(code)
+      case code
+        when EventHub::STATUS_INITIAL       then return 'STATUS_INITIAL'
+        when EventHub::STATUS_SUCCESS       then return 'STATUS_SUCCESS'
+        when EventHub::STATUS_RETRY         then return 'STATUS_RETRY'
+        when EventHub::STATUS_RETRY_PENDING then return 'STATUS_RETRY_PENDING'
+        when EventHub::STATUS_INVALID       then return 'STATUS_INVALID'
+        when EventHub::STATUS_DEADLETTER    then return 'STATUS_DEADLETTER'
+      end
+    end
+
+  end
+
+end

data/lib/eventhub/message_processor.rb

@@ -0,0 +1,31 @@
+class EventHub::MessageProcessor
+  attr_reader :processor
+
+  def initialize(processor)
+    @processor = processor
+  end
+
+  def process(params, payload)
+    messages_to_send = []
+
+    # try to convert to EventHub message
+    message = EventHub::Message.from_json(payload)
+    EventHub.logger.info("-> #{message.to_s}")
+
+    message.append_to_execution_history(self.processor.name)
+
+    if message.invalid?
+      messages_to_send << message
+      EventHub.logger.info("-> #{message.to_s} => Put to queue [#{EventHub::EH_X_INBOUND}].")
+    else
+      # pass received message to handler or dervied handler
+      if processor.method(:handle_message).arity == 1
+        messages_to_send = Array(processor.handle_message(message))
+      else
+        messages_to_send = Array(processor.handle_message(message,params))
+      end
+    end
+
+    messages_to_send
+  end
+end