eventhub-processor2 1.0.0

data/lib/eventhub/actor_heartbeat.rb

@@ -0,0 +1,121 @@
+# EventHub module
+module EventHub
+  # Heartbeat class
+  class ActorHeartbeat
+    include Celluloid
+    include Helper
+    finalizer :cleanup
+
+    def initialize(processor_instance)
+      @processor_instance = processor_instance
+      async.start
+    end
+
+    def start
+      EventHub.logger.info('Heartbeat is starting...')
+
+      every(60) { EventHub.logger.info("Running actors: #{Celluloid::Actor.all.size}: #{Celluloid::Actor.all.map{ |a| a.class }.join(', ') }") }
+
+      publish(heartbeat(action: 'started'))
+      loop do
+        sleep Configuration.processor[:heartbeat_cycle_in_s]
+        EventHub.logger.info('Running heartbeat...')
+        publish(heartbeat)
+      end
+    end
+
+    def cleanup
+      EventHub.logger.info('Heartbeat is cleanig up...')
+      publish(heartbeat(action: 'stopped'))
+    end
+
+    private
+
+    def publish(message)
+      connection = Bunny.new(bunny_connection_properties)
+      connection.start
+      channel = connection.create_channel
+      channel.confirm_select
+      exchange = channel.direct(EventHub::EH_X_INBOUND, durable: true)
+      exchange.publish(message, persistent: true)
+      success = channel.wait_for_confirms
+
+      unless success
+        raise 'Published heartbeat message has '\
+          'not been confirmed by the server'
+      end
+    ensure
+      connection.close if connection
+    end
+
+    def heartbeat(args = { action: 'running' })
+      message = EventHub::Message.new
+      message.origin_module_id  = EventHub::Configuration.name
+      message.origin_type       = 'processor'
+      message.origin_site_id    = 'global'
+
+      message.process_name      = 'event_hub.heartbeat'
+
+      now = Time.now
+
+      # message structure needs more changes
+      message.body = {
+        version: @processor_instance.send(:version),
+        action:  args[:action],
+        pid:     Process.pid,
+        process_name: 'event_hub.heartbeat',
+        heartbeat: {
+          started: now_stamp(started_at),
+          stamp_last_beat: now_stamp(now),
+          uptime_in_ms: (now - started_at) * 1000,
+          heartbeat_cycle_in_ms: Configuration.processor[:heartbeat_cycle_in_s] * 1000,
+          queues_consuming_from: EventHub::Configuration.processor[:listener_queues],
+          queues_publishing_to: [EventHub::EH_X_INBOUND], # needs more dynamic in the future
+          host: Socket.gethostname,
+          addresses: addresses,
+          messages: messages_statistics
+        }
+      }
+      message.to_json
+    end
+
+    def started_at
+      @processor_instance.started_at
+    end
+
+    def statistics
+      @processor_instance.statistics
+    end
+
+    def addresses
+      interfaces = Socket.getifaddrs.select do |interface|
+        !interface.addr.ipv4_loopback? && !interface.addr.ipv6_loopback?
+      end
+
+      interfaces.map do |interface|
+        begin
+          {
+            interface: interface.name,
+            host_name: Socket.gethostname,
+            ip_address: interface.addr.ip_address
+          }
+        rescue
+          nil # will be ignored
+        end
+      end.compact
+    end
+
+    def messages_statistics
+      {
+        total: statistics.messages_total,
+        successful: statistics.messages_successful,
+        unsuccessful: statistics.messages_unsuccessful,
+        average_size: statistics.messages_average_size,
+        average_process_time_in_ms:
+          statistics.messages_average_process_time * 1000,
+        total_process_time_in_ms:
+          statistics.messages_total_process_time * 1000
+      }
+    end
+  end
+end

data/lib/eventhub/actor_listener.rb

@@ -0,0 +1,161 @@
+# EventHub module
+module EventHub
+  # Listner Class
+  class ActorListener
+    include Celluloid
+    include Helper
+    finalizer :cleanup
+
+    def initialize(processor_instance)
+      @actor_watchdog = ActorWatchdog.new_link
+      @connections= {}
+      @processor_instance = processor_instance
+      start
+    end
+
+    def start
+      EventHub.logger.info('Listener is starting...')
+      EventHub::Configuration.processor[:listener_queues].each_with_index do |queue_name, index|
+        async.listen(queue_name: queue_name, index: index)
+      end
+    end
+
+    def restart
+      raise 'Listener is restarting...'
+    end
+
+    def listen(args = {})
+      with_listen(args) do |connection, channel, consumer, queue, queue_name|
+        EventHub.logger.info("Listening to queue [#{queue_name}]")
+        consumer.on_delivery do |delivery_info, metadata, payload|
+          EventHub.logger.info("#{queue_name}: [#{delivery_info.delivery_tag}]"\
+                                 ' delivery')
+
+          # EventHub::logger.debug("delivery_info: #{delivery_info.inspect}")
+          # EventHub::logger.debug("metadata: #{metadata.inspect}")
+
+          @processor_instance.statistics.measure(payload.size) do
+            handle_payload(payload: payload,
+                           connection: connection,
+                           queue_name: queue_name,
+                           content_type: metadata[:content_type],
+                           priority: metadata[:priority],
+                           delivery_tag: delivery_info.delivery_tag
+                           )
+            channel.acknowledge(delivery_info.delivery_tag, false)
+          end
+
+          EventHub.logger.info("#{queue_name}: [#{delivery_info.delivery_tag}]"\
+                               ' acknowledged')
+        end
+        queue.subscribe_with(consumer, block: false)
+      end
+    end
+
+    def with_listen(args = {}, &block)
+      connection = Bunny.new(bunny_connection_properties)
+      connection.start
+      queue_name = args[:queue_name]
+      @connections[queue_name] = connection
+      channel = connection.create_channel
+      channel.prefetch(1)
+      queue = channel.queue(queue_name, durable: true)
+      consumer = EventHub::Consumer.new(channel,
+                                        queue,
+                                        EventHub::Configuration.name +
+                                          '-' +
+                                          args[:index].to_s,
+                                        false)
+      yield connection, channel, consumer, queue, queue_name
+    end
+
+    def handle_payload(args = {})
+      response_messages = []
+      connection = args[:connection]
+
+      # convert to EventHub message
+      message = EventHub::Message.from_json(args[:payload])
+
+      # append to execution history
+      message.append_to_execution_history(EventHub::Configuration.name)
+
+      # return invalid messages to dispatcher
+      if message.invalid?
+        response_messages << message
+        EventHub.logger.info("-> #{message.to_s} => return invalid to dispatcher")
+      else
+        begin
+          response_messages = @processor_instance.send(:handle_message,
+                                                       message,
+                                                       pass_arguments(args))
+        rescue => exception
+          # this catches unexpected exceptions in handle message method
+          # deadletter the message via dispatcher
+          message.status_code = EventHub::STATUS_DEADLETTER
+          message.status_message = exception
+          EventHub.logger.info("-> #{message.to_s} => return exception to diaptcher")
+          response_messages << message
+        end
+      end
+
+      Array(response_messages).each do |message|
+        publish(message: message.to_json, connection: connection)
+      end
+    end
+
+    def pass_arguments(args = {})
+      keys_to_pass = [:queue_name, :content_type, :priority, :delivery_tag]
+      args.select{ |key| keys_to_pass.include?(key) }
+    end
+
+    def publish(args = {})
+      with_publish(args) do |connection, exchange_name, message|
+        begin
+          channel = connection.create_channel
+          channel.confirm_select
+          exchange = channel.direct(exchange_name, durable: true)
+          exchange.publish(message, persistent: true)
+
+          success = channel.wait_for_confirms
+
+          unless success
+            raise 'Published message from Listener actor '\
+              'has not been confirmed by the server'
+          end
+        ensure
+          channel.close if channel
+        end
+      end
+    end
+
+
+    def with_publish(args = {}, &block)
+      message = args[:message]
+      return if message.nil?
+
+      need_to_close = false
+      connection = args[:connection]
+      if connection.nil?
+        connection = Bunny.new(bunny_connection_properties)
+        connection.start
+        need_to_close = true
+      end
+
+      exchange_name = args[:exchange_name] || EH_X_INBOUND
+
+      yield connection, exchange_name, message
+    ensure
+      connection.close if connection && need_to_close
+    end
+
+
+    def cleanup
+      EventHub.logger.info('Listener is cleanig up...')
+      # close all open connections
+      @connections.values.each do |connection|
+        connection.close if connection
+      end
+    end
+
+  end
+end

data/lib/eventhub/actor_watchdog.rb

@@ -0,0 +1,41 @@
+# EventHub module
+module EventHub
+  # Watchdog class
+  class ActorWatchdog
+    include Celluloid
+    include Helper
+    finalizer :cleanup
+
+    def initialize
+      async.start
+    end
+
+    def start
+      loop do
+        EventHub.logger.info('Running watchdog...')
+        watch
+        sleep Configuration.processor[:watchdog_cycle_in_s]
+      end
+    end
+
+    def cleanup
+      EventHub.logger.info('Watchdog is cleanig up...')
+    end
+
+    private
+
+    def watch
+      connection = Bunny.new(bunny_connection_properties)
+      connection.start
+
+      EventHub::Configuration.processor[:listener_queues].each do |queue_name|
+        unless connection.queue_exists?(queue_name)
+          EventHub.logger.warn("Queue [#{queue_name}] is missing")
+          raise "Queue [#{queue_name}] is missing"
+        end
+      end
+    ensure
+      connection.close if connection
+    end
+  end
+end

data/lib/eventhub/configuration.rb

@@ -0,0 +1,114 @@
+# EventHub module
+module EventHub
+  # Configuraiton module
+  module Configuration
+    # it's a singleton (don't allow to instantiate this class)
+    extend self
+
+    attr_reader :name           # name of processor
+    attr_reader :environment    # environment the processor is running
+    attr_reader :detached       # run processor run as a daemon
+    attr_reader :config_file    # name of configuration file
+    attr_reader :config_data    # data from configuration file
+
+    @name = 'undefined'
+    @environment = 'development'
+    @detached = false
+    @config_file = File.join(Dir.getwd, 'config', "#{@name}.json")
+    @config_data = {}
+
+    # set name of processor
+    def name=(value)
+      @name = value
+    end
+
+    def reset
+      @name = 'undefined'
+      @environment = 'development'
+      @detached = false
+      @config_file = File.join(Dir.getwd, 'config', "#{@name}.json")
+      @config_data = {}
+    end
+
+    # parse options from argument list
+    def parse_options(argv = ARGV)
+      @config_file = File.join(Dir.getwd, 'config', "#{@name}.json")
+
+      OptionParser.new do |opts|
+        note = 'Define environment'
+        opts.on('-e', '--environment ENVIRONMENT', note) do |environment|
+          @environment = environment
+        end
+
+        opts.on('-d', '--detached', 'Run processor detached as a daemon') do
+          @detached = true
+        end
+
+        note = 'Define configuration file'
+        opts.on('-c', '--config CONFIG', note) do |config|
+          @config_file = config
+        end
+      end.parse!(argv)
+
+      true
+    rescue OptionParser::InvalidOption => e
+      EventHub.logger.warn("Argument Parsing: #{e}")
+      false
+    rescue OptionParser::MissingArgument => e
+      EventHub.logger.warn("Argument Parsing: #{e}")
+      false
+    end
+
+    # load configuration from file
+    def load!(args = {})
+      # for better rspec testing
+      @config_file = args[:config_file] if args[:config_file]
+      @environment = args[:environment] if args[:environment]
+
+      new_data = {}
+      begin
+        new_data = JSON.parse(File.read(@config_file), symbolize_names: true)
+      rescue => e
+        EventHub.logger.warn("Exception while loading configuration file: #{e}")
+        EventHub.logger.info('Using default configuration values')
+      end
+
+      deep_merge!(@config_data, default_configuration)
+      new_data = new_data[@environment.to_sym]
+      deep_merge!(@config_data, new_data)
+    end
+
+    # Deep merging of hashes
+    # deep_merge by Stefan Rusterholz, see http://www.ruby-forum.com/topic/142809
+    def deep_merge!(target, data)
+      return if data.nil?
+      merger = proc do |_, v1, v2|
+        v1.is_a?(Hash) && v2.is_a?(Hash) ? v1.merge(v2, &merger) : v2
+      end
+      target.merge! data, &merger
+    end
+
+    def method_missing(name, *_args, &_block)
+      @config_data[name.to_sym] ||
+        fail(NoMethodError, "unknown configuration [#{name}]", caller)
+    end
+
+    def default_configuration
+      {
+        server: {
+          user: 'guest',
+          password: 'guest',
+          host: 'localhost',
+          vhost: 'event_hub',
+          port: 5672,
+          tls: false
+        },
+        processor: {
+          heartbeat_cycle_in_s: 300,
+          watchdog_cycle_in_s: 15,
+          listener_queues: [@name]
+        }
+      }
+    end
+  end
+end

data/lib/eventhub/constant.rb

@@ -0,0 +1,35 @@
+module EventHub
+  EH_X_INBOUND             = 'event_hub.inbound'
+
+  STATUS_INITIAL           = 0         # To be set when dispatcher needs to dispatch to first process step.
+
+  STATUS_SUCCESS           = 200       # To be set to indicate successful processed message. Dispatcher will routes message to the next step.
+
+  STATUS_RETRY             = 300       # To be set to trigger retry cycle controlled by the dispatcher
+  STATUS_RETRY_PENDING     = 301       # Set and used by the dispatcher only.
+                                       # Set before putting the message into a retry queue.
+                                       # Once message has been retried it will sent do the same step with status.code = STATUS_SUCCESS
+
+  STATUS_INVALID           = 400       # To be set to indicate invalid message (not json, invalid Event Hub Message).
+                                       # Dispatcher will publish message to the invalid queue.
+
+  STATUS_DEADLETTER        = 500       # To be set by dispatcher, processor or channel adapters to indicate
+                                       # that message needs to be dead-lettered. Rejected messages could miss the
+                                       # status.code = STATUS_DEADLETTER due to the RabbitMQ deadletter exchange mechanism.
+
+  STATUS_SCHEDULE          = 600       # To be set to trigger scheduler based on schedule block, proceses next process step
+  STATUS_SCHEDULE_RETRY    = 601       # To be set to trigger scheduler based on schedule block, retry actual process step
+  STATUS_SCHEDULE_PENDING  = 602       # Set and used by the dispatcher only. Set before putting the scheduled message to the schedule queue.
+
+  STATUS_CODE_TRANSLATION = {
+    STATUS_INITIAL => 'STATUS_INITIAL',
+    STATUS_SUCCESS => 'STATUS_SUCCESS',
+    STATUS_RETRY => 'STATUS_RETRY',
+    STATUS_RETRY_PENDING => 'STATUS_RETRY_PENDING',
+    STATUS_INVALID => 'STATUS_INVALID',
+    STATUS_DEADLETTER => 'STATUS_DEADLETTER',
+    STATUS_SCHEDULE => 'STATUS_SCHEDULE',
+    STATUS_SCHEDULE_RETRY => 'STATUS_SCHEDULE_RETRY',
+    STATUS_SCHEDULE_PENDING =>'STATUS_SCHEDULE_PENDING',
+  }
+end

data/lib/eventhub/consumer.rb

@@ -0,0 +1,9 @@
+# EventHub module
+module EventHub
+  # Heartbeat class
+  class Consumer < Bunny::Consumer
+    def handle_cancellation(_)
+      EventHub.logger.error("Consumer reports cancellation")
+    end
+  end
+end

data/lib/eventhub/hash_extensions.rb

@@ -0,0 +1,49 @@
+# HashExtensions module
+module HashExtensions
+  # ClassMethods module
+  module ClassMethods
+  end
+
+  # InstanceMethods module
+  module InstanceMethods
+    # get value from provided key path
+    # e.g. hash.get(%w(event_hub plate.queue1 retry_s))
+    # "a" => { "b" => { "c" => { "value"}}}
+    def get(arg)
+      path = arg.is_a?(String) ? arg.split('.') : arg
+      path.inject(self, :[])
+    end
+
+    # set value from provided key path, e.h. hash.set('a.b.c','new value')
+    # if overwrite is false, value will be set if it was nil previously
+    def set(arg, value, overwrite = true)
+      *key_path, last = arg.is_a?(String) ? arg.split('.') : arg
+      if overwrite
+        key_path.inject(self) { |h, key| h.key?(key) ? h[key] :  h[key] = {} } [last] = value
+      else
+        key_path.inject(self) { |h, key| h.key?(key) ? h[key] :  h[key] = {} } [last] ||= value
+      end
+    end
+
+    # get all keys with path,
+    # { 'a' => 'v1', 'b' => { 'c' => 'v2'}}.all_keys_with_path => ['a','b.c']
+    def all_keys_with_path(parent = nil)
+      a = []
+      each do |k, v|
+        if v.is_a?(Hash)
+          a << v.all_keys_with_path([parent, k].compact.join('.'))
+        else
+          a << [parent, k].compact.join('.').to_s
+        end
+      end
+      a.flatten
+    end
+  end
+
+  def self.included(receiver)
+    receiver.extend         ClassMethods
+    receiver.send :include, InstanceMethods
+  end
+end
+
+HashExtensions.included(Hash)

data/lib/eventhub/helper.rb

@@ -0,0 +1,55 @@
+require 'optparse'
+
+# EventHub module
+module EventHub
+  # Helper module
+  module Helper
+    # Extracts processor name from given class instance.
+    # Removes 'EventHub' module from name.
+
+    # Examples:
+    # EventHub::Namespace::Demo => namespace.demo
+    # EventHub::NameSpace::Demo => name_space.demo
+    # EventHub::NameSpace::DemoProcessor => name_space.demo_processor
+    # NameSpace::Demo => name_space.demo
+    def get_name_from_class(instance)
+      instance.class.to_s.split('::').map do |element|
+        next if element == 'EventHub'
+        element.split(/(?=[A-Z])/).join('_').downcase
+      end.compact.join('.')
+    end
+
+    def bunny_connection_properties
+      server = EventHub::Configuration.server
+
+      if Configuration.server[:tls]
+        {
+          user: server[:user],
+          password: server[:password],
+          host: server[:host],
+          vhost: server[:vhost],
+          port: server[:port],
+          tls: server[:tls],
+          logger: Logger.new('/dev/null'), # logs from Bunny not required
+          network_recovery_interval: 15
+        }
+      else
+        {
+          user: server[:user],
+          password: server[:password],
+          host: server[:host],
+          vhost: server[:vhost],
+          port: server[:port],
+          logger: Logger.new('/dev/null'), # logs from Bunny not required
+          network_recovery_interval: 15
+        }
+      end
+    end
+
+    # Formats stamp into UTC format
+    def now_stamp(now=nil)
+      now ||= Time.now
+      now.utc.strftime("%Y-%m-%dT%H:%M:%S.%6NZ")
+    end
+  end
+end

data/lib/eventhub/logger.rb

@@ -0,0 +1,14 @@
+# EventHub module
+module EventHub
+  def self.logger
+    unless @logger
+      @logger = ::EventHub::Components::MultiLogger.new
+      @logger.add_device(Logger.new(STDOUT))
+      @logger.add_device(
+        EventHub::Components::Logger.logstash(Configuration.name,
+                                              Configuration.environment)
+      )
+    end
+    @logger
+  end
+end