manageiq-messaging 0.1.0

data/lib/manageiq/messaging/kafka/common.rb

@@ -0,0 +1,105 @@
+module ManageIQ
+  module Messaging
+    module Kafka
+      module Common
+        require 'manageiq/messaging/common'
+        include ManageIQ::Messaging::Common
+
+        GROUP_FOR_QUEUE_MESSAGES = 'manageiq_messaging_queue_group'.freeze
+
+        private
+
+        def producer
+          @producer ||= kafka_client.producer
+        end
+
+        def topic_consumer(persist_ref)
+          # persist_ref enables consumer to receive messages sent when consumer is temporarily offline
+          # it also enables consumers to do load balancing when multiple consumers join the with the same ref.
+          @consumer.try(:stop) unless @persist_ref == persist_ref
+          @persist_ref = persist_ref
+          @topic_consumer ||= kafka_client.consumer(:group_id => persist_ref)
+        end
+
+        def queue_consumer
+          # all queue consumers join the same group so that each message can be processed by one and only one consumer
+          @queue_consumer ||= kafka_client.consumer(:group_id => GROUP_FOR_QUEUE_MESSAGES)
+        end
+
+        trap("TERM") do
+          @consumer.try(:stop)
+          @consumer = nil
+        end
+
+        def raw_publish(commit, body, options)
+          producer.produce(encode_body(options[:headers], body), options)
+          producer.deliver_messages if commit
+          logger.info("Published to topic(#{options[:topic]}), msg(#{payload_log(body.inspect)})")
+        end
+
+        def queue_for_publish(options)
+          body, kafka_opts = for_publish(options)
+          kafka_opts[:headers][:message_type] = options[:message] if options[:message]
+          kafka_opts[:headers][:class_name] = options[:class_name] if options[:class_name]
+
+          [body, kafka_opts]
+        end
+
+        def topic_for_publish(options)
+          body, kafka_opts = for_publish(options)
+          kafka_opts[:headers][:event_type] = options[:event] if options[:event]
+
+          [body, kafka_opts]
+        end
+
+        def for_publish(options)
+          kafka_opts = {:topic => address(options), :headers => {}}
+          kafka_opts[:partition_key] = options[:group_name] if options[:group_name]
+          kafka_opts[:headers][:sender] = options[:sender] if options[:sender]
+
+          body = options[:payload] || ''
+
+          [body, kafka_opts]
+        end
+
+        def address(options)
+          if options[:affinity]
+            "#{options[:service]}.#{options[:affinity]}"
+          else
+            options[:service]
+          end
+        end
+
+        def process_queue_message(queue, message)
+          payload = decode_body(message.headers, message.value)
+          sender, message_type, class_name = parse_message_headers(message.headers)
+          logger.info("Message received: queue(#{queue}), message(#{payload_log(payload)}), sender(#{sender}), type(#{message_type})")
+          [sender, message_type, class_name, payload]
+        end
+
+        def process_topic_message(topic, message)
+          begin
+            payload = decode_body(message.headers, message.value)
+            sender, event_type = parse_event_headers(message.headers)
+            logger.info("Event received: topic(#{topic}), event(#{payload_log(payload)}), sender(#{sender}), type(#{event_type})")
+            yield sender, event_type, payload
+            logger.info("Event processed")
+          rescue StandardError => e
+            logger.error("Event processing error: #{e.message}")
+            logger.error(e.backtrace.join("\n"))
+          end
+        end
+
+        def parse_message_headers(headers)
+          return [nil, nil, nil] unless headers.kind_of?(Hash)
+          headers.values_at('sender', 'message_type', 'class_name')
+        end
+
+        def parse_event_headers(headers)
+          return [nil, nil] unless headers.kind_of?(Hash)
+          headers.values_at('sender', 'event_type')
+        end
+      end
+    end
+  end
+end

data/lib/manageiq/messaging/kafka/queue.rb

@@ -0,0 +1,41 @@
+module ManageIQ
+  module Messaging
+    module Kafka
+      module Queue
+        private
+
+        def publish_message_impl(options)
+          raise ArgumentError, "Kafka messaging implementation does not take a block" if block_given?
+          raw_publish(true, *queue_for_publish(options))
+        end
+
+        def publish_messages_impl(messages)
+          messages.each { |msg_options| raw_publish(false, *queue_for_publish(msg_options)) }
+          producer.deliver_messages
+        end
+
+        def subscribe_messages_impl(options)
+          topic = address(options)
+
+          consumer = queue_consumer
+          consumer.subscribe(topic)
+          consumer.each_batch do |batch|
+            logger.info("Batch message received: queue(#{topic})")
+            begin
+              messages = batch.messages.collect do |message|
+                sender, message_type, _class_name, payload = process_queue_message(topic, message)
+                ManageIQ::Messaging::ReceivedMessage.new(sender, message_type, payload, nil)
+              end
+
+              yield messages
+            rescue StandardError => e
+              logger.error("Event processing error: #{e.message}")
+              logger.error(e.backtrace.join("\n"))
+            end
+            logger.info("Batch message processed")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/manageiq/messaging/kafka/topic.rb

@@ -0,0 +1,28 @@
+module ManageIQ
+  module Messaging
+    module Kafka
+      module Topic
+        private
+
+        def publish_topic_impl(options)
+          raw_publish(true, *topic_for_publish(options))
+        end
+
+        def subscribe_topic_impl(options, &block)
+          topic = address(options)
+          persist_ref = options[:persist_ref]
+
+          if persist_ref
+            consumer = topic_consumer(persist_ref)
+            consumer.subscribe(topic, :start_from_beginning => false)
+            consumer.each_message { |message| process_topic_message(topic, message, &block) }
+          else
+            kafka_client.each_message(:topic => topic, :start_from_beginning => false) do |message|
+              process_topic_message(topic, message, &block)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/manageiq/messaging/null_logger.rb

@@ -0,0 +1,11 @@
+require 'logger'
+
+module ManageIQ
+  module Messaging
+    class NullLogger < Logger
+      def initialize(*_args); end
+
+      def add(*_args, &_block); end
+    end
+  end
+end

data/lib/manageiq/messaging/received_message.rb

@@ -0,0 +1,11 @@
+module ManageIQ
+  module Messaging
+    class ReceivedMessage
+      attr_accessor :sender, :message, :payload, :ack_ref
+
+      def initialize(sender, message, payload, ack_ref)
+        @sender, @message, @payload, @ack_ref = sender, message, payload, ack_ref
+      end
+    end
+  end
+end

data/lib/manageiq/messaging/stomp.rb

@@ -0,0 +1,7 @@
+module ManageIQ
+  module Messaging
+    module Stomp
+      autoload :Client, 'manageiq/messaging/stomp/client'
+    end
+  end
+end

data/lib/manageiq/messaging/stomp/background_job.rb

@@ -0,0 +1,61 @@
+module ManageIQ
+  module Messaging
+    module Stomp
+      module BackgroundJob
+        private
+
+        def subscribe_background_job_impl(options)
+          queue_name, headers = queue_for_subscribe(options)
+
+          subscribe(queue_name, headers) do |msg|
+            begin
+              ack(msg)
+              assert_options(msg.headers, ['class_name', 'message_type'])
+
+              msg_options = decode_body(msg.headers, msg.body)
+              msg_options = {} if msg_options.empty?
+              logger.info("Processing background job: queue(#{queue_name}), job(#{msg_options.inspect}), headers(#{msg.headers})")
+              result = run_job(msg_options.merge(:class_name => msg.headers['class_name'], :method_name => msg.headers['message_type']))
+              logger.info("Background job completed")
+
+              correlation_ref = msg.headers['correlation_id']
+              send_response(options[:service], correlation_ref, result) if correlation_ref
+            rescue Timeout::Error
+              logger.warn("Background job timed out")
+              if Object.const_defined?('ActiveRecord::Base')
+                begin
+                  logger.info("Reconnecting to DB after timeout error during queue deliver")
+                  ActiveRecord::Base.connection.reconnect!
+                rescue => err
+                  logger.error("Error encountered during <ActiveRecord::Base.connection.reconnect!> error:#{err.class.name}: #{err.message}")
+                end
+              end
+            rescue => e
+              logger.error("Background job error: #{e.message}")
+              logger.error(e.backtrace.join("\n"))
+            end
+          end
+        end
+
+        def run_job(options)
+          assert_options(options, [:class_name, :method_name])
+
+          instance_id = options[:instance_id]
+          args = options[:args]
+          miq_callback = options[:miq_callback]
+
+          obj = Object.const_get(options[:class_name])
+          obj = obj.find(instance_id) if instance_id
+
+          msg_timeout = 600 # TODO: configurable per message
+          result = Timeout.timeout(msg_timeout) do
+            obj.send(options[:method_name], *args)
+          end
+
+          run_job(miq_callback) if miq_callback
+          result
+        end
+      end
+    end
+  end
+end

data/lib/manageiq/messaging/stomp/client.rb

@@ -0,0 +1,85 @@
+module ManageIQ
+  module Messaging
+    module Stomp
+      # Messaging client implementation using Stomp protocol with ActiveMQ Artemis being
+      # the underlying supporting system.
+      # Do not directly instantiate an instance from this class. Use
+      # +ManageIQ::Messaging::Client.open+ method.
+      #
+      # Artemis specific connection options accepted by +open+ method:
+      # * :client_ref (A reference string to identify the client)
+      # * :host (Single host name)
+      # * :port (host port number)
+      # * :username
+      # * :password
+      # * :heartbeat (Whether the client should do heart-beating. Default to true)
+      #
+      # Artemis specific +publish_message+ options:
+      # * :expires_on
+      # * :deliver_on
+      # * :priority
+      # * :group_name
+      #
+      # Artemis specific +publish_topic+ options:
+      # * :expires_on
+      # * :deliver_on
+      # * :priority
+      #
+      # Artemis specific +subscribe_topic+ options:
+      # * :persist_ref
+      #
+      # +:persist_ref+ must be paired with +:client_ref+ option in +Client.open+ method.
+      # They jointly create a unique group name. Without such group every topic subscriber
+      # receives a copy of each message only when they are active. This is the default.
+      # If multiple topic subscribers join with the same group each message is consumed
+      # by only one of the subscribers. This allows a load balancing among the subscribers.
+      # Also any messages sent when all members of the group are offline will be persisted
+      # and delivered when any member in the group is back online. Each message is still
+      # copied and delivered to other subscribes belongs to other groups or no group.
+      #
+      # Artemis specific +subscribe_messages+ options:
+      # * :limit ()
+      class Client < ManageIQ::Messaging::Client
+        require 'stomp'
+        require 'manageiq/messaging/stomp/common'
+        require 'manageiq/messaging/stomp/queue'
+        require 'manageiq/messaging/stomp/background_job'
+        require 'manageiq/messaging/stomp/topic'
+
+        include Common
+        include Queue
+        include BackgroundJob
+        include Topic
+
+        private *delegate(:subscribe, :unsubscribe, :publish, :to => :stomp_client)
+        delegate :ack, :close, :to => :stomp_client
+
+        attr_accessor :encoding
+
+        private
+
+        attr_reader :stomp_client
+
+        def initialize(options)
+          host = options.slice(:host, :port)
+          host[:passcode] = options[:password] if options[:password]
+          host[:login] = options[:username] if options[:username]
+
+          headers = {}
+          if options[:heartbeat].nil? || options[:heartbeat]
+            headers.merge!(
+              :host             => options[:host],
+              :"accept-version" => "1.2",
+              :"heart-beat"     => "2000,0"
+            )
+          end
+          headers[:"client-id"] = options[:client_ref] if options[:client_ref]
+
+          @encoding = options[:encoding] || 'yaml'
+          require "json" if @encoding == "json"
+          @stomp_client = ::Stomp::Client.new(:hosts => [host], :connect_headers => headers)
+        end
+      end
+    end
+  end
+end

data/lib/manageiq/messaging/stomp/common.rb

@@ -0,0 +1,84 @@
+module ManageIQ
+  module Messaging
+    module Stomp
+      module Common
+        require 'manageiq/messaging/common'
+        include ManageIQ::Messaging::Common
+
+        private
+
+        def raw_publish(address, body, headers)
+          publish(address, encode_body(headers, body), headers)
+          logger.info("Published to address(#{address}), msg(#{payload_log(body.inspect)}), headers(#{headers.inspect})")
+        end
+
+        def queue_for_publish(options)
+          affinity = options[:affinity] || 'none'
+          address = "queue/#{options[:service]}.#{affinity}"
+
+          headers = {:"destination-type" => 'ANYCAST', :persistent => true}
+          headers[:expires]            = options[:expires_on].to_i * 1000 if options[:expires_on]
+          headers[:AMQ_SCHEDULED_TIME] = options[:deliver_on].to_i * 1000 if options[:deliver_on]
+          headers[:priority]           = options[:priority] if options[:priority]
+          headers[:_AMQ_GROUP_ID]      = options[:group_name] if options[:group_name]
+
+          [address, headers]
+        end
+
+        def queue_for_subscribe(options)
+          affinity = options[:affinity] || 'none'
+          queue_name = "queue/#{options[:service]}.#{affinity}"
+
+          headers = {:"subscription-type" => 'ANYCAST', :ack => 'client'}
+
+          [queue_name, headers]
+        end
+
+        def topic_for_publish(options)
+          address = "topic/#{options[:service]}"
+
+          headers = {:"destination-type" => 'MULTICAST', :persistent => true}
+          headers[:expires]            = options[:expires_on].to_i * 1000 if options[:expires_on]
+          headers[:AMQ_SCHEDULED_TIME] = options[:deliver_on].to_i * 1000 if options[:deliver_on]
+          headers[:priority]           = options[:priority] if options[:priority]
+
+          [address, headers]
+        end
+
+        def topic_for_subscribe(options)
+          queue_name = "topic/#{options[:service]}"
+
+          headers = {:"subscription-type" => 'MULTICAST', :ack => 'client'}
+          headers[:"durable-subscription-name"] = options[:persist_ref] if options[:persist_ref]
+
+          [queue_name, headers]
+        end
+
+        def send_response(service, correlation_ref, result)
+          response_options = {
+            :service  => "#{service}.response",
+            :affinity => correlation_ref
+          }
+          address, response_headers = queue_for_publish(response_options)
+          raw_publish(address, result || '', response_headers.merge(:correlation_id => correlation_ref))
+        end
+
+        def receive_response(service, correlation_ref)
+          response_options = {
+            :service  => "#{service}.response",
+            :affinity => correlation_ref
+          }
+          queue_name, response_headers = queue_for_subscribe(response_options)
+          subscribe(queue_name, response_headers) do |msg|
+            ack(msg)
+            begin
+              yield decode_body(msg.headers, msg.body)
+            ensure
+              unsubscribe(queue_name)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/manageiq/messaging/stomp/queue.rb

@@ -0,0 +1,53 @@
+module ManageIQ
+  module Messaging
+    module Stomp
+      module Queue
+        private
+
+        def publish_message_impl(options, &block)
+          address, headers = queue_for_publish(options)
+          headers[:sender]         = options[:sender] if options[:sender]
+          headers[:message_type]   = options[:message] if options[:message]
+          headers[:class_name]     = options[:class_name] if options[:class_name]
+          headers[:correlation_id] = Time.now.to_i.to_s if block_given?
+
+          raw_publish(address, options[:payload] || '', headers)
+
+          return unless block_given?
+
+          receive_response(options[:service], headers[:correlation_id], &block)
+        end
+
+        def publish_messages_impl(messages)
+          messages.each { |msg_options| publish_message(msg_options) }
+        end
+
+        def subscribe_messages_impl(options)
+          queue_name, headers = queue_for_subscribe(options)
+
+          # for STOMP we can get message one at a time
+          subscribe(queue_name, headers) do |msg|
+            begin
+              sender = msg.headers['sender']
+              message_type = msg.headers['message_type']
+              message_body = decode_body(msg.headers, msg.body)
+              logger.info("Message received: queue(#{queue_name}), msg(#{payload_log(message_body)}), headers(#{msg.headers})")
+
+              result = yield [ManageIQ::Messaging::ReceivedMessage.new(sender, message_type, message_body, msg)]
+              logger.info("Message processed")
+
+              correlation_ref = msg.headers['correlation_id']
+              if correlation_ref
+                result = result.first if result.kind_of?(Array)
+                send_response(options[:service], correlation_ref, result)
+              end
+            rescue => e
+              logger.error("Message processing error: #{e.message}")
+              logger.error(e.backtrace.join("\n"))
+            end
+          end
+        end
+      end
+    end
+  end
+end