manageiq-messaging 0.1.3 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8859fa0a68ffe54c38283145348077bcbc78e004636e38960cc0d2ff70923f7a
-  data.tar.gz: '088f6d9d2dc80e753b16a0bbdce58dc789aec1d5eada50c0aa3b74fd78109006'
+  metadata.gz: 503935968b3f9866a7c638f6f5902f4a4fff7bcacab799b12cce939f2e9922c8
+  data.tar.gz: d1681ca75ff3c5500a3f8d0a68dd622d8dff60ae7e7bdc19f1a54dbe51a01e24
 SHA512:
-  metadata.gz: 457f22d4a96c0ec19a05cb2a0ebe3a9f665f62583d7337ede95ddf786e899b78dcd86173c219988d34ef8d4deafc188d8bcd184728e8362e4b66c772875370f5
-  data.tar.gz: b496df16b18b87754bf22107fe0d7f30a2dee767f6673cdd51330880ea290539967397b32f629cb4ff24fb24e428390ba9478cc0c2bbc359c404e0a5cd358976
+  metadata.gz: 4717a96bfdb879318756f037baaa00030e9c1f76a33a04bbe276a0296c2490a9b2fd5abfba6c5fa21ea15fc623659c5987767b01c4e1a95f363f6b307f421829
+  data.tar.gz: e26d52f01f2e871cdd99b377e49b6d5914584506f65df11bedd325b56ba794607ffaac033fb837a75ffb6b40722723e144458da6842777c0d152311826cf7d10

data/.codeclimate.yml

@@ -34,6 +34,7 @@ engines:
   rubocop:
     enabled: true
     config: '.rubocop_cc.yml'
+    channel: 'rubocop-0-69'
 prepare:
   fetch:
   - url: "https://raw.githubusercontent.com/ManageIQ/guides/master/.rubocop_base.yml"

data/.rubocop.yml

@@ -1,4 +1,5 @@
+inherit_gem:
+  manageiq-style: .rubocop_base.yml
 inherit_from:
-- https://raw.githubusercontent.com/ManageIQ/guides/master/.rubocop_base.yml
 # put all local rubocop config into .rubocop_local.yml as it will be loaded by .rubocop_cc.yml as well
 - .rubocop_local.yml

data/.travis.yml

@@ -1,9 +1,8 @@
-sudo: false
 language: ruby
 rvm:
-  - 2.3.1
   - 2.4.5
+  - 2.5.7
+  - 2.6.5
 before_install: gem install bundler -v 1.13.0
 after_script: bundle exec codeclimate-test-reporter
-sudo: false
 cache: bundler

data/CHANGES

@@ -7,3 +7,21 @@
 
 = 0.1.2 - 11-Dec-2018
 * Allow to set max_bytes to each_batch when subscribe to a Kafka topic.
+
+= 0.1.4 - 3-Apr-2019
+* Add an #ack method to a ReceivedMessage to simplify manual acknowledgements
+* Allow caller to provide a session_timeout to kafka consumers, default of 30sec
+
+= 0.1.5 - 6-Jun-2019
+* Allow caller to provide extra headers to the message
+
+= 0.1.6 - 6-July-2020
+* Rescue message body decoding errors. Re-raise errors raised by users code of processing received messages. 
+
+= 1.0.0 - 28-Sep-2020
+* Switch to use rdkafka client
+
+= 1.0.1 - 25-Dec-2020
+* Allow all kafka options to be passed
+* remove sudo:false from .travis.yml
+* Use manageiq-style

data/README.md

@@ -1,10 +1,9 @@
 # ManageIQ Messaging Client
 
 [![Gem Version](https://badge.fury.io/rb/manageiq-messaging.svg)](http://badge.fury.io/rb/manageiq-messaging)
-[![Build Status](https://travis-ci.org/ManageIQ/manageiq-messaging.svg)](https://travis-ci.org/ManageIQ/manageiq-messaging)
+[![Build Status](https://travis-ci.com/ManageIQ/manageiq-messaging.svg)](https://travis-ci.com/ManageIQ/manageiq-messaging)
 [![Code Climate](https://codeclimate.com/github/ManageIQ/manageiq-messaging.svg)](https://codeclimate.com/github/ManageIQ/manageiq-messaging)
 [![Test Coverage](https://codeclimate.com/github/ManageIQ/manageiq-messaging/badges/coverage.svg)](https://codeclimate.com/github/ManageIQ/manageiq-messaging/coverage)
-[![Dependency Status](https://gemnasium.com/ManageIQ/manageiq-messaging.svg)](https://gemnasium.com/ManageIQ/manageiq-messaging)
 [![Security](https://hakiri.io/github/ManageIQ/manageiq-messaging/master.svg)](https://hakiri.io/github/ManageIQ/manageiq-messaging/master)
 
 Client library for ManageIQ components to exchange messages through its internal message bus.
@@ -83,7 +82,7 @@ This is the one-to-one publish/subscribe pattern. Multiple subscribers can subsc
   client.subscribe_messages(:service => 'ems_operation', :affinity => 'ems_amazon1', :auto_ack => false) do |messages|
     messages.each do |msg|
       # do stuff with msg.message and msg.payload
-      client.ack(msg.ack_ref)
+      msg.ack
     end
   end
 
@@ -156,6 +155,24 @@ This is the one-to-many publish/subscribe pattern. Multiple subscribers can subs
 
 By default, events are delivered to live subscribers only. Some messaging systems support persistence with options.
 
+### Add your own headers to a message (Queue or Topic)
+
+If you want you can add in your own headers to the send message
+
+```
+  client.publish_topic(
+    :service => 'provider_events',
+    :event   => 'powered_on',
+    :headers => {:request_id => "12345"},
+    :payload => {:ems_ref => 'uid987'}
+
+  )
+
+  client.subscribe_topic(:service => 'provider_events', :persist_ref => 'automate_1') do |msg|
+    puts "Received event #{msg.message} with request-id: #{msg.headers['request_id']}"
+  end
+```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/examples/message.rb

@@ -26,7 +26,7 @@ class ProducerConsumer < Common
       client.subscribe_messages(:service => 'ems_operation', :affinity => 'ems_amazon1', :auto_ack => false) do |messages|
         messages.each do |msg|
           do_stuff(msg)
-          client.ack(msg.ack_ref)
+          msg.ack
         end
       end
       sleep(5)

data/lib/manageiq/messaging/client.rb

@@ -40,6 +40,7 @@ module ManageIQ
       def self.open(options)
         protocol = options[:protocol] || :Stomp
         client = Object.const_get("ManageIQ::Messaging::#{protocol}::Client").new(options)
+
         return client unless block_given?
 
         begin
@@ -58,6 +59,7 @@ module ManageIQ
       # * :message    (e.g. method name or message type)
       # * :payload    (message body, a string or an user object that can be serialized)
       # * :sender     (optional, identify the sender)
+      # * :headers    (optional, additional headers to add to the message)
       # Other options are underlying messaging system specific.
       #
       # Optionally a call back block can be provided to wait on the consumer to send
@@ -106,9 +108,9 @@ module ManageIQ
       #       msg.sender
       #       msg.message
       #       msg.payload
-      #       msg.ack_ref #used to ack the message
+      #       msg.ack_ref
       #
-      #       client.ack(msg.ack_ref) # needed only when options[:auto_ack] is false
+      #       msg.ack # needed only when options[:auto_ack] is false
       #       # process the message
       #     end
       #   end
@@ -122,7 +124,7 @@ module ManageIQ
       # message is proccessed. Any un-acked message will be redelivered to next subscriber
       # AFTER the current subscriber disconnects normally or abnormally (e.g. crashed).
       #
-      # To ack a message call +ack+(+msg.ack_ref+)
+      # To ack a message call +msg.ack+
       def subscribe_messages(options, &block)
         raise "A block is required" unless block_given?
         assert_options(options, [:service])
@@ -164,6 +166,7 @@ module ManageIQ
       # * :event   (event name)
       # * :payload (message body, a string or an user object that can be serialized)
       # * :sender  (optional, identify the sender)
+      # * :headers (optional, additional headers to add to the message)
       # Other options are underlying messaging system specific.
       #
       def publish_topic(options)
@@ -189,9 +192,9 @@ module ManageIQ
       #     msg.sender
       #     msg.message
       #     msg.payload
-      #     msg.ack_ref #used to ack the message
+      #     msg.ack_ref
       #
-      #     client.ack(msg.ack_ref) # needed only when options[:auto_ack] is false
+      #     msg.ack # needed only when options[:auto_ack] is false
       #     # process the message
       #   end
       #
@@ -204,7 +207,7 @@ module ManageIQ
       # message is proccessed. Any un-acked message will be redelivered to next subscriber
       # AFTER the current subscriber disconnects normally or abnormally (e.g. crashed).
       #
-      # To ack a message call +ack+(+msg.ack_ref+)
+      # To ack a message call +msg.ack+
       def subscribe_topic(options, &block)
         raise "A block is required" unless block_given?
         assert_options(options, [:service])

data/lib/manageiq/messaging/common.rb

@@ -18,7 +18,7 @@ module ManageIQ
 
       def decode_body(headers, raw_body)
         return raw_body unless headers.kind_of?(Hash)
-        case headers["encoding"]
+        case headers["encoding"] || headers[:encoding]
         when "json"
           JSON.parse(raw_body)
         when "yaml"
@@ -26,6 +26,10 @@ module ManageIQ
         else
           raw_body
         end
+      rescue => e # JSON or YAML parsing error
+        logger.error("Error decoding message body: #{e.message}")
+        logger.error(e.backtrace.join("\n"))
+        raw_body
       end
 
       def payload_log(payload)

data/lib/manageiq/messaging/kafka/client.rb

@@ -10,25 +10,22 @@ module ManageIQ
       # * :hosts (Array of Kafka cluster hosts, or)
       # * :host (Single host name)
       # * :port (host port number)
-      # * :ssl_ca_cert (security options)
-      # * :ssl_client_cert
-      # * :ssl_client_cert_key
-      # * :sasl_gssapi_principal
-      # * :sasl_gssapi_keytab
-      # * :sasl_plain_username
-      # * :sasl_plain_password
-      # * :sasl_scram_username
-      # * :sasl_scram_password
-      # * :sasl_scram_mechanism
+      #
+      # For additional security options, please refer to 
+      # https://github.com/edenhill/librdkafka/wiki/Using-SSL-with-librdkafka and
+      # https://github.com/edenhill/librdkafka/wiki/Using-SASL-with-librdkafka 
+      # 
       #
       # Kafka specific +publish_message+ options:
       # * :group_name (Used as Kafka partition_key)
       #
       # Kafka specific +subscribe_topic+ options:
       # * :persist_ref (Used as Kafka group_id)
+      # * :session_timeout (Max time in seconds allowed to process a message, default is 30)
       #
       # Kafka specific +subscribe_messages+ options:
       # * :max_bytes (Max batch size to read, default is 10Mb)
+      # * :session_timeout (Max time in seconds allowed to process a message, default is 30)
       #
       # Without +:persist_ref+ every topic subscriber receives a copy of each message
       # only when they are active. If multiple topic subscribers join with the same
@@ -40,7 +37,7 @@ module ManageIQ
       #
       # +subscribe_background_job+ is currently not implemented.
       class Client < ManageIQ::Messaging::Client
-        require 'kafka'
+        require 'rdkafka'
         require 'manageiq/messaging/kafka/common'
         require 'manageiq/messaging/kafka/queue'
         require 'manageiq/messaging/kafka/background_job'
@@ -51,27 +48,27 @@ module ManageIQ
         include BackgroundJob
         include Topic
 
-        private *delegate(:subscribe, :unsubscribe, :publish, :to => :kafka_client)
-        delegate :close, :to => :kafka_client
-
         attr_accessor :encoding
 
         def ack(ack_ref)
-          @queue_consumer.try(:mark_message_as_processed, ack_ref)
-          @topic_consumer.try(:mark_message_as_processed, ack_ref)
+          ack_ref.commit
+        rescue Rdkafka::RdkafkaError => e
+          logger.warn("ack failed with error #{e.message}")
+          raise unless e.message =~ /no_offset/
         end
 
         def close
-          @topic_consumer.try(:stop)
-          @topic_consumer = nil
-          @queue_consumer.try(:stop)
-          @queue_consumer = nil
-
-          @producer.try(:shutdown)
+          @producer&.close
           @producer = nil
 
-          kafka_client.close
-          @kafka_client = nil
+          @consumer&.close
+          @consumer = nil
+        end
+
+        # list all topics
+        def topics
+          native_kafka = producer.instance_variable_get(:@native_kafka)
+          Rdkafka::Metadata.new(native_kafka).topics.collect { |topic| topic[:topic_name] }
         end
 
         private
@@ -85,12 +82,12 @@ module ManageIQ
           @encoding = options[:encoding] || 'yaml'
           require "json" if @encoding == "json"
 
-          connection_opts = {}
-          connection_opts[:client_id] = options[:client_ref] if options[:client_ref]
-
-          connection_opts.merge!(options.slice(:ssl_ca_cert, :ssl_client_cert, :ssl_client_cert_key, :sasl_gssapi_principal, :sasl_gssapi_keytab, :sasl_plain_username, :sasl_plain_password, :sasl_scram_username, :sasl_scram_password, :sasl_scram_mechanism))
+          connection_opts = {:"bootstrap.servers" => hosts.join(',')}
+          connection_opts[:"client.id"] = options[:client_ref] if options[:client_ref]
+          connection_opts.merge!(options.except(:port, :host, :hosts, :encoding, :protocol, :client_ref))
 
-          @kafka_client = ::Kafka.new(hosts, connection_opts)
+          ::Rdkafka::Config.logger = logger
+          @kafka_client = ::Rdkafka::Config.new(connection_opts)
         end
       end
     end

data/lib/manageiq/messaging/kafka/common.rb

@@ -5,46 +5,35 @@ module ManageIQ
         require 'manageiq/messaging/common'
         include ManageIQ::Messaging::Common
 
-        GROUP_FOR_QUEUE_MESSAGES = 'manageiq_messaging_queue_group_'.freeze
-
         private
 
         def producer
           @producer ||= kafka_client.producer
         end
 
-        def topic_consumer(persist_ref)
-          # persist_ref enables consumer to receive messages sent when consumer is temporarily offline
-          # it also enables consumers to do load balancing when multiple consumers join the with the same ref.
-          @topic_consumer.try(:stop) unless @persist_ref == persist_ref
-          @persist_ref = persist_ref
-          @topic_consumer ||= kafka_client.consumer(:group_id => persist_ref)
-        end
-
-        def queue_consumer(topic)
-          # all queue consumers join the same group so that each message can be processed by one and only one consumer
-          @queue_consumer.try(:stop) unless @queue_topic == topic
-          @queue_topic = topic
-          @queue_consumer ||= kafka_client.consumer(:group_id => GROUP_FOR_QUEUE_MESSAGES + topic)
-        end
-
-        trap("TERM") do
-          @topic_consumer.try(:stop)
-          @topic_consumer = nil
-          @queue_consumer.try(:stop)
-          @queue_consumer = nil
+        def consumer(beginning, options)
+          @consumer&.close
+          kafka_client[:"group.id"] = options[:persist_ref]
+          kafka_client[:"auto.offset.reset"] = beginning ? 'smallest' : 'largest'
+          kafka_client[:"enable.auto.commit"] = !!auto_ack?(options)
+          kafka_client[:"session.timeout.ms"] = options[:session_timeout] * 1000 if options[:session_timeout].present?
+          kafka_client[:"group.instance.id"] = options[:group_instance_id] if options[:group_instance_id].present?
+          @consumer = kafka_client.consumer
         end
 
-        def raw_publish(commit, body, options)
-          producer.produce(encode_body(options[:headers], body), options)
-          producer.deliver_messages if commit
-          logger.info("Published to topic(#{options[:topic]}), msg(#{payload_log(body.inspect)})")
+        def raw_publish(wait, body, options)
+          options[:payload] = encode_body(options[:headers], body)
+          producer.produce(options).tap do |handle|
+            handle.wait if wait
+            logger.info("Published to topic(#{options[:topic]}), msg(#{payload_log(body.inspect)})")
+          end
         end
 
         def queue_for_publish(options)
           body, kafka_opts = for_publish(options)
           kafka_opts[:headers][:message_type] = options[:message] if options[:message]
           kafka_opts[:headers][:class_name] = options[:class_name] if options[:class_name]
+          kafka_opts[:headers].merge!(options[:headers].except(*message_header_keys)) if options.key?(:headers)
 
           [body, kafka_opts]
         end
@@ -52,13 +41,15 @@ module ManageIQ
         def topic_for_publish(options)
           body, kafka_opts = for_publish(options)
           kafka_opts[:headers][:event_type] = options[:event] if options[:event]
+          kafka_opts[:headers].merge!(options[:headers].except(*event_header_keys)) if options.key?(:headers)
 
           [body, kafka_opts]
         end
 
         def for_publish(options)
-          kafka_opts = {:topic => address(options), :headers => {}}
+          kafka_opts = {:topic => address(options)}
           kafka_opts[:partition_key] = options[:group_name] if options[:group_name]
+          kafka_opts[:headers] = {}
           kafka_opts[:headers][:sender] = options[:sender] if options[:sender]
 
           body = options[:payload] || ''
@@ -74,34 +65,54 @@ module ManageIQ
           end
         end
 
-        def process_queue_message(queue, message)
-          payload = decode_body(message.headers, message.value)
-          sender, message_type, class_name = parse_message_headers(message.headers)
-          logger.info("Message received: queue(#{queue}), message(#{payload_log(payload)}), sender(#{sender}), type(#{message_type})")
-          [sender, message_type, class_name, payload]
+        def process_queue_message(queue_consumer, queue, message)
+          begin
+            payload = decode_body(message.headers, message.payload)
+            sender, message_type, _class_name = parse_message_headers(message.headers)
+            client_headers = message.headers.except(*message_header_keys).with_indifferent_access
+
+            logger.info("Message received: queue(#{queue}), message(#{payload_log(payload)}), sender(#{sender}), type(#{message_type})")
+            yield [ManageIQ::Messaging::ReceivedMessage.new(sender, message_type, payload, client_headers, queue_consumer, self)]
+            logger.info("Messsage processed")
+          rescue StandardError => e
+            logger.error("Message processing error: #{e.message}")
+            logger.error(e.backtrace.join("\n"))
+            raise
+          end
         end
 
-        def process_topic_message(topic, message)
+        def process_topic_message(topic_consumer, topic, message)
           begin
-            payload = decode_body(message.headers, message.value)
+            payload = decode_body(message.headers, message.payload)
             sender, event_type = parse_event_headers(message.headers)
+            client_headers = message.headers.except(*event_header_keys).with_indifferent_access
+
             logger.info("Event received: topic(#{topic}), event(#{payload_log(payload)}), sender(#{sender}), type(#{event_type})")
-            yield ManageIQ::Messaging::ReceivedMessage.new(sender, event_type, payload, message)
+            yield ManageIQ::Messaging::ReceivedMessage.new(sender, event_type, payload, client_headers, topic_consumer, self)
             logger.info("Event processed")
           rescue StandardError => e
             logger.error("Event processing error: #{e.message}")
             logger.error(e.backtrace.join("\n"))
+            raise
           end
         end
 
+        def message_header_keys
+          [:sender, :message_type, :class_name]
+        end
+
         def parse_message_headers(headers)
           return [nil, nil, nil] unless headers.kind_of?(Hash)
-          headers.values_at('sender', 'message_type', 'class_name')
+          headers.values_at(*message_header_keys)
+        end
+
+        def event_header_keys
+          [:sender, :event_type]
         end
 
         def parse_event_headers(headers)
           return [nil, nil] unless headers.kind_of?(Hash)
-          headers.values_at('sender', 'event_type')
+          headers.values_at(*event_header_keys)
         end
       end
     end

data/lib/manageiq/messaging/kafka/queue.rb

@@ -2,6 +2,8 @@ module ManageIQ
   module Messaging
     module Kafka
       module Queue
+        GROUP_FOR_QUEUE_MESSAGES = ENV['QUEUE_MESSAGES_GROUP_PREFIX'].freeze || 'manageiq_messaging_queue_group_'.freeze
+
         private
 
         def publish_message_impl(options)
@@ -10,33 +12,18 @@ module ManageIQ
         end
 
         def publish_messages_impl(messages)
-          messages.each { |msg_options| raw_publish(false, *queue_for_publish(msg_options)) }
-          producer.deliver_messages
+          handles = messages.collect { |msg_options| raw_publish(false, *queue_for_publish(msg_options)) }
+          handles.each(&:wait)
         end
 
-        def subscribe_messages_impl(options)
+        def subscribe_messages_impl(options, &block)
           topic = address(options)
+          options[:persist_ref] = GROUP_FOR_QUEUE_MESSAGES + topic
 
-          batch_options = {}
-          batch_options[:automatically_mark_as_processed] = auto_ack?(options)
-          batch_options[:max_bytes] = options[:max_bytes] if options.key?(:max_bytes)
-
-          consumer = queue_consumer(topic)
-          consumer.subscribe(topic)
-          consumer.each_batch(batch_options) do |batch|
-            logger.info("Batch message received: queue(#{topic})")
-            begin
-              messages = batch.messages.collect do |message|
-                sender, message_type, _class_name, payload = process_queue_message(topic, message)
-                ManageIQ::Messaging::ReceivedMessage.new(sender, message_type, payload, message)
-              end
-
-              yield messages
-            rescue StandardError => e
-              logger.error("Event processing error: #{e.message}")
-              logger.error(e.backtrace.join("\n"))
-            end
-            logger.info("Batch message processed")
+          queue_consumer = consumer(true, options)
+          queue_consumer.subscribe(topic)
+          queue_consumer.each do |message|
+            process_queue_message(queue_consumer, topic, message, &block)
           end
         end
       end

data/lib/manageiq/messaging/kafka/topic.rb

@@ -1,7 +1,11 @@
+require 'socket'
+
 module ManageIQ
   module Messaging
     module Kafka
       module Topic
+        GROUP_FOR_ADHOC_LISTENERS = Socket.gethostname.freeze
+
         private
 
         def publish_topic_impl(options)
@@ -10,18 +14,12 @@ module ManageIQ
 
         def subscribe_topic_impl(options, &block)
           topic = address(options)
-          persist_ref = options[:persist_ref]
 
-          if persist_ref
-            consumer = topic_consumer(persist_ref)
-            consumer.subscribe(topic, :start_from_beginning => false)
-            consumer.each_message(:automatically_mark_as_processed => auto_ack?(options)) do |message|
-              process_topic_message(topic, message, &block)
-            end
-          else
-            kafka_client.each_message(:topic => topic, :start_from_beginning => false) do |message|
-              process_topic_message(topic, message, &block)
-            end
+          options[:persist_ref] = "#{GROUP_FOR_ADHOC_LISTENERS}_#{Time.now.to_i}" unless options[:persist_ref]
+          topic_consumer = consumer(false, options)
+          topic_consumer.subscribe(topic)
+          topic_consumer.each do |message|
+            process_topic_message(topic_consumer, topic, message, &block)
           end
         end
       end

data/lib/manageiq/messaging/received_message.rb

@@ -1,10 +1,14 @@
 module ManageIQ
   module Messaging
     class ReceivedMessage
-      attr_accessor :sender, :message, :payload, :ack_ref
+      attr_accessor :sender, :message, :payload, :headers, :ack_ref, :client
 
-      def initialize(sender, message, payload, ack_ref)
-        @sender, @message, @payload, @ack_ref = sender, message, payload, ack_ref
+      def initialize(sender, message, payload, headers, ack_ref, client)
+        @sender, @message, @payload, @headers, @ack_ref, @client = sender, message, payload, headers, ack_ref, client
+      end
+
+      def ack
+        client.ack(ack_ref)
       end
     end
   end

data/lib/manageiq/messaging/stomp/common.rb

@@ -17,6 +17,8 @@ module ManageIQ
           address = "queue/#{options[:service]}.#{affinity}"
 
           headers = {:"destination-type" => 'ANYCAST', :persistent => true}
+          headers.merge!(options[:headers].except(*internal_header_keys)) if options.key?(:headers)
+
           headers[:expires]            = options[:expires_on].to_i * 1000 if options[:expires_on]
           headers[:AMQ_SCHEDULED_TIME] = options[:deliver_on].to_i * 1000 if options[:deliver_on]
           headers[:priority]           = options[:priority] if options[:priority]
@@ -38,6 +40,8 @@ module ManageIQ
           address = "topic/#{options[:service]}"
 
           headers = {:"destination-type" => 'MULTICAST', :persistent => true}
+          headers.merge!(options[:headers].except(*internal_header_keys)) if options.key?(:headers)
+
           headers[:expires]            = options[:expires_on].to_i * 1000 if options[:expires_on]
           headers[:AMQ_SCHEDULED_TIME] = options[:deliver_on].to_i * 1000 if options[:deliver_on]
           headers[:priority]           = options[:priority] if options[:priority]
@@ -63,6 +67,10 @@ module ManageIQ
           raw_publish(address, result || '', response_headers.merge(:correlation_id => correlation_ref))
         end
 
+        def internal_header_keys
+          [:"destination-type", :persistent, :expires, :AMQ_SCHEDULED_TIME, :priority, :_AMQ_GROUP_ID]
+        end
+
         def receive_response(service, correlation_ref)
           response_options = {
             :service  => "#{service}.response",

data/lib/manageiq/messaging/stomp/queue.rb

@@ -33,9 +33,15 @@ module ManageIQ
               sender = msg.headers['sender']
               message_type = msg.headers['message_type']
               message_body = decode_body(msg.headers, msg.body)
+              client_headers = msg.headers.except(*internal_header_keys)
+
               logger.info("Message received: queue(#{queue_name}), msg(#{payload_log(message_body)}), headers(#{msg.headers})")
 
-              result = yield [ManageIQ::Messaging::ReceivedMessage.new(sender, message_type, message_body, msg)]
+              messages = [
+                ManageIQ::Messaging::ReceivedMessage.new(sender, message_type, message_body, client_headers, msg, self)
+              ]
+
+              result = yield messages
               logger.info("Message processed")
 
               correlation_ref = msg.headers['correlation_id']
@@ -46,6 +52,7 @@ module ManageIQ
             rescue => e
               logger.error("Message processing error: #{e.message}")
               logger.error(e.backtrace.join("\n"))
+              raise
             end
           end
         end

data/lib/manageiq/messaging/stomp/topic.rb

@@ -21,13 +21,16 @@ module ManageIQ
 
               sender = event.headers['sender']
               event_type = event.headers['event_type']
+              client_headers = event.headers.except(*internal_header_keys)
+
               event_body = decode_body(event.headers, event.body)
               logger.info("Event received: queue(#{queue_name}), event(#{event_body}), headers(#{event.headers})")
-              yield ManageIQ::Messaging::ReceivedMessage.new(sender, event_type, event_body, event)
+              yield ManageIQ::Messaging::ReceivedMessage.new(sender, event_type, event_body, client_headers, event, self)
               logger.info("Event processed")
             rescue => e
               logger.error("Event processing error: #{e.message}")
               logger.error(e.backtrace.join("\n"))
+              raise
             end
           end
         end

data/lib/manageiq/messaging/version.rb

@@ -1,5 +1,5 @@
 module ManageIQ
   module Messaging
-    VERSION = "0.1.3"
+    VERSION = "1.0.1"
   end
 end

data/manageiq-messaging.gemspec

@@ -4,9 +4,10 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'manageiq/messaging/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "manageiq-messaging"
-  spec.version       = ManageIQ::Messaging::VERSION
-  spec.authors       = ["ManageIQ Authors"]
+  spec.name                  = "manageiq-messaging"
+  spec.version               = ManageIQ::Messaging::VERSION
+  spec.required_ruby_version = '>= 2.4'
+  spec.authors               = ["ManageIQ Authors"]
 
   spec.summary       = 'Client library for ManageIQ components to exchange messages through its internal message bus.'
   spec.description   = 'Client library for ManageIQ components to exchange messages through its internal message bus.'
@@ -20,14 +21,14 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency 'activesupport', '>= 4.2.2'
-  spec.add_dependency 'ruby-kafka', '~> 0.7.0'
+  spec.add_dependency 'activesupport', '~> 5.2.4.3'
+  spec.add_dependency 'rdkafka', '~> 0.8'
   spec.add_dependency 'stomp', '~> 1.4.4'
 
-  spec.add_development_dependency "bundler", "~> 1.13"
+  spec.add_development_dependency "bundler"
   spec.add_development_dependency "codeclimate-test-reporter", "~> 1.0.0"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "manageiq-style"
+  spec.add_development_dependency "rake", ">= 12.3.3"
   spec.add_development_dependency "rspec", "~> 3.0"
-  spec.add_development_dependency "rubocop"
   spec.add_development_dependency "simplecov"
 end

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: manageiq-messaging
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 1.0.1
 platform: ruby
 authors:
 - ManageIQ Authors
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2019-02-25 00:00:00.000000000 Z
+date: 2020-12-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.2.2
+        version: 5.2.4.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.2.2
+        version: 5.2.4.3
 - !ruby/object:Gem::Dependency
-  name: ruby-kafka
+  name: rdkafka
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.7.0
+        version: '0.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.7.0
+        version: '0.8'
 - !ruby/object:Gem::Dependency
   name: stomp
   requirement: !ruby/object:Gem::Requirement
@@ -56,16 +56,16 @@ dependencies:
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.13'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.13'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: codeclimate-test-reporter
   requirement: !ruby/object:Gem::Requirement
@@ -81,47 +81,47 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 1.0.0
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: manageiq-style
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
-  name: rubocop
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -138,7 +138,7 @@ dependencies:
         version: '0'
 description: Client library for ManageIQ components to exchange messages through its
   internal message bus.
-email: 
+email:
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -186,7 +186,7 @@ homepage: http://github.com/ManageIQ/manageiq-messaging
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -194,16 +194,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
-signing_key: 
+rubygems_version: 3.2.0.rc.2
+signing_key:
 specification_version: 4
 summary: Client library for ManageIQ components to exchange messages through its internal
   message bus.