manageiq-messaging 0.1.6 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 31a6d45fedb5996e9ab5a270aeacd53a1683871abbb60294ae1aee500d35b972
-  data.tar.gz: a530afba35d2d6ff1f754d8d263c46b1b042322743d8b24effd0f81fab30e2c8
+  metadata.gz: aa6536d137d3873d7b73cecfc02add52d949bf4c240846b9df4c0a46b02be560
+  data.tar.gz: 240ffe6f8dacc50a80759335f4d748dc826675cf38e0567117809582580489a0
 SHA512:
-  metadata.gz: 59ac72471bd2324d150874c34f35eb26b31dd2175f45ae6eb10073047e50417aa37d66604326367b45dac685b1c69a84f94dcf679bad2fcb08349bc447573f17
-  data.tar.gz: '0497bacb543ed6e4cbf4b44fa6db86e35c59fff88e1338a259361d06a18df7438f8a494d23ca775df3b0f97344eac6e4c4738e4c12a7b7c49c8d188ac18c4c69'
+  metadata.gz: 45b6e6592e772ffc7373489116a72403d5b6fb2a2e322b4d5d6c629ee6eddd575c3eaf1e7d92afe504b361eaaaa46047e0d85b8ed640272d41098b5d185d8a45
+  data.tar.gz: e95db7f79892be46eb800e4f241b932282cdebf2f4ffb30a65bfea8a85f24161b9fdc5afbca4d182170aa7fb33f85edcf59881b51a8be80db868bf65bbf6da5b

data/.travis.yml

@@ -1,8 +1,9 @@
 sudo: false
 language: ruby
 rvm:
-  - 2.3.1
   - 2.4.5
+  - 2.5.7
+  - 2.6.5
 before_install: gem install bundler -v 1.13.0
 after_script: bundle exec codeclimate-test-reporter
 sudo: false

data/CHANGES

@@ -17,3 +17,6 @@
 
 = 0.1.6 - 6-July-2020
 * Rescue message body decoding errors. Re-raise errors raised by users code of processing received messages. 
+
+= 1.0.0 - 28-Sep-2020
+* Switch to use rdkafka client

data/lib/manageiq/messaging/client.rb

@@ -40,6 +40,7 @@ module ManageIQ
       def self.open(options)
         protocol = options[:protocol] || :Stomp
         client = Object.const_get("ManageIQ::Messaging::#{protocol}::Client").new(options)
+
         return client unless block_given?
 
         begin

data/lib/manageiq/messaging/common.rb

@@ -18,7 +18,7 @@ module ManageIQ
 
       def decode_body(headers, raw_body)
         return raw_body unless headers.kind_of?(Hash)
-        case headers["encoding"]
+        case headers["encoding"] || headers[:encoding]
         when "json"
           JSON.parse(raw_body)
         when "yaml"

data/lib/manageiq/messaging/kafka/client.rb

@@ -10,16 +10,11 @@ module ManageIQ
       # * :hosts (Array of Kafka cluster hosts, or)
       # * :host (Single host name)
       # * :port (host port number)
-      # * :ssl_ca_cert (security options)
-      # * :ssl_client_cert
-      # * :ssl_client_cert_key
-      # * :sasl_gssapi_principal
-      # * :sasl_gssapi_keytab
-      # * :sasl_plain_username
-      # * :sasl_plain_password
-      # * :sasl_scram_username
-      # * :sasl_scram_password
-      # * :sasl_scram_mechanism
+      #
+      # For additional security options, please refer to 
+      # https://github.com/edenhill/librdkafka/wiki/Using-SSL-with-librdkafka and
+      # https://github.com/edenhill/librdkafka/wiki/Using-SASL-with-librdkafka 
+      # 
       #
       # Kafka specific +publish_message+ options:
       # * :group_name (Used as Kafka partition_key)
@@ -42,7 +37,7 @@ module ManageIQ
       #
       # +subscribe_background_job+ is currently not implemented.
       class Client < ManageIQ::Messaging::Client
-        require 'kafka'
+        require 'rdkafka'
         require 'manageiq/messaging/kafka/common'
         require 'manageiq/messaging/kafka/queue'
         require 'manageiq/messaging/kafka/background_job'
@@ -53,27 +48,27 @@ module ManageIQ
         include BackgroundJob
         include Topic
 
-        private *delegate(:subscribe, :unsubscribe, :publish, :to => :kafka_client)
-        delegate :close, :to => :kafka_client
-
         attr_accessor :encoding
 
         def ack(ack_ref)
-          @queue_consumer.try(:mark_message_as_processed, ack_ref)
-          @topic_consumer.try(:mark_message_as_processed, ack_ref)
+          ack_ref.commit
+        rescue Rdkafka::RdkafkaError => e
+          logger.warn("ack failed with error #{e.message}")
+          raise unless e.message =~ /no_offset/
         end
 
         def close
-          @topic_consumer.try(:stop)
-          @topic_consumer = nil
-          @queue_consumer.try(:stop)
-          @queue_consumer = nil
-
-          @producer.try(:shutdown)
+          @producer&.close
           @producer = nil
 
-          kafka_client.close
-          @kafka_client = nil
+          @consumer&.close
+          @consumer = nil
+        end
+
+        # list all topics
+        def topics
+          native_kafka = producer.instance_variable_get(:@native_kafka)
+          Rdkafka::Metadata.new(native_kafka).topics.collect { |topic| topic[:topic_name] }
         end
 
         private
@@ -87,12 +82,11 @@ module ManageIQ
           @encoding = options[:encoding] || 'yaml'
           require "json" if @encoding == "json"
 
-          connection_opts = {}
-          connection_opts[:client_id] = options[:client_ref] if options[:client_ref]
-
-          connection_opts.merge!(options.slice(:ssl_ca_cert, :ssl_client_cert, :ssl_client_cert_key, :sasl_gssapi_principal, :sasl_gssapi_keytab, :sasl_plain_username, :sasl_plain_password, :sasl_scram_username, :sasl_scram_password, :sasl_scram_mechanism))
+          connection_opts = {:"bootstrap.servers" => hosts.join(',')}
+          connection_opts[:"client.id"] = options[:client_ref] if options[:client_ref]
 
-          @kafka_client = ::Kafka.new(hosts, connection_opts)
+          ::Rdkafka::Config.logger = logger
+          @kafka_client = ::Rdkafka::Config.new(connection_opts)
         end
       end
     end

data/lib/manageiq/messaging/kafka/common.rb

@@ -5,48 +5,28 @@ module ManageIQ
         require 'manageiq/messaging/common'
         include ManageIQ::Messaging::Common
 
-        GROUP_FOR_QUEUE_MESSAGES = 'manageiq_messaging_queue_group_'.freeze
-
         private
 
         def producer
           @producer ||= kafka_client.producer
         end
 
-        def topic_consumer(persist_ref, session_timeout = nil)
-          # persist_ref enables consumer to receive messages sent when consumer is temporarily offline
-          # it also enables consumers to do load balancing when multiple consumers join the with the same ref.
-          @topic_consumer.try(:stop) unless @persist_ref == persist_ref
-          @persist_ref = persist_ref
-
-          consumer_opts = {:group_id => persist_ref}
-          consumer_opts[:session_timeout] = session_timeout if session_timeout.present?
-
-          @topic_consumer ||= kafka_client.consumer(consumer_opts)
-        end
-
-        def queue_consumer(topic, session_timeout = nil)
-          # all queue consumers join the same group so that each message can be processed by one and only one consumer
-          @queue_consumer.try(:stop) unless @queue_topic == topic
-          @queue_topic = topic
-
-          consumer_opts = {:group_id => GROUP_FOR_QUEUE_MESSAGES + topic}
-          consumer_opts[:session_timeout] = session_timeout if session_timeout.present?
-
-          @queue_consumer ||= kafka_client.consumer(consumer_opts)
+        def consumer(beginning, options)
+          @consumer&.close
+          kafka_client[:"group.id"] = options[:persist_ref]
+          kafka_client[:"auto.offset.reset"] = beginning ? 'smallest' : 'largest'
+          kafka_client[:"enable.auto.commit"] = !!auto_ack?(options)
+          kafka_client[:"session.timeout.ms"] = options[:session_timeout] * 1000 if options[:session_timeout].present?
+          kafka_client[:"group.instance.id"] = options[:group_instance_id] if options[:group_instance_id].present?
+          @consumer = kafka_client.consumer
         end
 
-        trap("TERM") do
-          @topic_consumer.try(:stop)
-          @topic_consumer = nil
-          @queue_consumer.try(:stop)
-          @queue_consumer = nil
-        end
-
-        def raw_publish(commit, body, options)
-          producer.produce(encode_body(options[:headers], body), options)
-          producer.deliver_messages if commit
-          logger.info("Published to topic(#{options[:topic]}), msg(#{payload_log(body.inspect)})")
+        def raw_publish(wait, body, options)
+          options[:payload] = encode_body(options[:headers], body)
+          producer.produce(options).tap do |handle|
+            handle.wait if wait
+            logger.info("Published to topic(#{options[:topic]}), msg(#{payload_log(body.inspect)})")
+          end
         end
 
         def queue_for_publish(options)
@@ -68,8 +48,8 @@ module ManageIQ
 
         def for_publish(options)
           kafka_opts = {:topic => address(options)}
-          kafka_opts[:partition_key]    = options[:group_name] if options[:group_name]
-          kafka_opts[:headers]          = {}
+          kafka_opts[:partition_key] = options[:group_name] if options[:group_name]
+          kafka_opts[:headers] = {}
           kafka_opts[:headers][:sender] = options[:sender] if options[:sender]
 
           body = options[:payload] || ''
@@ -85,23 +65,30 @@ module ManageIQ
           end
         end
 
-        def process_queue_message(queue, message)
-          payload = decode_body(message.headers, message.value)
-          sender, message_type, class_name = parse_message_headers(message.headers)
-          client_headers = message.headers.except(*message_header_keys)
+        def process_queue_message(queue_consumer, queue, message)
+          begin
+            payload = decode_body(message.headers, message.payload)
+            sender, message_type, _class_name = parse_message_headers(message.headers)
+            client_headers = message.headers.except(*message_header_keys).with_indifferent_access
 
-          logger.info("Message received: queue(#{queue}), message(#{payload_log(payload)}), sender(#{sender}), type(#{message_type})")
-          [sender, message_type, class_name, payload, client_headers]
+            logger.info("Message received: queue(#{queue}), message(#{payload_log(payload)}), sender(#{sender}), type(#{message_type})")
+            yield [ManageIQ::Messaging::ReceivedMessage.new(sender, message_type, payload, client_headers, queue_consumer, self)]
+            logger.info("Messsage processed")
+          rescue StandardError => e
+            logger.error("Message processing error: #{e.message}")
+            logger.error(e.backtrace.join("\n"))
+            raise
+          end
         end
 
-        def process_topic_message(topic, message)
+        def process_topic_message(topic_consumer, topic, message)
           begin
-            payload = decode_body(message.headers, message.value)
+            payload = decode_body(message.headers, message.payload)
             sender, event_type = parse_event_headers(message.headers)
-            client_headers = message.headers.except(*event_header_keys)
+            client_headers = message.headers.except(*event_header_keys).with_indifferent_access
 
             logger.info("Event received: topic(#{topic}), event(#{payload_log(payload)}), sender(#{sender}), type(#{event_type})")
-            yield ManageIQ::Messaging::ReceivedMessage.new(sender, event_type, payload, client_headers, message, self)
+            yield ManageIQ::Messaging::ReceivedMessage.new(sender, event_type, payload, client_headers, topic_consumer, self)
             logger.info("Event processed")
           rescue StandardError => e
             logger.error("Event processing error: #{e.message}")
@@ -111,7 +98,7 @@ module ManageIQ
         end
 
         def message_header_keys
-          ['sender', 'message_type', 'class_name']
+          [:sender, :message_type, :class_name]
         end
 
         def parse_message_headers(headers)
@@ -120,12 +107,12 @@ module ManageIQ
         end
 
         def event_header_keys
-          ['sender', 'event_type']
+          [:sender, :event_type]
         end
 
         def parse_event_headers(headers)
           return [nil, nil] unless headers.kind_of?(Hash)
-          headers.values_at('sender', 'event_type')
+          headers.values_at(*event_header_keys)
         end
       end
     end

data/lib/manageiq/messaging/kafka/queue.rb

@@ -2,6 +2,8 @@ module ManageIQ
   module Messaging
     module Kafka
       module Queue
+        GROUP_FOR_QUEUE_MESSAGES = ENV['QUEUE_MESSAGES_GROUP_PREFIX'].freeze || 'manageiq_messaging_queue_group_'.freeze
+
         private
 
         def publish_message_impl(options)
@@ -10,35 +12,18 @@ module ManageIQ
         end
 
         def publish_messages_impl(messages)
-          messages.each { |msg_options| raw_publish(false, *queue_for_publish(msg_options)) }
-          producer.deliver_messages
+          handles = messages.collect { |msg_options| raw_publish(false, *queue_for_publish(msg_options)) }
+          handles.each(&:wait)
         end
 
-        def subscribe_messages_impl(options)
+        def subscribe_messages_impl(options, &block)
           topic = address(options)
-          session_timeout = options[:session_timeout]
-
-          batch_options = {}
-          batch_options[:automatically_mark_as_processed] = auto_ack?(options)
-          batch_options[:max_bytes] = options[:max_bytes] if options.key?(:max_bytes)
-
-          consumer = queue_consumer(topic, session_timeout)
-          consumer.subscribe(topic)
-          consumer.each_batch(batch_options) do |batch|
-            logger.info("Batch message received: queue(#{topic})")
-            begin
-              messages = batch.messages.collect do |message|
-                sender, message_type, _class_name, payload = process_queue_message(topic, message)
-                ManageIQ::Messaging::ReceivedMessage.new(sender, message_type, payload, headers, message, self)
-              end
+          options[:persist_ref] = GROUP_FOR_QUEUE_MESSAGES + topic
 
-              yield messages
-            rescue StandardError => e
-              logger.error("Event processing error: #{e.message}")
-              logger.error(e.backtrace.join("\n"))
-              raise
-            end
-            logger.info("Batch message processed")
+          queue_consumer = consumer(true, options)
+          queue_consumer.subscribe(topic)
+          queue_consumer.each do |message|
+            process_queue_message(queue_consumer, topic, message, &block)
           end
         end
       end

data/lib/manageiq/messaging/kafka/topic.rb

@@ -1,7 +1,11 @@
+require 'socket'
+
 module ManageIQ
   module Messaging
     module Kafka
       module Topic
+        GROUP_FOR_ADHOC_LISTENERS = Socket.gethostname.freeze
+
         private
 
         def publish_topic_impl(options)
@@ -10,19 +14,12 @@ module ManageIQ
 
         def subscribe_topic_impl(options, &block)
           topic = address(options)
-          persist_ref     = options[:persist_ref]
-          session_timeout = options[:session_timeout]
 
-          if persist_ref
-            consumer = topic_consumer(persist_ref, session_timeout)
-            consumer.subscribe(topic, :start_from_beginning => false)
-            consumer.each_message(:automatically_mark_as_processed => auto_ack?(options)) do |message|
-              process_topic_message(topic, message, &block)
-            end
-          else
-            kafka_client.each_message(:topic => topic, :start_from_beginning => false) do |message|
-              process_topic_message(topic, message, &block)
-            end
+          options[:persist_ref] = "#{GROUP_FOR_ADHOC_LISTENERS}_#{Time.now.to_i}" unless options[:persist_ref]
+          topic_consumer = consumer(false, options)
+          topic_consumer.subscribe(topic)
+          topic_consumer.each do |message|
+            process_topic_message(topic_consumer, topic, message, &block)
           end
         end
       end

data/lib/manageiq/messaging/version.rb

@@ -1,5 +1,5 @@
 module ManageIQ
   module Messaging
-    VERSION = "0.1.6"
+    VERSION = "1.0.0"
   end
 end

data/manageiq-messaging.gemspec

@@ -4,9 +4,10 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'manageiq/messaging/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "manageiq-messaging"
-  spec.version       = ManageIQ::Messaging::VERSION
-  spec.authors       = ["ManageIQ Authors"]
+  spec.name                  = "manageiq-messaging"
+  spec.version               = ManageIQ::Messaging::VERSION
+  spec.required_ruby_version = '>= 2.4'
+  spec.authors               = ["ManageIQ Authors"]
 
   spec.summary       = 'Client library for ManageIQ components to exchange messages through its internal message bus.'
   spec.description   = 'Client library for ManageIQ components to exchange messages through its internal message bus.'
@@ -21,7 +22,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency 'activesupport', '~> 5.2.4.3'
-  spec.add_dependency 'ruby-kafka', '~> 1.0'
+  spec.add_dependency 'rdkafka', '~> 0.8'
   spec.add_dependency 'stomp', '~> 1.4.4'
 
   spec.add_development_dependency "bundler"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: manageiq-messaging
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 1.0.0
 platform: ruby
 authors:
 - ManageIQ Authors
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-07-06 00:00:00.000000000 Z
+date: 2020-09-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -25,19 +25,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 5.2.4.3
 - !ruby/object:Gem::Dependency
-  name: ruby-kafka
+  name: rdkafka
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0.8'
 - !ruby/object:Gem::Dependency
   name: stomp
   requirement: !ruby/object:Gem::Requirement
@@ -152,7 +152,7 @@ dependencies:
         version: '0'
 description: Client library for ManageIQ components to exchange messages through its
   internal message bus.
-email: 
+email:
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -200,7 +200,7 @@ homepage: http://github.com/ManageIQ/manageiq-messaging
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -208,15 +208,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.1.3
+signing_key:
 specification_version: 4
 summary: Client library for ManageIQ components to exchange messages through its internal
   message bus.