manageiq-messaging 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8859fa0a68ffe54c38283145348077bcbc78e004636e38960cc0d2ff70923f7a
-  data.tar.gz: '088f6d9d2dc80e753b16a0bbdce58dc789aec1d5eada50c0aa3b74fd78109006'
+  metadata.gz: 78b0fa6b782a5bb3e21552abd1c0b54f473bdc460c722f896a7fecc7389259b3
+  data.tar.gz: f0ff3e82e5108b2241cebcfb2cbf9dde8208c3cf21e61b355d1cec9292562a64
 SHA512:
-  metadata.gz: 457f22d4a96c0ec19a05cb2a0ebe3a9f665f62583d7337ede95ddf786e899b78dcd86173c219988d34ef8d4deafc188d8bcd184728e8362e4b66c772875370f5
-  data.tar.gz: b496df16b18b87754bf22107fe0d7f30a2dee767f6673cdd51330880ea290539967397b32f629cb4ff24fb24e428390ba9478cc0c2bbc359c404e0a5cd358976
+  metadata.gz: 331823865454b700433699cb00f8938311ba91cf0ad87a807ffbc955be6e2d7060e5fc3fabbe91fea5f52b72eaf74a49d0909e94a63a665ccb2f44f9e56f76a0
+  data.tar.gz: a3e6c91fc5ba3a476908d26baf2f5cb3d1d4c768d63e69d3e318442b35f4bf71a3cf501d9dc12bbd53cdba66121f19d2c87ec14dd7eabc19a8aa4aefa40dc302

data/CHANGES

@@ -7,3 +7,7 @@
 
 = 0.1.2 - 11-Dec-2018
 * Allow to set max_bytes to each_batch when subscribe to a Kafka topic.
+
+= 0.1.4 - 3-Apr-2019
+* Add an #ack method to a ReceivedMessage to simplify manual acknowledgements
+* Allow caller to provide a session_timeout to kafka consumers, default of 30sec

data/README.md

@@ -83,7 +83,7 @@ This is the one-to-one publish/subscribe pattern. Multiple subscribers can subsc
   client.subscribe_messages(:service => 'ems_operation', :affinity => 'ems_amazon1', :auto_ack => false) do |messages|
     messages.each do |msg|
       # do stuff with msg.message and msg.payload
-      client.ack(msg.ack_ref)
+      msg.ack
     end
   end
 

data/examples/message.rb

@@ -26,7 +26,7 @@ class ProducerConsumer < Common
       client.subscribe_messages(:service => 'ems_operation', :affinity => 'ems_amazon1', :auto_ack => false) do |messages|
         messages.each do |msg|
           do_stuff(msg)
-          client.ack(msg.ack_ref)
+          msg.ack
         end
       end
       sleep(5)

data/lib/manageiq/messaging/client.rb

@@ -106,9 +106,9 @@ module ManageIQ
       #       msg.sender
       #       msg.message
       #       msg.payload
-      #       msg.ack_ref #used to ack the message
+      #       msg.ack_ref
       #
-      #       client.ack(msg.ack_ref) # needed only when options[:auto_ack] is false
+      #       msg.ack # needed only when options[:auto_ack] is false
       #       # process the message
       #     end
       #   end
@@ -122,7 +122,7 @@ module ManageIQ
       # message is proccessed. Any un-acked message will be redelivered to next subscriber
       # AFTER the current subscriber disconnects normally or abnormally (e.g. crashed).
       #
-      # To ack a message call +ack+(+msg.ack_ref+)
+      # To ack a message call +msg.ack+
       def subscribe_messages(options, &block)
         raise "A block is required" unless block_given?
         assert_options(options, [:service])
@@ -189,9 +189,9 @@ module ManageIQ
       #     msg.sender
       #     msg.message
       #     msg.payload
-      #     msg.ack_ref #used to ack the message
+      #     msg.ack_ref
       #
-      #     client.ack(msg.ack_ref) # needed only when options[:auto_ack] is false
+      #     msg.ack # needed only when options[:auto_ack] is false
       #     # process the message
       #   end
       #
@@ -204,7 +204,7 @@ module ManageIQ
       # message is proccessed. Any un-acked message will be redelivered to next subscriber
       # AFTER the current subscriber disconnects normally or abnormally (e.g. crashed).
       #
-      # To ack a message call +ack+(+msg.ack_ref+)
+      # To ack a message call +msg.ack+
       def subscribe_topic(options, &block)
         raise "A block is required" unless block_given?
         assert_options(options, [:service])

data/lib/manageiq/messaging/kafka/client.rb

@@ -26,9 +26,11 @@ module ManageIQ
       #
       # Kafka specific +subscribe_topic+ options:
       # * :persist_ref (Used as Kafka group_id)
+      # * :session_timeout (Max time in seconds allowed to process a message, default is 30)
       #
       # Kafka specific +subscribe_messages+ options:
       # * :max_bytes (Max batch size to read, default is 10Mb)
+      # * :session_timeout (Max time in seconds allowed to process a message, default is 30)
       #
       # Without +:persist_ref+ every topic subscriber receives a copy of each message
       # only when they are active. If multiple topic subscribers join with the same

data/lib/manageiq/messaging/kafka/common.rb

@@ -13,19 +13,27 @@ module ManageIQ
           @producer ||= kafka_client.producer
         end
 
-        def topic_consumer(persist_ref)
+        def topic_consumer(persist_ref, session_timeout = nil)
           # persist_ref enables consumer to receive messages sent when consumer is temporarily offline
           # it also enables consumers to do load balancing when multiple consumers join the with the same ref.
           @topic_consumer.try(:stop) unless @persist_ref == persist_ref
           @persist_ref = persist_ref
-          @topic_consumer ||= kafka_client.consumer(:group_id => persist_ref)
+
+          consumer_opts = {:group_id => persist_ref}
+          consumer_opts[:session_timeout] = session_timeout if session_timeout.present?
+
+          @topic_consumer ||= kafka_client.consumer(consumer_opts)
         end
 
-        def queue_consumer(topic)
+        def queue_consumer(topic, session_timeout = nil)
           # all queue consumers join the same group so that each message can be processed by one and only one consumer
           @queue_consumer.try(:stop) unless @queue_topic == topic
           @queue_topic = topic
-          @queue_consumer ||= kafka_client.consumer(:group_id => GROUP_FOR_QUEUE_MESSAGES + topic)
+
+          consumer_opts = {:group_id => GROUP_FOR_QUEUE_MESSAGES + topic}
+          consumer_opts[:session_timeout] = session_timeout if session_timeout.present?
+
+          @queue_consumer ||= kafka_client.consumer(consumer_opts)
         end
 
         trap("TERM") do
@@ -86,7 +94,7 @@ module ManageIQ
             payload = decode_body(message.headers, message.value)
             sender, event_type = parse_event_headers(message.headers)
             logger.info("Event received: topic(#{topic}), event(#{payload_log(payload)}), sender(#{sender}), type(#{event_type})")
-            yield ManageIQ::Messaging::ReceivedMessage.new(sender, event_type, payload, message)
+            yield ManageIQ::Messaging::ReceivedMessage.new(sender, event_type, payload, message, self)
             logger.info("Event processed")
           rescue StandardError => e
             logger.error("Event processing error: #{e.message}")

data/lib/manageiq/messaging/kafka/queue.rb

@@ -16,19 +16,20 @@ module ManageIQ
 
         def subscribe_messages_impl(options)
           topic = address(options)
+          session_timeout = options[:session_timeout]
 
           batch_options = {}
           batch_options[:automatically_mark_as_processed] = auto_ack?(options)
           batch_options[:max_bytes] = options[:max_bytes] if options.key?(:max_bytes)
 
-          consumer = queue_consumer(topic)
+          consumer = queue_consumer(topic, session_timeout)
           consumer.subscribe(topic)
           consumer.each_batch(batch_options) do |batch|
             logger.info("Batch message received: queue(#{topic})")
             begin
               messages = batch.messages.collect do |message|
                 sender, message_type, _class_name, payload = process_queue_message(topic, message)
-                ManageIQ::Messaging::ReceivedMessage.new(sender, message_type, payload, message)
+                ManageIQ::Messaging::ReceivedMessage.new(sender, message_type, payload, message, self)
               end
 
               yield messages

data/lib/manageiq/messaging/kafka/topic.rb

@@ -10,10 +10,11 @@ module ManageIQ
 
         def subscribe_topic_impl(options, &block)
           topic = address(options)
-          persist_ref = options[:persist_ref]
+          persist_ref     = options[:persist_ref]
+          session_timeout = options[:session_timeout]
 
           if persist_ref
-            consumer = topic_consumer(persist_ref)
+            consumer = topic_consumer(persist_ref, session_timeout)
             consumer.subscribe(topic, :start_from_beginning => false)
             consumer.each_message(:automatically_mark_as_processed => auto_ack?(options)) do |message|
               process_topic_message(topic, message, &block)

data/lib/manageiq/messaging/received_message.rb

@@ -1,10 +1,14 @@
 module ManageIQ
   module Messaging
     class ReceivedMessage
-      attr_accessor :sender, :message, :payload, :ack_ref
+      attr_accessor :sender, :message, :payload, :ack_ref, :client
 
-      def initialize(sender, message, payload, ack_ref)
-        @sender, @message, @payload, @ack_ref = sender, message, payload, ack_ref
+      def initialize(sender, message, payload, ack_ref, client)
+        @sender, @message, @payload, @ack_ref, @client = sender, message, payload, ack_ref, client
+      end
+
+      def ack
+        client.ack(ack_ref)
       end
     end
   end

data/lib/manageiq/messaging/stomp/queue.rb

@@ -35,7 +35,7 @@ module ManageIQ
               message_body = decode_body(msg.headers, msg.body)
               logger.info("Message received: queue(#{queue_name}), msg(#{payload_log(message_body)}), headers(#{msg.headers})")
 
-              result = yield [ManageIQ::Messaging::ReceivedMessage.new(sender, message_type, message_body, msg)]
+              result = yield [ManageIQ::Messaging::ReceivedMessage.new(sender, message_type, message_body, msg, self)]
               logger.info("Message processed")
 
               correlation_ref = msg.headers['correlation_id']

data/lib/manageiq/messaging/stomp/topic.rb

@@ -23,7 +23,7 @@ module ManageIQ
               event_type = event.headers['event_type']
               event_body = decode_body(event.headers, event.body)
               logger.info("Event received: queue(#{queue_name}), event(#{event_body}), headers(#{event.headers})")
-              yield ManageIQ::Messaging::ReceivedMessage.new(sender, event_type, event_body, event)
+              yield ManageIQ::Messaging::ReceivedMessage.new(sender, event_type, event_body, event, self)
               logger.info("Event processed")
             rescue => e
               logger.error("Event processing error: #{e.message}")

data/lib/manageiq/messaging/version.rb

@@ -1,5 +1,5 @@
 module ManageIQ
   module Messaging
-    VERSION = "0.1.3"
+    VERSION = "0.1.4"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: manageiq-messaging
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - ManageIQ Authors
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-02-25 00:00:00.000000000 Z
+date: 2019-04-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -202,7 +202,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 2.7.6.2
 signing_key: 
 specification_version: 4
 summary: Client library for ManageIQ components to exchange messages through its internal