manageiq-messaging 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e14f157513c6d74acad91f2afd7c3ec932e331d6
-  data.tar.gz: b0cd36a056c1cca5a9ea93f484b64b36e1495034
+  metadata.gz: 24d7b7196e5413465ff914e77ca504f91e96dc16
+  data.tar.gz: 808545abce4d4629cdcd32a565ba63800005f331
 SHA512:
-  metadata.gz: 2a07451e8cf57e968a6ad20c5964deca42b5c26392ba22ee52b1586f46ee0f376fc8136eb0ff81093db55e04b6140f4941b07368dedea8ef9c2527709cec45d7
-  data.tar.gz: 19c118466aa8202534457ca03c5760f9f987bd024297490d226a2465e9be7d70d2ac0ab8d18c00b2374305b80481b231884ca0b87b723feb1f4a9af185f8110d
+  metadata.gz: 05346bab1741de40e0a6d5fdf2abf6e87c0ad059b220671f5b62f91e917e984e8b97dc86281afa4a647818c08bea98ea59656b93f2062128a88c25fce255fb58
+  data.tar.gz: 7ea84ea24c5199330a78bc6169637c94a1cae7e270aae06986aff43c12d20df24db35060288fcd0149591281ece366654342c7ec8cd8d78448c606802d4b312b

data/CHANGES

@@ -1,2 +1,7 @@
 = 0.1.0 - 4-Oct-2018
 * Initial release
+
+= 0.1.1 - 20-Nov-2018
+* By default upon receiving a message or an event, it is automatically acknowledged. But
+  the subscriber can decide to turn off the auto ack feature and ack it in the callback block.
+

data/README.md

@@ -80,7 +80,7 @@ This is the one-to-one publish/subscribe pattern. Multiple subscribers can subsc
     }
   )
 
-  client.subscribe_messages(:service => 'ems_operation', :affinity => 'ems_amazon1') do |messages|
+  client.subscribe_messages(:service => 'ems_operation', :affinity => 'ems_amazon1', :auto_ack => false) do |messages|
     messages.each do |msg|
       # do stuff with msg.message and msg.payload
       client.ack(msg.ack_ref)
@@ -149,8 +149,8 @@ This is the one-to-many publish/subscribe pattern. Multiple subscribers can subs
       :timestamp => '1501091391'
     })
 
-  client.subscribe_topic(:service => 'provider_events', :persist_ref => 'automate_1') do |sender, event, payload|
-    # do stuff with event and payload. sender may be nil if not set by the publisher
+  client.subscribe_topic(:service => 'provider_events', :persist_ref => 'automate_1') do |msg|
+    # do stuff with msg.sender, msg.message, and msg.payload. sender may be nil if not set by the publisher
   end
 ```
 

data/examples/message.rb

@@ -23,7 +23,7 @@ class ProducerConsumer < Common
       puts "produced 5 messages"
 
       puts "consumer"
-      client.subscribe_messages(:service => 'ems_operation', :affinity => 'ems_amazon1') do |messages|
+      client.subscribe_messages(:service => 'ems_operation', :affinity => 'ems_amazon1', :auto_ack => false) do |messages|
         messages.each do |msg|
           do_stuff(msg)
           client.ack(msg.ack_ref)

data/lib/manageiq/messaging/client.rb

@@ -94,6 +94,7 @@ module ManageIQ
       # Expected keys in +options+ are:
       # * :service  (service and affinity are used to determine the queue)
       # * :affinity (optional)
+      # * :auto_ack (default true, if it is false, client.ack method must be explicitly called)
       # Other options are underlying messaging system specific.
       #
       # A callback block is needed to consume the messages:
@@ -107,16 +108,18 @@ module ManageIQ
       #       msg.payload
       #       msg.ack_ref #used to ack the message
       #
-      #       client.ack(msg.ack_ref)
+      #       client.ack(msg.ack_ref) # needed only when options[:auto_ack] is false
       #       # process the message
       #     end
       #   end
       #
-      # Some messaging systems require the subscriber to ack each message in the
+      # With the auto_ack option default to true, the message will be automatically
+      # acked immediately after the delivery.
+      # Some messaging systems allow the subscriber to ack each message in the
       # callback block. The code in the block can decide when to ack according
       # to whether a message can be retried. Ack the message in the beginning of
       # processing if the message is not re-triable; otherwise ack it after the
-      # message is done. Any un-acked message will be redelivered to next subscriber
+      # message is proccessed. Any un-acked message will be redelivered to next subscriber
       # AFTER the current subscriber disconnects normally or abnormally (e.g. crashed).
       #
       # To ack a message call +ack+(+msg.ack_ref+)
@@ -131,6 +134,7 @@ module ManageIQ
       # Expected keys in +options+ are:
       # * :service  (service and affinity are used to determine the queue)
       # * :affinity (optional)
+      # * :auto_ack (default true, if it is false, client.ack method must be explicitly called)
       # Other options are underlying messaging system specific.
       #
       # This subscriber consumes messages sent through +publish_message+ with required
@@ -146,7 +150,7 @@ module ManageIQ
       #       }
       #     )
       #
-      # Background job assumes each job is not re-triable. It will ack as soon as a request
+      # Background job assumes each job is not re-triable. It is auto-acked as soon as a request
       # is received
       def subscribe_background_job(options)
         assert_options(options, [:service])
@@ -179,10 +183,28 @@ module ManageIQ
       #
       # A callback block is needed to consume the topic:
       #
-      #   client.subcribe_topic(:service => 'provider_events') do |sender, event, payload|
-      #     # sender, event, and payload are from publish_topic
+      #   client.subcribe_topic(:service => 'provider_events', :auto_ack => false) do |msg|
+      #     # msg is a type of ManageIQ::Messaging::ReceivedMessage
+      #     # attributes in msg
+      #     msg.sender
+      #     msg.message
+      #     msg.payload
+      #     msg.ack_ref #used to ack the message
+      #
+      #     client.ack(msg.ack_ref) # needed only when options[:auto_ack] is false
+      #     # process the message
       #   end
       #
+      # With the auto_ack option default to true, the message will be automatically
+      # acked immediately after the delivery.
+      # Some messaging systems allow the subscriber to ack each message in the
+      # callback block. The code in the block can decide when to ack according
+      # to whether a message can be retried. Ack the message in the beginning of
+      # processing if the message is not re-triable; otherwise ack it after the
+      # message is proccessed. Any un-acked message will be redelivered to next subscriber
+      # AFTER the current subscriber disconnects normally or abnormally (e.g. crashed).
+      #
+      # To ack a message call +ack+(+msg.ack_ref+)
       def subscribe_topic(options, &block)
         raise "A block is required" unless block_given?
         assert_options(options, [:service])

data/lib/manageiq/messaging/common.rb

@@ -31,6 +31,10 @@ module ManageIQ
       def payload_log(payload)
         payload.to_s[0..100]
       end
+
+      def auto_ack?(options)
+        options.key?(:auto_ack) ? options[:auto_ack] : true
+      end
     end
   end
 end

data/lib/manageiq/messaging/kafka/client.rb

@@ -53,12 +53,16 @@ module ManageIQ
 
         attr_accessor :encoding
 
-        def ack(*_args)
+        def ack(ack_ref)
+          @queue_consumer.try(:mark_message_as_processed, ack_ref)
+          @topic_consumer.try(:mark_message_as_processed, ack_ref)
         end
 
         def close
-          @consumer.try(:stop)
-          @consumer = nil
+          @topic_consumer.try(:stop)
+          @topic_consumer = nil
+          @queue_consumer.try(:stop)
+          @queue_consumer = nil
 
           @producer.try(:shutdown)
           @producer = nil

data/lib/manageiq/messaging/kafka/common.rb

@@ -16,7 +16,7 @@ module ManageIQ
         def topic_consumer(persist_ref)
           # persist_ref enables consumer to receive messages sent when consumer is temporarily offline
           # it also enables consumers to do load balancing when multiple consumers join the with the same ref.
-          @consumer.try(:stop) unless @persist_ref == persist_ref
+          @topic_consumer.try(:stop) unless @persist_ref == persist_ref
           @persist_ref = persist_ref
           @topic_consumer ||= kafka_client.consumer(:group_id => persist_ref)
         end
@@ -27,8 +27,10 @@ module ManageIQ
         end
 
         trap("TERM") do
-          @consumer.try(:stop)
-          @consumer = nil
+          @topic_consumer.try(:stop)
+          @topic_consumer = nil
+          @queue_consumer.try(:stop)
+          @queue_consumer = nil
         end
 
         def raw_publish(commit, body, options)
@@ -82,7 +84,7 @@ module ManageIQ
             payload = decode_body(message.headers, message.value)
             sender, event_type = parse_event_headers(message.headers)
             logger.info("Event received: topic(#{topic}), event(#{payload_log(payload)}), sender(#{sender}), type(#{event_type})")
-            yield sender, event_type, payload
+            yield ManageIQ::Messaging::ReceivedMessage.new(sender, event_type, payload, message)
             logger.info("Event processed")
           rescue StandardError => e
             logger.error("Event processing error: #{e.message}")

data/lib/manageiq/messaging/kafka/queue.rb

@@ -19,12 +19,12 @@ module ManageIQ
 
           consumer = queue_consumer
           consumer.subscribe(topic)
-          consumer.each_batch do |batch|
+          consumer.each_batch(:automatically_mark_as_processed => auto_ack?(options)) do |batch|
             logger.info("Batch message received: queue(#{topic})")
             begin
               messages = batch.messages.collect do |message|
                 sender, message_type, _class_name, payload = process_queue_message(topic, message)
-                ManageIQ::Messaging::ReceivedMessage.new(sender, message_type, payload, nil)
+                ManageIQ::Messaging::ReceivedMessage.new(sender, message_type, payload, message)
               end
 
               yield messages

data/lib/manageiq/messaging/kafka/topic.rb

@@ -15,7 +15,9 @@ module ManageIQ
           if persist_ref
             consumer = topic_consumer(persist_ref)
             consumer.subscribe(topic, :start_from_beginning => false)
-            consumer.each_message { |message| process_topic_message(topic, message, &block) }
+            consumer.each_message(:automatically_mark_as_processed => auto_ack?(options)) do |message|
+              process_topic_message(topic, message, &block)
+            end
           else
             kafka_client.each_message(:topic => topic, :start_from_beginning => false) do |message|
               process_topic_message(topic, message, &block)

data/lib/manageiq/messaging/stomp/queue.rb

@@ -28,6 +28,8 @@ module ManageIQ
           # for STOMP we can get message one at a time
           subscribe(queue_name, headers) do |msg|
             begin
+              ack(msg) if auto_ack?(options)
+
               sender = msg.headers['sender']
               message_type = msg.headers['message_type']
               message_body = decode_body(msg.headers, msg.body)

data/lib/manageiq/messaging/stomp/topic.rb

@@ -17,13 +17,13 @@ module ManageIQ
 
           subscribe(queue_name, headers) do |event|
             begin
-              ack(event)
+              ack(event) if auto_ack?(options)
 
               sender = event.headers['sender']
               event_type = event.headers['event_type']
               event_body = decode_body(event.headers, event.body)
               logger.info("Event received: queue(#{queue_name}), event(#{event_body}), headers(#{event.headers})")
-              yield sender, event_type, event_body
+              yield ManageIQ::Messaging::ReceivedMessage.new(sender, event_type, event_body, event)
               logger.info("Event processed")
             rescue => e
               logger.error("Event processing error: #{e.message}")

data/lib/manageiq/messaging/version.rb

@@ -1,5 +1,5 @@
 module ManageIQ
   module Messaging
-    VERSION = "0.1.0"
+    VERSION = "0.1.1"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: manageiq-messaging
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - ManageIQ Authors
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-10-08 00:00:00.000000000 Z
+date: 2018-11-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport