phobos 1.8.3.pre.beta1 → 1.8.3.pre.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 94cb69f1956951dadecde1cfee5acf2eba368fcba756658242f9d3578a38bd5b
-  data.tar.gz: 9ad68c2dde68e2360d0cd4ab9882795c5d900c99e56a94c527880ec826c26ff3
+  metadata.gz: 0d9c3d6d826d8865116eeeef7183e1d3eb60862d4673fb10c0464577707e1a8a
+  data.tar.gz: 7317cdfb2c8516cfa837f52c41d14181987ccdd12698eccfb8119def24c24ab1
 SHA512:
-  metadata.gz: 0eb8e4ff3dcc452312d45c5fd674c296efe89bbe3b74bf6e31c8e57795e21277e922daab460db01207ea9ba6b6517c2ea5328bd249b5221e4ccaa236bb7a0868
-  data.tar.gz: 2373edf0b9da8f3136dab604f0fe8a1705bb9fd1a2c649a32f19e1f278e593b1c97ead5d1f5468e8e99e8c29c90432826cd013c9e20599deba430c14fc0c9601
+  metadata.gz: 8f4eb5b010f7c628db69ddbf4c5cc4237d1afff4f8a42b462e9e47cf835a2bbf2950c827865640bb3456009a7fcdd2b900df3d1d758206e82ddb774ffef5c202
+  data.tar.gz: 5b3a4b2ac5755a296097c52ded7e5ead4fc866250ff24c16ca75e620dd056c315dbeba1055fee1c4f509d88619b1bb0aaea7c8b6cf61861ac72ed1fab6aa1ad7

data/CHANGELOG.md

@@ -3,9 +3,13 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
-
+``
 ## UNRELEASED
 
+## [1.8.3-beta2] - 2019-11-15
+- Add support for message headers in both produced and consumed
+  messages and batches.
+
 ## [1.8.3-beta1] - 2019-11-11
 - Automatically heartbeat after every message if necessary in batch 
   mode.

data/README.md

@@ -139,6 +139,8 @@ A handler is required to implement the method `#consume(payload, metadata)`.
 
 Instances of your handler will be created for every message, so keep a constructor without arguments. If `consume` raises an exception, Phobos will retry the message indefinitely, applying the back off configuration presented in the configuration file. The `metadata` hash will contain a key called `retry_count` with the current number of retries for this message. To skip a message, simply return from `#consume`.
 
+The `metadata` hash will also contain a key called `headers` with the headers of the consumed message.
+
 When the listener starts, the class method `.start` will be called with the `kafka_client` used by the listener. Use this hook as a chance to setup necessary code for your handler. The class method `.stop` will be called during listener shutdown.
 
 ```ruby
@@ -233,6 +235,9 @@ and your handler must include `BatchHandler`. Using a delivery method of `batch`
 assumes that you are still processing the messages one at a time and should
 use `Handler`.
 
+When using `inline_batch`, each instance of `Phobos::BatchMessage` will contain an
+instance method `headers` with the headers for that message.
+
 ```ruby
 class MyBatchHandler
   include Phobos::BatchHandler
@@ -282,6 +287,19 @@ my.producer.publish('topic', 'message-payload', 'partition and message key')
 MyProducer.producer.publish('topic', 'message-payload', 'partition and message key')
 ```
 
+The signature for the `publish` method is as follows:
+
+```ruby
+def publish(topic, payload, key = nil, partition_key = nil, headers = nil)
+```
+
+To produce messages with headers, 5 arguments will have to be passed to `publish`:
+
+```ruby
+my = MyProducer.new
+my.producer.publish('topic', 'message-payload', 'partition and message key', nil, { header_1: 'value 1' })
+```
+
 It is also possible to publish several messages at once:
 
 ```ruby
@@ -290,7 +308,7 @@ MyProducer
   .publish_list([
     { topic: 'A', payload: 'message-1', key: '1' },
     { topic: 'B', payload: 'message-2', key: '2' },
-    { topic: 'B', payload: 'message-3', key: '3' }
+    { topic: 'B', payload: 'message-3', key: '3', headers: { header_1: 'value 1', header_2: 'value 2' } }
   ])
 ```
 

data/lib/phobos/actions/process_batch_inline.rb

@@ -24,17 +24,10 @@ module Phobos
       end
 
       def execute
-        payloads = @batch.messages.map do |message|
-          Phobos::BatchMessage.new(
-            key: message.key,
-            partition: message.partition,
-            offset: message.offset,
-            payload: force_encoding(message.value)
-          )
-        end
+        batch = @batch.messages.map { |message| instantiate_batch_message(message) }
 
         begin
-          process_batch(payloads)
+          process_batch(batch)
         rescue StandardError => e
           handle_error(e, 'listener.retry_handler_error_batch',
                        "error processing inline batch, waiting #{backoff_interval}s")
@@ -44,14 +37,24 @@ module Phobos
 
       private
 
-      def process_batch(payloads)
+      def instantiate_batch_message(message)
+        Phobos::BatchMessage.new(
+          key: message.key,
+          partition: message.partition,
+          offset: message.offset,
+          payload: force_encoding(message.value),
+          headers: message.headers
+        )
+      end
+
+      def process_batch(batch)
         instrument('listener.process_batch_inline', @metadata) do |_metadata|
           handler = @listener.handler_class.new
 
-          preprocessed_payloads = handler.before_consume_batch(payloads, @metadata)
-          consume_block = proc { handler.consume_batch(preprocessed_payloads, @metadata) }
+          preprocessed_batch = handler.before_consume_batch(batch, @metadata)
+          consume_block = proc { handler.consume_batch(preprocessed_batch, @metadata) }
 
-          handler.around_consume_batch(preprocessed_payloads, @metadata, &consume_block)
+          handler.around_consume_batch(preprocessed_batch, @metadata, &consume_block)
         end
       end
     end

data/lib/phobos/actions/process_message.rb

@@ -16,7 +16,8 @@ module Phobos
           key: message.key,
           partition: message.partition,
           offset: message.offset,
-          retry_count: 0
+          retry_count: 0,
+          headers: message.headers
         )
       end
 

data/lib/phobos/batch_message.rb

@@ -2,17 +2,18 @@
 
 module Phobos
   class BatchMessage
-    attr_accessor :key, :partition, :offset, :payload
+    attr_accessor :key, :partition, :offset, :payload, :headers
 
-    def initialize(key:, partition:, offset:, payload:)
+    def initialize(key:, partition:, offset:, payload:, headers:)
       @key = key
       @partition = partition
       @offset = offset
       @payload = payload
+      @headers = headers
     end
 
     def ==(other)
-      [:key, :partition, :offset, :payload].all? do |s|
+      [:key, :partition, :offset, :payload, :headers].all? do |s|
         public_send(s) == other.public_send(s)
       end
     end

data/lib/phobos/producer.rb

@@ -15,18 +15,18 @@ module Phobos
         @host_obj = host_obj
       end
 
-      def publish(topic, payload, key = nil, partition_key = nil)
-        class_producer.publish(topic, payload, key, partition_key)
+      def publish(topic, payload, key = nil, partition_key = nil, headers = nil)
+        class_producer.publish(topic, payload, key, partition_key, headers)
       end
 
-      def async_publish(topic, payload, key = nil, partition_key = nil)
-        class_producer.async_publish(topic, payload, key, partition_key)
+      def async_publish(topic, payload, key = nil, partition_key = nil, headers = nil)
+        class_producer.async_publish(topic, payload, key, partition_key, headers)
       end
 
-      # @param messages [Array(Hash(:topic, :payload, :key))]
+      # @param messages [Array(Hash(:topic, :payload, :key, :headers))]
       #        e.g.: [
-      #          { topic: 'A', payload: 'message-1', key: '1' },
-      #          { topic: 'B', payload: 'message-2', key: '2' }
+      #          { topic: 'A', payload: 'message-1', key: '1', headers: { foo: 'bar' } },
+      #          { topic: 'B', payload: 'message-2', key: '2', headers: { foo: 'bar' } }
       #        ]
       #
       def publish_list(messages)
@@ -86,9 +86,9 @@ module Phobos
           producer_store[:sync_producer] = nil
         end
 
-        def publish(topic, payload, key = nil, partition_key = nil)
+        def publish(topic, payload, key = nil, partition_key = nil, headers = nil)
           publish_list([{ topic: topic, payload: payload, key: key,
-                          partition_key: partition_key }])
+                          partition_key: partition_key, headers: headers }])
         end
 
         def publish_list(messages)
@@ -109,9 +109,9 @@ module Phobos
           producer_store[:async_producer]
         end
 
-        def async_publish(topic, payload, key = nil, partition_key = nil)
+        def async_publish(topic, payload, key = nil, partition_key = nil, headers = nil)
           async_publish_list([{ topic: topic, payload: payload, key: key,
-                                partition_key: partition_key }])
+                                partition_key: partition_key, headers: headers }])
         end
 
         def async_publish_list(messages)
@@ -144,6 +144,7 @@ module Phobos
             partition_key = message[:partition_key] || message[:key]
             producer.produce(message[:payload], topic: message[:topic],
                                                 key: message[:key],
+                                                headers: message[:headers],
                                                 partition_key: partition_key)
           end
         end

data/lib/phobos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Phobos
-  VERSION = '1.8.3-beta1'
+  VERSION = '1.8.3-beta2'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: phobos
 version: !ruby/object:Gem::Version
-  version: 1.8.3.pre.beta1
+  version: 1.8.3.pre.beta2
 platform: ruby
 authors:
 - Túlio Ornelas
@@ -15,7 +15,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-12 00:00:00.000000000 Z
+date: 2019-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler