karafka 0.5.0.1 → 0.5.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d7e097bb23271902edf811b27a20afb3da70aaf4
-  data.tar.gz: d104e55ea9c218e6d931d4922252dd5a2dc38c23
+  metadata.gz: b15329e4a83277e01ced5cb6e7e0419d392707a6
+  data.tar.gz: 6080d3348e67cd407e37472e97a400f8d648bdb0
 SHA512:
-  metadata.gz: 8f010bd5993055cf89622d343aa09307539539b503ee42e722112ce7498b7cbc4c750db475518eec07bfe33d028f9befb1b2240b09ccfb44b90c786dae6f878e
-  data.tar.gz: 8feb1a74b19313ffb12c74152925e09d4307723bab316b46c5238ae7b8b9020cb3ab71f10886f78335cffcd6735e8a7aa4a882f17f583782aa2d055e49c4edef
+  metadata.gz: 109523139cb10a65641300fec95edd09b55a190ae912df450d614b016c2e76988511dbcd29510400bc7d0ce3fac139035736628dc5347e2919b927f7e0284e2d
+  data.tar.gz: ba2958568a92b148c956355b164f35ec6408f35a2efe55dcb880ac8fd8009acb951ed89904d71b2341ee66304def6f125b10aa4536a620c849512432d9c94e50

data/.gitignore

@@ -18,7 +18,6 @@ db/*.sqlite3
 **.war
 *.rbc
 *.sassc
-.rspec
 .redcar/
 .capistrano/
 .sass-cache

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper

data/.ruby-version

@@ -1 +1 @@
-2.3.1
+2.4.0

data/.travis.yml

@@ -2,5 +2,8 @@ language: ruby
 rvm:
   - 2.3.0
   - 2.3.1
+  - 2.3.2
+  - 2.3.3
+  - 2.4.0
 script:
   - bundle exec rake

data/CHANGELOG.md

@@ -1,8 +1,26 @@
 # Karafka framework changelog
 
+## 0.5.0.2
+- Gems update x3
+- Default Ruby set to 2.3.3
+- ~~Default Ruby set to 2.4.0~~
+- Readme updates to match bug fixes and resolved issues
+- #95 - Allow options into responder
+- #98 - Use parser when responding on a topic
+- #114 - Option to configure waterdrop connection pool timeout and concurrency
+- #118 - Added dot in topic validation format
+- #119 - add support for authentication using SSL
+- #121 - JSON as a default for standalone responders usage
+- #122 - Allow on capistrano role customization
+- #125 - Add support to batch incoming messages
+- #130 - start_from_beginning flag on routes and default
+- #128 - Monitor caller_label not working with super on inheritance
+- Renamed *inline* to *inline_mode* to stay consistent with flags that change the way karafka works (#125)
+- Dry-configurable dump to 0.5 with fixed proc value evaluation on retrieve patch (internal change)
+
 ## 0.5.0.1
 - Fixed inconsistency in responders non-required topic definition. Now only required: false available
-- #101
+- #101 - Responders fail when multiple_usage true and required false
 - fix error on startup from waterdrop #102
 - Waterdrop 0.3.2.1 with kafka.hosts instead of kafka_hosts
 - #105 - Karafka::Monitor#caller_label not working with inherited monitors

data/Gemfile.lock

@@ -1,22 +1,22 @@
 PATH
   remote: .
   specs:
-    karafka (0.5.0.1)
+    karafka (0.5.0.2)
       activesupport (~> 5.0)
       celluloid (~> 0.17)
-      dry-configurable (~> 0.1.7)
+      dry-configurable (~> 0.5)
       envlogic (~> 1.0)
       rake (~> 11.3)
       ruby-kafka (= 0.3.15)
       sidekiq (~> 4.2)
       thor (~> 0.19)
-      waterdrop (~> 0.3)
+      waterdrop (~> 0.3.2.1)
       worker-glass (~> 0.2)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.0.0.1)
+    activesupport (5.0.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (~> 0.7)
       minitest (~> 5.1)
@@ -54,13 +54,13 @@ GEM
     coercible (1.0.0)
       descendants_tracker (~> 0.0.1)
     colorize (0.8.1)
-    concurrent-ruby (1.0.2)
-    connection_pool (2.2.0)
+    concurrent-ruby (1.0.4)
+    connection_pool (2.2.1)
     descendants_tracker (0.0.4)
       thread_safe (~> 0.3, >= 0.3.1)
     diff-lcs (1.2.5)
     docile (1.1.5)
-    dry-configurable (0.1.7)
+    dry-configurable (0.5.0)
       concurrent-ruby (~> 1.0)
     envlogic (1.0.3)
       activesupport
@@ -91,7 +91,7 @@ GEM
     launchy (2.4.3)
       addressable (~> 2.3)
     method_source (0.8.2)
-    minitest (5.9.1)
+    minitest (5.10.1)
     null-logger (0.1.3)
     parser (2.3.1.2)
       ast (~> 2.2)
@@ -120,7 +120,7 @@ GEM
       rack
     rainbow (2.1.0)
     rake (11.3.0)
-    redis (3.3.1)
+    redis (3.3.3)
     reek (4.1.0)
       codeclimate-engine-rb (~> 0.3.1)
       parser (~> 2.3.1, >= 2.3.1.2)
@@ -166,7 +166,7 @@ GEM
     shoulda-context (1.2.1)
     shoulda-matchers (2.8.0)
       activesupport (>= 3.0.0)
-    sidekiq (4.2.3)
+    sidekiq (4.2.9)
       concurrent-ruby (~> 1.0)
       connection_pool (~> 2.2, >= 2.2.0)
       rack-protection (>= 1.5.0)
@@ -178,11 +178,11 @@ GEM
     simplecov-html (0.10.0)
     slop (3.6.0)
     sysexits (1.2.0)
-    thor (0.19.1)
+    thor (0.19.4)
     thread_safe (0.3.5)
     tilt (2.0.5)
     timecop (0.8.1)
-    timers (4.1.1)
+    timers (4.1.2)
       hitimes
     tzinfo (1.2.2)
       thread_safe (~> 0.1)
@@ -192,10 +192,10 @@ GEM
       coercible (~> 1.0)
       descendants_tracker (~> 0.0, >= 0.0.3)
       equalizer (~> 0.0, >= 0.0.9)
-    waterdrop (0.3.2.1)
+    waterdrop (0.3.2.2)
       bundler
       connection_pool
-      dry-configurable (~> 0.1.7)
+      dry-configurable (~> 0.5)
       null-logger
       rake
       ruby-kafka
@@ -214,4 +214,4 @@ DEPENDENCIES
   timecop
 
 BUNDLED WITH
-   1.12.5
+   1.13.7

data/README.md

@@ -31,7 +31,8 @@ Karafka not only handles incoming messages but also provides tools for building
         - [Parser](#parser)
         - [Interchanger](#interchanger)
         - [Responder](#responder)
-        - [Inline flag](#inline-flag)
+        - [Inline mode flag](#inline-mode-flag)
+        - [Batch mode flag](#batch-mode-flag)
     - [Receiving messages](#receiving-messages)
         - [Processing messages directly (without Sidekiq)](#processing-messages-directly-without-sidekiq)
     - [Sending messages from Karafka](#sending-messages-from-karafka)
@@ -45,6 +46,7 @@ Karafka not only handles incoming messages but also provides tools for building
         - [Registering topics](#registering-topics)
         - [Responding on topics](#responding-on-topics)
         - [Response validation](#response-validation)
+        - [Response partitioning](#response-partitioning)
   - [Monitoring and logging](#monitoring-and-logging)
       - [Example monitor with Errbit/Airbrake support](#example-monitor-with-errbitairbrake-support)
       - [Example monitor with NewRelic support](#example-monitor-with-newrelic-support)
@@ -110,8 +112,10 @@ Karafka has following configuration options:
 | Option                        | Required | Value type        | Description                                                                                                |
 |-------------------------------|----------|-------------------|------------------------------------------------------------------------------------------------------------|
 | name                          | true     | String            | Application name                                                                                           |
-| inline                        | false    | Boolean           | Do we want to perform logic without enqueuing it with Sidekiq (directly and asap)                          |
 | redis                         | true     | Hash              | Hash with Redis configuration options                                                                      |
+| inline_mode                   | false    | Boolean           | Do we want to perform logic without enqueuing it with Sidekiq (directly and asap)                          |
+| batch_mode                    | false    | Boolean           | Should the incoming messages be consumed in batches, or one at a time                                      |
+| start_from_beginning          | false    | Boolean           | Consume messages starting at the beginning or consume new messages that are produced at first run          |
 | monitor                       | false    | Object            | Monitor instance (defaults to Karafka::Monitor)                                                            |
 | logger                        | false    | Object            | Logger instance (defaults to Karafka::Logger)                                                              |
 | kafka.hosts                   | false    | Array<String>     | Kafka server hosts. If 1 provided, Karafka will discover cluster structure automatically                   |
@@ -119,6 +123,11 @@ Karafka has following configuration options:
 | kafka.offset_commit_interval  | false    | Integer           | The interval between offset commits in seconds                                                             |
 | kafka.offset_commit_threshold | false    | Integer           | The number of messages that can be processed before their offsets are committed                            |
 | kafka.heartbeat_interval      | false    | Integer           | The interval between heartbeats                                                                            |
+| kafka.ssl.ca_cert             | false    | String            | SSL CA certificate                                                                                         |
+| kafka.ssl.client_cert         | false    | String            | SSL client certificate                                                                                     |
+| kafka.ssl.client_cert_key     | false    | String            | SSL client certificate password                                                                            |
+| connection_pool.size          | false    | Integer           | Connection pool size for message producers connection pool                                                 |
+| connection_pool.timeout       | false    | Integer           | Connection pool timeout for message producers connection pool                                              |
 
 To apply this configuration, you need to use a *setup* method from the Karafka::App class (app.rb):
 
@@ -126,7 +135,8 @@ To apply this configuration, you need to use a *setup* method from the Karafka::
 class App < Karafka::App
   setup do |config|
     config.kafka.hosts = %w( 127.0.0.1:9092 )
-    config.inline = false
+    config.inline_mode = false
+    config.batch_mode = false
     config.redis = {
       url: 'redis://redis.example.com:7372/1'
     }
@@ -140,12 +150,14 @@ Note: You can use any library like [Settingslogic](https://github.com/binarylogi
 
 ### Configurators
 
-If you want to do some configurations after all of this is done, please add to config directory a proper file (needs to inherit from Karafka::Config::Base and implement setup method), after that everything will happen automatically.
+For additional setup and/or configuration tasks you can create custom configurators. Similar to Rails these are added to a `config/initializers` directory and run after app initialization.
+
+Your new configurator class must inherit from `Karafka::Setup::Configurators::Base` and implement a `setup` method.
 
 Example configuration class:
 
 ```ruby
-class ExampleConfigurator < Base
+class ExampleConfigurator < Karafka::Setup::Configurators::Base
   def setup
     ExampleClass.logger = Karafka.logger
     ExampleClass.redis = config.redis
@@ -216,7 +228,8 @@ There are also several other methods available (optional):
   - *parser* - Class name - name of a parser class that we want to use to parse incoming data
   - *interchanger* - Class name - name of a interchanger class that we want to use to format data that we put/fetch into/from *#perform_async*
   - *responder* - Class name - name of a responder that we want to use to generate responses to other Kafka topics based on our processed data
-  - *inline* - Boolean - Do we want to perform logic without enqueuing it with Sidekiq (directly and asap) - overwrites global app setting
+  - *inline_mode* - Boolean - Do we want to perform logic without enqueuing it with Sidekiq (directly and asap) - overwrites global app setting
+  - *batch_mode* - Boolean - Handle the incoming messages in batch, or one at a time - overwrites global app setting
 
 ```ruby
 App.routes.draw do
@@ -227,7 +240,8 @@ App.routes.draw do
     parser Parsers::BinaryToJson
     interchanger Interchangers::Binary
     responder BinaryVideoProcessingResponder
-    inline true
+    inline_mode true
+    batch_mode true
   end
 
   topic :new_videos do
@@ -297,9 +311,14 @@ Keep in mind, that params might be in two states: parsed or unparsed when passed
 
 ##### Parser
 
- - *parser* - Class name - name of a parser class that we want to use to parse incoming data
+ - *parser* - Class name - name of a parser class that we want to use to serialize and deserialize incoming and outgoing data.
+
+Karafka by default will parse messages with a Json parser. If you want to change this behaviour you need to set a custom parser for each route. Parser needs to have a following class methods:
+
+  - *parse* - method used to parse incoming string into an object/hash
+  - *generate* - method used in responders in order to convert objects into strings that have desired format
 
-Karafka by default will parse messages with a JSON parser. If you want to change this behaviour you need to set a custom parser for each route. Parser needs to have a #parse method and raise an error that is a ::Karafka::Errors::ParserError descendant when problem appears during the parsing process.
+and raise an error that is a ::Karafka::Errors::ParserError descendant when problem appears during the parsing process.
 
 ```ruby
 class XmlParser
@@ -310,6 +329,10 @@ class XmlParser
   rescue REXML::ParseException
     raise ParserError
   end
+
+  def self.generate(object)
+    object.to_xml
+  end
 end
 
 App.routes.draw do
@@ -320,13 +343,13 @@ App.routes.draw do
 end
 ```
 
-Note that parsing failure won't stop the application flow. Instead, Karafka will assign the raw message inside the :message key of params. That way you can handle raw message inside the Sidekiq worker (you can implement error detection, etc - any "heavy" parsing logic can and should be implemented there).
+Note that parsing failure won't stop the application flow. Instead, Karafka will assign the raw message inside the :message key of params. That way you can handle raw message inside the Sidekiq worker (you can implement error detection, etc. - any "heavy" parsing logic can and should be implemented there).
 
 ##### Interchanger
 
  - *interchanger* - Class name - name of an interchanger class that we want to use to format data that we put/fetch into/from #perform_async.
 
-Custom interchangers target issues with non-standard (binary, etc) data that we want to store when we do #perform_async. This data might be corrupted when fetched in a worker (see [this](https://github.com/karafka/karafka/issues/30) issue). With custom interchangers, you can encode/compress data before it is being passed to scheduling and decode/decompress it when it gets into the worker.
+Custom interchangers target issues with non-standard (binary, etc.) data that we want to store when we do #perform_async. This data might be corrupted when fetched in a worker (see [this](https://github.com/karafka/karafka/issues/30) issue). With custom interchangers, you can encode/compress data before it is being passed to scheduling and decode/decompress it when it gets into the worker.
 
 **Warning**: if you decide to use slow interchangers, they might significantly slow down Karafka.
 
@@ -353,7 +376,7 @@ end
 
   - *responder* - Class name - name of a responder that we want to use to generate responses to other Kafka topics based on our processed data.
 
-Responders are used to design the response that should be generated and sent to proper Kafka topics, once processing is done. It allows programmers to build not only data-consuming apps, but to build apps that consume data and, then, based on the business logic output send this processed data onwards (similary to how Bash pipelines work).
+Responders are used to design the response that should be generated and sent to proper Kafka topics, once processing is done. It allows programmers to build not only data-consuming apps, but to build apps that consume data and, then, based on the business logic output send this processed data onwards (similarly to how Bash pipelines work).
 
 ```ruby
 class Responder < ApplicationResponder
@@ -369,9 +392,9 @@ end
 
 For more details about responders, please go to the [using responders](#using-responders) section.
 
-##### Inline flag
+##### Inline mode flag
 
-Inline flag allows you to disable Sidekiq usage by performing your #perform method business logic in the main Karafka server process.
+Inline mode flag allows you to disable Sidekiq usage by performing your #perform method business logic in the main Karafka server process.
 
 This flag be useful when you want to:
 
@@ -380,6 +403,12 @@ This flag be useful when you want to:
 
 Note: Keep in mind, that by using this, you can significantly slow down Karafka. You also loose all the advantages of Sidekiq processing (reentrancy, retries, etc).
 
+##### Batch mode flag
+
+Batch mode allows you to increase the overall throughput of your kafka consumer by handling incoming messages in batches, instead of one at a time.
+
+Note: The downside of increasing throughput is a slight increase in latency. Also keep in mind, that the client commits the offset of the batch's messages only **after** the entire batch has been scheduled into Sidekiq (or processed in case of inline mode).
+
 ### Receiving messages
 
 Karafka framework has a long running server process that is responsible for receiving messages.
@@ -403,7 +432,7 @@ If you don't want to use Sidekiq for processing and you would rather process mes
 ```ruby
 class App < Karafka::App
   setup do |config|
-    config.inline = false
+    config.inline_mode = true
     # Rest of the config
   end
 end
@@ -415,12 +444,12 @@ or per route (when you want to treat some routes in a different way):
 App.routes.draw do
   topic :binary_video_details do
     controller Videos::DetailsController
-    inline true
+    inline_mode true
   end
 end
 ```
 
-Note: it can slow Karafka significantly if you do heavy stuff that way.
+Note: it can slow Karafka down significantly if you do heavy stuff that way.
 
 ### Sending messages from Karafka
 
@@ -507,20 +536,20 @@ end
 #### Controllers callbacks
 
 You can add any number of *before_enqueue* callbacks. It can be a method or a block.
-before_enqueue acts in a similar way to Rails before_action so it should perform "lightweight" operations. You have access to params inside. Based on it you can define which data you want to receive and which not.
+before_enqueue acts in a similar way to Rails before_action so it should perform "lightweight" operations. You have access to params inside. Based on them you can define which data you want to receive and which you do not.
 
-**Warning**: keep in mind, that all *before_enqueue* blocks/methods are executed after messages are received. This is not executed in Sidekiq, but right after receiving the incoming message. This means, that if you perform "heavy duty" operations there, Karafka might significantly slow down.
+**Warning**: keep in mind, that all *before_enqueue* blocks/methods are executed after messages are received. This is not executed in Sidekiq, but right after receiving the incoming message. This means, that if you perform "heavy duty" operations there, Karafka might slow down significantly.
 
 If any of callbacks throws :abort - *perform* method will be not enqueued to the worker (the execution chain will stop).
 
-Once you run consumer - messages from Kafka server will be send to a proper controller (based on topic name).
+Once you run a consumer - messages from Kafka server will be send to a proper controller (based on topic name).
 
 Presented example controller will accept incoming messages from a Kafka topic named :karafka_topic
 
 ```ruby
   class TestController < ApplicationController
     # before_enqueue has access to received params.
-    # You can modify them before enqueue it to sidekiq queue.
+    # You can modify them before enqueuing it to sidekiq.
     before_enqueue {
       params.merge!(received_time: Time.now.to_s)
     }
@@ -577,6 +606,8 @@ class ExampleResponder < ApplicationResponder
 end
 ```
 
+When passing data back to Kafka, responder uses parser **#generate** method to convert message object to a string. It will use parser of a route for which a current message was directed. By default it uses Karafka::Parsers::Json parser.
+
 Note: You can use responders outside of controllers scope, however it is not recommended because then, they won't be listed when executing **karafka flow** CLI command.
 
 #### Registering topics
@@ -604,7 +635,7 @@ When you receive a single HTTP request, you generate a single HTTP response. Thi
 
 To handle responding, you need to define *#respond* instance method. This method should accept the same amount of arguments passed into *#respond_with* method.
 
-In order to send a message to a given topic, you have to use *#respond_to* method that accepts two arguments:
+In order to send a message to a given topic, you have to use **#respond_to** method that accepts two arguments:
 
   - topic name (Symbol)
   - data you want to send (if data is not string, responder will try to run #to_json method on the incoming data)
@@ -630,12 +661,36 @@ end
 
 In order to ensure the dataflow is as intended, responder will validate what and where was sent, making sure that:
 
-  - Only topics that were registered were used (no typos, etc)
+  - Only topics that were registered were used (no typos, etc.)
   - Only a single message was sent to a topic that was registered without a **multiple_usage** flag
   - Any topic that was registered with **required** flag (default behavior) has been used
 
 This is an automatic process and does not require any triggers.
 
+#### Response partitioning
+
+Kafka topics are partitioned, which means that  you can assing messages to partitions based on your business logic. To do so from responders, you can pass one of the following keyword arguments as a last option of a **#respond_to** method:
+
+* partition - use it when you want to send a given message to a certain partition
+* partition_key - use it when you want to ensure that a certain group of messages is delivered to the same partition, but you don't which partition it will be.
+
+```ruby
+class ExampleResponder < ApplicationResponder
+  topic :regular_topic
+  topic :different_topic
+
+  def respond(user, profile)
+    respond_to :regular_topic, user, partition: 12
+    # This will send user details to a partition based on the first letter
+    # of login which means that for example all users with login starting
+    # with "a" will go to the same partition on the different_topic
+    respond_to :different_topic, user, partition_key: user.login[0].downcase
+  end
+end
+```
+
+If no keys are passed, the producer will randomly assign a partition.
+
 ## Monitoring and logging
 
 Karafka provides a simple monitor (Karafka::Monitor) with a really small API. You can use it to develop your own monitoring system (using for example NewRelic). By default, the only thing that is hooked up to this monitoring is a Karafka logger (Karafka::Logger). It is based on a standard [Ruby logger](http://ruby-doc.org/stdlib-2.2.3/libdoc/logger/rdoc/Logger.html).
@@ -647,7 +702,7 @@ class App < Karafka::App
   setup do |config|
     # Other setup stuff...
     config.logger = MyCustomLogger.new
-    config.monitor = CustomMonitor.new
+    config.monitor = CustomMonitor.instance
   end
 end
 ```

data/karafka.gemspec

@@ -21,11 +21,11 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'worker-glass', '~> 0.2'
   spec.add_dependency 'celluloid', '~> 0.17'
   spec.add_dependency 'envlogic', '~> 1.0'
-  spec.add_dependency 'waterdrop', '~> 0.3'
+  spec.add_dependency 'waterdrop', '~> 0.3.2.1'
   spec.add_dependency 'rake', '~> 11.3'
   spec.add_dependency 'thor', '~> 0.19'
   spec.add_dependency 'activesupport', '~> 5.0'
-  spec.add_dependency 'dry-configurable', '~> 0.1.7'
+  spec.add_dependency 'dry-configurable', '~> 0.5'
   spec.required_ruby_version = '>= 2.3.0'
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }

data/lib/karafka/base_controller.rb

@@ -103,7 +103,7 @@ module Karafka
     # will schedule a perform task in sidekiq
     def schedule
       run_callbacks :schedule do
-        inline ? perform_inline : perform_async
+        inline_mode ? perform_inline : perform_async
       end
     end
 
@@ -149,7 +149,7 @@ module Karafka
       raise(Errors::ResponderMissing, self.class) unless responder
 
       Karafka.monitor.notice(self.class, data: data)
-      responder.new.call(*data)
+      responder.new(parser).call(*data)
     end
 
     # Executes perform code immediately (without enqueuing)

data/lib/karafka/base_responder.rb

@@ -16,6 +16,15 @@ module Karafka
   #     end
   #   end
   #
+  # @example Responding to a topic with extra options
+  #   class Responder < BaseResponder
+  #     topic :new_action
+  #
+  #     def respond(data)
+  #       respond_to :new_action, data, partition_key: 'thing'
+  #     end
+  #   end
+  #
   # @example Marking topic as not required (we won't have to use it)
   #   class Responder < BaseResponder
   #     topic :required_topic
@@ -65,8 +74,11 @@ module Karafka
     end
 
     # Creates a responder object
+    # @param parser_class [Class] parser class that we can use to generate appropriate string
+    #   or nothing if we want to default to Karafka::Parsers::Json
     # @return [Karafka::BaseResponder] base responder descendant responder
-    def initialize
+    def initialize(parser_class = Karafka::Parsers::Json)
+      @parser_class = parser_class
       @messages_buffer = {}
     end
 
@@ -94,14 +106,13 @@ module Karafka
     # as many times as we need. Especially when we have 1:n flow
     # @param topic [Symbol, String] topic to which we want to respond
     # @param data [String, Object] string or object that we want to send
-    # @note Note that if we pass object here (not a string), this method will invoke a #to_json
-    #   on it.
+    # @param options [Hash] options for waterdrop (e.g. partition_key)
     # @note Respond to does not accept multiple data arguments.
-    def respond_to(topic, data)
-      Karafka.monitor.notice(self.class, topic: topic, data: data)
+    def respond_to(topic, data, options = {})
+      Karafka.monitor.notice(self.class, topic: topic, data: data, options: options)
 
       messages_buffer[topic.to_s] ||= []
-      messages_buffer[topic.to_s] << (data.is_a?(String) ? data : data.to_json)
+      messages_buffer[topic.to_s] << [@parser_class.generate(data), options]
     end
 
     # Checks if we met all the topics requirements. It will fail if we didn't send a message to
@@ -122,7 +133,9 @@ module Karafka
     #   what we send is legit and it will go to a proper topics
     def deliver!
       messages_buffer.each do |topic, data_elements|
-        data_elements.each { |data| ::WaterDrop::Message.new(topic, data).send! }
+        data_elements.each do |(data, options)|
+          ::WaterDrop::Message.new(topic, data, options).send!
+        end
       end
     end
   end

data/lib/karafka/capistrano/karafka.cap

@@ -2,6 +2,7 @@
 # @see https://github.com/seuros/capistrano-puma/blob/master/lib/capistrano/tasks/puma.rake
 namespace :load do
   task :defaults do
+    set :karafka_role, :app
     set :karafka_default_hooks, -> { true }
     set :karafka_env, -> { fetch(:karafka_env, fetch(:environment)) }
     set :karafka_pid, -> { File.join(shared_path, 'tmp', 'pids', 'karafka.pid') }
@@ -17,7 +18,7 @@ end
 namespace :karafka do
   desc 'Stop Karafka'
   task :stop do
-    on roles(:app) do |host|
+    on roles(fetch(:karafka_role)) do |host|
       within shared_path do
         # If there's no pidfile it means that Karafka is not running
         next unless test "cat #{fetch(:karafka_pid)}"
@@ -40,7 +41,7 @@ namespace :karafka do
 
   desc 'Start Karafka'
   task :start do
-    on roles(:app) do |host|
+    on roles(fetch(:karafka_role)) do |host|
       within current_path do
         # We use all 3 because when combined with Sinatra/Rails it will use their parts as well
         # so we want to set proper env for any of them
@@ -63,7 +64,7 @@ namespace :karafka do
 
   desc 'Status Karafka'
   task :status do
-    on roles(:app) do |host|
+    on roles(fetch(:karafka_role)) do |host|
       if test "cat #{fetch(:karafka_pid)}"
         pid = capture "cat #{fetch(:karafka_pid)}"
 

data/lib/karafka/cli/info.rb

@@ -12,7 +12,8 @@ module Karafka
         info = [
           "Karafka framework version: #{Karafka::VERSION}",
           "Application name: #{config.name}",
-          "Inline mode: #{config.inline}",
+          "Inline mode: #{config.inline_mode}",
+          "Batch mode: #{config.batch_mode}",
           "Number of threads: #{config.concurrency}",
           "Boot file: #{Karafka.boot_file}",
           "Environment: #{Karafka.env}",

data/lib/karafka/connection/topic_consumer.rb

@@ -15,7 +15,9 @@ module Karafka
       # @yieldparam [Kafka::FetchedMessage] kafka fetched message
       # @note This will yield with a raw message - no preprocessing or reformatting
       def fetch_loop
-        kafka_consumer.each_message do |message|
+        send(
+          @route.batch_mode ? :consume_each_batch : :consume_each_message
+        ) do |message|
           yield(message)
         end
       end
@@ -28,27 +30,53 @@ module Karafka
 
       private
 
+      # Consumes messages from Kafka in batches
+      # @yieldparam [Kafka::FetchedMessage] kafka fetched message
+      def consume_each_batch
+        kafka_consumer.each_batch do |batch|
+          batch.messages.each do |message|
+            yield(message)
+          end
+        end
+      end
+
+      # Consumes messages from Kafka one by one
+      # @yieldparam [Kafka::FetchedMessage] kafka fetched message
+      def consume_each_message
+        kafka_consumer.each_message do |message|
+          yield(message)
+        end
+      end
+
       # @return [Kafka::Consumer] returns a ready to consume Kafka consumer
       #   that is set up to consume a given routes topic
       def kafka_consumer
-        return @kafka_consumer if @kafka_consumer
-
-        kafka = Kafka.new(
-          seed_brokers: ::Karafka::App.config.kafka.hosts,
-          logger: ::Karafka.logger,
-          client_id: ::Karafka::App.config.name
-        )
-
-        @kafka_consumer = kafka.consumer(
+        @kafka_consumer ||= kafka.consumer(
           group_id: @route.group,
           session_timeout: ::Karafka::App.config.kafka.session_timeout,
           offset_commit_interval: ::Karafka::App.config.kafka.offset_commit_interval,
           offset_commit_threshold: ::Karafka::App.config.kafka.offset_commit_threshold,
           heartbeat_interval: ::Karafka::App.config.kafka.heartbeat_interval
-        )
+        ).tap do |consumer|
+          consumer.subscribe(
+            @route.topic,
+            start_from_beginning: @route.start_from_beginning
+          )
+        end
+      end
 
-        @kafka_consumer.subscribe(@route.topic)
-        @kafka_consumer
+      # @return [Kafka] returns a Kafka
+      # @note We don't cache it internally because we cache kafka_consumer that uses kafka
+      #   object instance
+      def kafka
+        Kafka.new(
+          seed_brokers: ::Karafka::App.config.kafka.hosts,
+          logger: ::Karafka.logger,
+          client_id: ::Karafka::App.config.name,
+          ssl_ca_cert: ::Karafka::App.config.kafka.ssl.ca_cert,
+          ssl_client_cert: ::Karafka::App.config.kafka.ssl.client_cert,
+          ssl_client_cert_key: ::Karafka::App.config.kafka.ssl.client_cert_key
+        )
       end
     end
   end

data/lib/karafka/monitor.rb

@@ -73,8 +73,19 @@ module Karafka
     def caller_label
       # We need to calculate ancestors because if someone inherits
       # from this  class, caller chains is longer
-      index = self.class.ancestors.index(Karafka::Monitor) + 1
-      caller_locations(index, 2)[1].label
+      index = self.class.ancestors.index(Karafka::Monitor)
+      # caller_locations has a differs in result whether it is a subclass of
+      # Karafka::Monitor, the basic Karafka::Monitor itself or a super for a subclass.
+      # So to cover all the cases we need to differentiate.
+      # @see https://github.com/karafka/karafka/issues/128
+      # @note It won't work if the monitor caller_label caller class is defined using
+      #   define method
+      super_execution = caller_locations(1, 2)[0].label == caller_locations(1, 2)[1].label
+
+      scope = super_execution ? 1 : nil
+      scope ||= index.positive? ? 0 : 1
+
+      caller_locations(index + 1, 2)[scope].label
     end
 
     # @return [Logger] logger instance

data/lib/karafka/params/params.rb

@@ -92,7 +92,7 @@ module Karafka
       def parse(content)
         self[:parser].parse(content)
         # We catch both of them, because for default JSON - we use JSON parser directly
-      rescue ::Karafka::Errors::ParserError, JSON::ParserError => e
+      rescue ::Karafka::Errors::ParserError => e
         Karafka.monitor.notice_error(self.class, e)
         return { message: content }
       ensure

data/lib/karafka/parsers/json.rb

@@ -0,0 +1,36 @@
+module Karafka
+  # Module for all supported by default parsers for incoming/outgoing data
+  module Parsers
+    # Default Karafka Json parser for serializing and deserializing data
+    class Json
+      # @param content [String] content based on which we want to get our hash
+      # @return [Hash] hash with parsed JSON data
+      # @example
+      #   Json.parse("{\"a\":1}") #=> { 'a' => 1 }
+      def self.parse(content)
+        ::JSON.parse(content)
+      rescue ::JSON::ParserError => e
+        raise ::Karafka::Errors::ParserError, e
+      end
+
+      # @param content [Object] any object that we want to convert to a json string
+      # @return [String] Valid JSON string containing serialized data
+      # @raise [Karafka::Errors::ParserError] raised when we don't have a way to parse
+      #   given content to a json string format
+      # @note When string is passed to this method, we assume that it is already a json
+      #   string and we don't serialize it again. This allows us to serialize data before
+      #   it is being forwarded to a parser if we want to have a custom (not that simple)
+      #   json serialization
+      #
+      # @example From an ActiveRecord object
+      #   Json.generate(Repository.first) #=> "{\"repository\":{\"id\":\"04b504e0\"}}"
+      # @example From a string (no changes)
+      #   Json.generate("{\"a\":1}") #=> "{\"a\":1}"
+      def self.generate(content)
+        return content if content.is_a?(String)
+        return content.to_json if content.respond_to?(:to_json)
+        raise Karafka::Errors::ParserError, content
+      end
+    end
+  end
+end

data/lib/karafka/patches/dry_configurable.rb

@@ -0,0 +1,33 @@
+module Karafka
+  # Namespace for patches of external gems/libraries
+  module Patches
+    # Patch that will allow to use proc based lazy evaluated settings with Dry Configurable
+    # @see https://github.com/dry-rb/dry-configurable/blob/master/lib/dry/configurable.rb
+    module DryConfigurable
+      # We overwrite ::Dry::Configurable::Config to change on proc behaviour
+      # Unfortunately it does not provide an on call proc evaluation, so
+      # this feature had to be added here on demand/
+      # @param args Any arguments that DryConfigurable::Config accepts
+      def initialize(*args)
+        super
+
+        @config.each do |key, _value|
+          rebuild(key)
+        end
+      end
+
+      private
+
+      # Method that rebuilds a given accessor, so when it consists a proc value, it will
+      # evaluate it upon return
+      # @param method_name [Symbol] name of an accessor that we want to rebuild
+      def rebuild(method_name)
+        define_singleton_method method_name do
+          super().is_a?(Proc) ? super().call : super()
+        end
+      end
+    end
+  end
+end
+
+::Dry::Configurable::Config.prepend(Karafka::Patches::DryConfigurable)

data/lib/karafka/routing/builder.rb

@@ -18,7 +18,8 @@ module Karafka
         parser
         interchanger
         responder
-        inline
+        inline_mode
+        batch_mode
       ).freeze
 
       # All those options should be set on the route level

data/lib/karafka/routing/route.rb

@@ -9,8 +9,9 @@ module Karafka
     # - parser - What parsed do we want to use to unparse the data (optional)
     # - interchanger - What interchanger to encode/decode data do we want to use (optional)
     class Route
-      # Only ASCII alphanumeric characters and underscore and dash are allowed in topics and groups
-      NAME_FORMAT = /\A(\w|\-)+\z/
+      # Only ASCII alphanumeric characters, underscore, dash and dots
+      # are allowed in topics and groups
+      NAME_FORMAT = /\A(\w|\-|\.)+\z/
 
       # Options that we can set per each route
       ATTRIBUTES = %i(
@@ -20,7 +21,9 @@ module Karafka
         parser
         interchanger
         responder
-        inline
+        inline_mode
+        batch_mode
+        start_from_beginning
       ).freeze
 
       ATTRIBUTES.each { |attr| attr_writer(attr) }
@@ -63,9 +66,9 @@ module Karafka
       end
 
       # @return [Class] Parser class (not instance) that we want to use to unparse Kafka messages
-      # @note If not provided - will use JSON as default
+      # @note If not provided - will use Json as default
       def parser
-        @parser ||= JSON
+        @parser ||= Karafka::Parsers::Json
       end
 
       # @return [Class] Interchanger class (not an instance) that we want to use to interchange
@@ -77,9 +80,25 @@ module Karafka
       # @return [Boolean] Should we perform execution in the background (default) or
       #   inline. This can be set globally and overwritten by a per route setting
       # @note This method can be set to false, so direct assigment ||= would not work
-      def inline
-        return @inline unless @inline.nil?
-        @inline = Karafka::App.config.inline
+      def inline_mode
+        return @inline_mode unless @inline_mode.nil?
+        @inline_mode = Karafka::App.config.inline_mode
+      end
+
+      # @return [Boolean] Should the consumer handle incoming events one at a time, or in batch
+      def batch_mode
+        return @batch_mode unless @batch_mode.nil?
+        @batch_mode = Karafka::App.config.batch_mode
+      end
+
+      # For each topic subscription it's possible to decide whether to consume messages starting
+      # at the beginning of the topic or to just consume new messages that are produced to
+      # the topic.
+      # @return [Boolean] Should we consume from the beggining or from new incoming messages on
+      #   the first run
+      def start_from_beginning
+        return @start_from_beginning unless @start_from_beginning.nil?
+        @start_from_beginning = Karafka::App.config.start_from_beginning
       end
 
       # Checks if topic and group have proper format (acceptable by Kafka)

data/lib/karafka/setup/config.rb

@@ -15,8 +15,8 @@ module Karafka
       # Available settings
       # option name [String] current app name - used to provide default Kafka groups namespaces
       setting :name
-      # If inline is set to true, we won't enqueue jobs, instead we will run them immediately
-      setting :inline, false
+      # If inline_mode is set to true, we won't enqueue jobs, instead we will run them immediately
+      setting :inline_mode, false
       # option logger [Instance] logger that we want to use
       setting :logger, ::Karafka::Logger.instance
       # option monitor [Instance] monitor that we will to use (defaults to Karafka::Monitor)
@@ -25,6 +25,24 @@ module Karafka
       # Note that redis could be rewriten using nested options, but it is a sidekiq specific
       # stuff and we don't want to touch it
       setting :redis
+      # If batch_mode is true, incoming messages will be handled in batch, otherwsie one at a time.
+      setting :batch_mode, false
+      #  whether to consume messages starting at the beginning or to just consume new messages
+      setting :start_from_beginning, true
+
+      # Connection pool options are used for producer (Waterdrop)
+      # They are configured automatically based on Sidekiq concurrency and number of routes
+      # The bigger one is selected as we need to be able to send messages from both places
+      setting :connection_pool do
+        # Connection pool size for producers. Note that we take a bigger number because there
+        # are cases when we might have more sidekiq threads than Karafka routes (small app)
+        # or the opposite for bigger systems
+        setting :size, -> { [::Karafka::App.routes.count, Sidekiq.options[:concurrency]].max }
+        # How long should we wait for a working resource from the pool before rising timeout
+        # With a proper connection pool size, this should never happen
+        setting :timeout, 5
+      end
+
       # option kafka [Hash] - optional - kafka configuration options (hosts)
       setting :kafka do
         # Array with at least one host
@@ -42,6 +60,16 @@ module Karafka
         # option heartbeat_interval [Integer] the interval between heartbeats; must be less
         #   than the session window.
         setting :heartbeat_interval, 10
+
+        # SSL authentication related settings
+        setting :ssl do
+          # option ca_cert [String] SSL CA certificate
+          setting :ca_cert, nil
+          # option client_cert [String] SSL client certificate
+          setting :client_cert, nil
+          # option client_cert_key [String] SSL client certificate password
+          setting :client_cert_key, nil
+        end
       end
 
       # This is configured automatically, don't overwrite it!

data/lib/karafka/setup/configurators/water_drop.rb

@@ -7,8 +7,8 @@ module Karafka
         def setup
           ::WaterDrop.setup do |water_config|
             water_config.send_messages = true
-            water_config.connection_pool_size = config.concurrency
-            water_config.connection_pool_timeout = 1
+            water_config.connection_pool_size = config.connection_pool.size
+            water_config.connection_pool_timeout = config.connection_pool.timeout
             water_config.kafka.hosts = config.kafka.hosts
             water_config.raise_on_failure = true
           end

data/lib/karafka/templates/app.rb.example

@@ -13,7 +13,8 @@ class App < Karafka::App
     config.redis = {
       url: 'redis://localhost:6379'
     }
-    config.inline = false
+    config.inline_mode = false
+    config.batch_mode = false
   end
 
   routes.draw do

data/lib/karafka/version.rb

@@ -2,5 +2,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '0.5.0.1'
+  VERSION = '0.5.0.2'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 0.5.0.1
+  version: 0.5.0.2
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-25 00:00:00.000000000 Z
+date: 2017-02-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -102,14 +102,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: 0.3.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: 0.3.2.1
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -158,14 +158,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.7
+        version: '0.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.7
+        version: '0.5'
 description: " Framework used to simplify Apache Kafka based Ruby applications development "
 email:
 - maciej@mensfeld.pl
@@ -177,6 +177,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".rspec"
 - ".ruby-gemset"
 - ".ruby-version"
 - ".travis.yml"
@@ -217,7 +218,8 @@ files:
 - lib/karafka/monitor.rb
 - lib/karafka/params/interchanger.rb
 - lib/karafka/params/params.rb
-- lib/karafka/patches/dry/configurable/config.rb
+- lib/karafka/parsers/json.rb
+- lib/karafka/patches/dry_configurable.rb
 - lib/karafka/process.rb
 - lib/karafka/responders/builder.rb
 - lib/karafka/responders/topic.rb
@@ -262,7 +264,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.10
 signing_key: 
 specification_version: 4
 summary: Ruby based framework for working with Apache Kafka

data/lib/karafka/patches/dry/configurable/config.rb

@@ -1,37 +0,0 @@
-# Patch that will allow to use proc based lazy evaluated settings with Dry Configurable
-# @see https://github.com/dry-rb/dry-configurable/blob/master/lib/dry/configurable.rb
-module Dry
-  # Configurable module for Dry-Configurable
-  module Configurable
-    # Config node instance struct
-    class Config
-      # @param args [Array] All arguments that a Struct accepts
-      def initialize(*args)
-        super
-        setup_dynamics
-      end
-
-      private
-
-      # Method that sets up all the proc based lazy evaluated dynamic config values
-      def setup_dynamics
-        each_pair do |key, value|
-          next unless value.is_a?(Proc)
-
-          rebuild(key)
-        end
-      end
-
-      # Method that rebuilds a given accessor, so when it consists a proc value, it will
-      # evaluate it upon return
-      # @param method_name [Symbol] name of an accessor that we want to rebuild
-      def rebuild(method_name)
-        metaclass = class << self; self; end
-
-        metaclass.send(:define_method, method_name) do
-          super().is_a?(Proc) ? super().call : super()
-        end
-      end
-    end
-  end
-end