karafka 1.2.0 → 1.2.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0bb0a1f72768ebf4bf720ebda57ebc26a0178275adabffd494b24e1612e9b38a
-  data.tar.gz: f586038a0498227e8a287cc173a23e77bb99b6cb8d453c39be55a220e2a0d361
+  metadata.gz: d14a343dffbcc91c326964cc833bb82e9daedfdc4913eb6478c69e6caa5a995c
+  data.tar.gz: 14c3b5a9275dae6a5fb70790b4caf579cd2e1d9a6ac5d56307d950090012ec3d
 SHA512:
-  metadata.gz: e2b862da6372bc91f76bc01c20eb21ccde9d61d67749ccce97eadd2f739484dbd68a149d08d6e275d94d95b6ce9610d86c2d01e85bb1841a80e048ed832810ac
-  data.tar.gz: 06fb89700e59f810ec984b54f0424cfdeb98c79e17b8629010f011e68edff5fa1e345576a7b28ed07aff440739ed11824016da6083601a92b8f83fcca68e576e
+  metadata.gz: 890c1b249d3457984c2c392dc1bed2413f36704d1268247bd67b2fa70cd3489370786a71e6be6804aa9dc1e02136bc92e97a338c917577929f2a8e0aabaf07bc
+  data.tar.gz: cf9a0043a4b82797bf0f6d36a8ba18c97702e22dd9fea25d5c1df13ca3b30b4255244d828e6cd3fba68807d2717209ae626d2a99a25f497a7e15b04d80168d6f

data/.coditsu.yml

@@ -0,0 +1,3 @@
+api_key: <%= ENV['CODITSU_API_KEY'] %>
+api_secret: <%= ENV['CODITSU_API_SECRET'] %>
+repository_id: <%= ENV['CODITSU_REPOSITORY_ID'] %>

data/.ruby-version

@@ -1 +1 @@
-2.5.0
+2.6.1

data/.travis.yml

@@ -1,18 +1,49 @@
-language: ruby
+services:
+  - docker
+
+dist: trusty
 sudo: false
-rvm:
-  - 2.3.0
-  - 2.3.1
-  - 2.3.2
-  - 2.3.3
-  - 2.3.4
-  - 2.4.0
-  - 2.4.1
-  - 2.4.2
-  - 2.5.0
-  - jruby-head
-script: bundle exec rspec spec/
-env:
-  global:
-    - JRUBY_OPTS='--debug'
-install: bundle install --jobs=3 --retry=3
+cache: bundler
+
+git:
+  depth: false
+
+test: &test
+  stage: Test
+  language: ruby
+  before_install:
+    - gem install bundler
+    - gem update --system
+  script: bundle exec rspec
+
+jobs:
+  include:
+    - <<: *test
+      rvm: 2.6.1
+    - <<: *test
+      rvm: 2.5.3
+    - <<: *test
+      rvm: 2.4.5
+    - <<: *test
+      rvm: 2.3.8
+
+    - stage: coditsu
+      language: ruby
+      rvm: 2.6.1
+      before_install:
+        - gem update --system
+        - gem install bundler
+      before_script:
+        - docker create -v /sources --name sources alpine:3.4 /bin/true
+        - docker cp ./ sources:/sources
+      script: >
+        docker run
+        -e CODITSU_API_KEY
+        -e CODITSU_API_SECRET
+        -e CODITSU_REPOSITORY_ID
+        --volumes-from sources
+        coditsu/build-runner:latest
+
+stages:
+  - coditsu
+  - test

data/CHANGELOG.md

@@ -1,5 +1,54 @@
 # Karafka framework changelog
 
+## 1.2.13
+- Add support for parameter sasl_over_ssl in ruby-kafka
+
+## 1.2.12
+- #29 - Consumer class names must have the word "Consumer" in it in order to work (Sidekiq backend)
+
+## 1.2.11
+- [#470](https://github.com/karafka/karafka/issues/470) Karafka not working with dry-configurable 0.8
+
+## 1.2.10
+- [#453](https://github.com/karafka/karafka/pull/453) require `Forwardable` module
+
+## 1.2.9
+- Critical exceptions now will cause consumer to stop instead of retrying without a break
+- #412 - Fix dry-inflector dependency lock in gemspec
+- #414 - Backport to 1.2 the delayed retry upon failure
+- #437 - Raw message is no longer added to params after ParserError raised
+
+## 1.2.8
+- #408 - Responder Topic Lookup Bug on Heroku
+
+## 1.2.7
+- Unlock Ruby-kafka version with a warning
+
+## 1.2.6
+- Lock WaterDrop to 1.2.3
+- Lock Ruby-Kafka to 0.6.x (support for 0.7 will be added in Karafka 1.3)
+
+## 1.2.5
+- #354 - Expose consumer heartbeat
+- #373 - Async producer not working properly with responders
+
+## 1.2.4
+- #332 - Fetcher for max queue size
+
+## 1.2.3
+- #313 - support PLAINTEXT and SSL for scheme
+- #320 - Pausing indefinetely with nil pause timeout doesn't work
+- #318 - Partition pausing doesn't work with custom topic mappers
+- Rename ConfigAdapter to ApiAdapter to better reflect what it does
+- #317 - Manual offset committing doesn't work with custom topic mappers
+
+## 1.2.2
+- #312 - Broken for ActiveSupport 5.2.0
+
+## 1.2.1
+- #304 - Unification of error instrumentation event details
+- #306 - Using file logger from within a trap context upon shutdown is impossible
+
 ## 1.2.0
 - Spec improvements
 - #260 - Specs missing randomization

data/Gemfile

@@ -5,6 +5,10 @@ source 'https://rubygems.org'
 gemspec
 
 group :development, :test do
+  gem 'benchmark-ips'
+end
+
+group :test do
   gem 'rspec'
   gem 'simplecov'
   gem 'timecop'

data/Gemfile.lock

@@ -1,123 +1,126 @@
 PATH
   remote: .
   specs:
-    karafka (1.2.0)
+    karafka (1.2.13)
       activesupport (>= 4.0)
-      dry-configurable (~> 0.7)
-      dry-inflector (~> 0.1.1)
-      dry-monitor (~> 0.1)
+      dry-configurable (~> 0.8)
+      dry-inflector (~> 0.1)
+      dry-monitor (~> 0.3)
       dry-validation (~> 0.11)
       envlogic (~> 1.0)
       multi_json (>= 1.12)
       rake (>= 11.3)
       require_all (>= 1.4)
-      ruby-kafka (>= 0.5.3)
-      thor (~> 0.19)
-      waterdrop (~> 1.2)
+      ruby-kafka (>= 0.6)
+      thor (~> 0.20)
+      waterdrop (~> 1.2.4)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.1.5)
+    activesupport (5.2.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (~> 0.7)
+      i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-    concurrent-ruby (1.0.5)
-    delivery_boy (0.2.4)
-      king_konf (~> 0.1.8)
-      ruby-kafka (~> 0.5.1)
+    benchmark-ips (2.7.2)
+    concurrent-ruby (1.1.5)
+    delivery_boy (0.2.7)
+      king_konf (~> 0.2)
+      ruby-kafka (~> 0.5)
     diff-lcs (1.3)
-    docile (1.3.0)
-    dry-configurable (0.7.0)
+    digest-crc (0.4.1)
+    docile (1.3.1)
+    dry-configurable (0.8.2)
       concurrent-ruby (~> 1.0)
-    dry-container (0.6.0)
+      dry-core (~> 0.4, >= 0.4.7)
+    dry-container (0.7.0)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.1, >= 0.1.3)
-    dry-core (0.4.5)
+    dry-core (0.4.7)
       concurrent-ruby (~> 1.0)
-    dry-equalizer (0.2.0)
-    dry-events (0.1.0)
+    dry-equalizer (0.2.2)
+    dry-events (0.1.1)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.4)
       dry-equalizer (~> 0.2)
-    dry-inflector (0.1.1)
-    dry-logic (0.4.2)
-      dry-container (~> 0.2, >= 0.2.6)
+    dry-inflector (0.1.2)
+    dry-logic (0.6.1)
+      concurrent-ruby (~> 1.0)
       dry-core (~> 0.2)
       dry-equalizer (~> 0.2)
-    dry-monitor (0.1.2)
+    dry-monitor (0.3.0)
       dry-configurable (~> 0.5)
+      dry-core (~> 0.4)
       dry-equalizer (~> 0.2)
       dry-events (~> 0.1)
-      rouge (~> 2.0, >= 2.2.1)
-    dry-types (0.12.2)
+    dry-types (0.14.1)
       concurrent-ruby (~> 1.0)
-      dry-configurable (~> 0.1)
       dry-container (~> 0.3)
-      dry-core (~> 0.2, >= 0.2.1)
+      dry-core (~> 0.4, >= 0.4.4)
       dry-equalizer (~> 0.2)
-      dry-logic (~> 0.4, >= 0.4.2)
-      inflecto (~> 0.0.0, >= 0.0.2)
-    dry-validation (0.11.1)
+      dry-inflector (~> 0.1, >= 0.1.2)
+      dry-logic (~> 0.5, >= 0.5)
+    dry-validation (0.13.2)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.1, >= 0.1.3)
       dry-core (~> 0.2, >= 0.2.1)
       dry-equalizer (~> 0.2)
-      dry-logic (~> 0.4, >= 0.4.0)
-      dry-types (~> 0.12.0)
+      dry-logic (~> 0.5, >= 0.5.0)
+      dry-types (~> 0.14.0)
     envlogic (1.1.0)
       dry-inflector (~> 0.1)
-    i18n (0.9.5)
+    i18n (1.6.0)
       concurrent-ruby (~> 1.0)
-    inflecto (0.0.2)
     json (2.1.0)
-    king_konf (0.1.10)
+    king_konf (0.3.7)
     minitest (5.11.3)
     multi_json (1.13.1)
     null-logger (0.1.5)
-    rake (12.3.1)
+    rake (12.3.2)
     require_all (2.0.0)
-    rouge (2.2.1)
-    rspec (3.7.0)
-      rspec-core (~> 3.7.0)
-      rspec-expectations (~> 3.7.0)
-      rspec-mocks (~> 3.7.0)
-    rspec-core (3.7.1)
-      rspec-support (~> 3.7.0)
-    rspec-expectations (3.7.0)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-mocks (3.7.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-support (3.7.1)
-    ruby-kafka (0.5.4)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
+    ruby-kafka (0.7.7)
+      digest-crc
     simplecov (0.16.1)
       docile (~> 1.1)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
-    thor (0.20.0)
+    thor (0.20.3)
     thread_safe (0.3.6)
     timecop (0.9.1)
     tzinfo (1.2.5)
       thread_safe (~> 0.1)
-    waterdrop (1.2.0)
+    waterdrop (1.2.5)
       delivery_boy (~> 0.2)
-      dry-configurable (~> 0.7)
-      dry-monitor (~> 0.1)
+      dry-configurable (~> 0.8)
+      dry-monitor (~> 0.3)
       dry-validation (~> 0.11)
-      null-logger
+      null-logger (~> 0.1)
+      ruby-kafka (>= 0.6)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
+  benchmark-ips
   karafka!
   rspec
   simplecov
   timecop
 
 BUNDLED WITH
-   1.16.1
+   2.0.1

data/README.md

@@ -8,15 +8,13 @@ Karafka allows you to capture everything that happens in your systems in large s
 
 Karafka not only handles incoming messages but also provides tools for building complex data-flow applications that receive and send messages.
 
-**Warning**: Wiki and all the docs refer to the 1.2.0.beta4. Sorry for the inconvenience. We will release the stable 1.2.0 version soon.
-
 ## How does it work
 
-Karafka provides a higher-level abstraction that allows you to focus on your business logic development, instead of focusing on implementing lower level abstraction layers. It provides developers with a set of tools that are dedicated for building multi-topic applications similarly to how Rails applications are being built.
+Karafka provides a higher-level abstraction that allows you to focus on your business logic development, instead of focusing on implementing lower level abstraction layers. It provides developers with a set of tools that are dedicated for building multi-topic applications similar to how Rails applications are being built.
 
 ### Some things you might wonder about:
 
-- You can integrate Karafka with **any** Ruby based application.
+- You can integrate Karafka with **any** Ruby-based application.
 - Karafka does **not** require Sidekiq or any other third party software (apart from Kafka itself).
 - Karafka works with Ruby on Rails but it is a **standalone** framework that can work without it.
 - Karafka has a **minimal** set of dependencies, so adding it won't be a huge burden for your already existing applications.
@@ -27,15 +25,30 @@ Karafka based applications can be easily deployed to any type of infrastructure,
 * Heroku
 * Capistrano
 * Docker
+* Terraform
+
+## Kafka 0.10 or prior
+
+If you're using Kafka 0.10, please lock `ruby-kafka` gem in your Gemfile to version `0.6.8`:
+
+```ruby
+gem 'karafka'
+gem 'ruby-kafka', '~> 0.6.8'
+```
 
 ## Support
 
-Karafka has a [Wiki pages](https://github.com/karafka/karafka/wiki) for almost everything and a pretty decent [FAQ](https://github.com/karafka/karafka/wiki/FAQ). It covers the whole installation, setup and deployment along with other useful details on how to run Karafka.
+Karafka has a [Wiki pages](https://github.com/karafka/karafka/wiki) for almost everything and a pretty decent [FAQ](https://github.com/karafka/karafka/wiki/FAQ). It covers the whole installation, setup, and deployment along with other useful details on how to run Karafka.
 
 If you have any questions about using Karafka, feel free to join our [Gitter](https://gitter.im/karafka/karafka) chat channel.
 
 ## Getting started
 
+If you're completely new to the subject, you can start with our "Kafka on Rails" articles series, that will get you up and running with the terminology and basic ideas behind using Kafka:
+
+- [Kafka on Rails: Using Kafka with Ruby on Rails – Part 1 – Kafka basics and its advantages](https://mensfeld.pl/2017/11/kafka-on-rails-using-kafka-with-ruby-on-rails-part-1-kafka-basics-and-its-advantages/)
+- [Kafka on Rails: Using Kafka with Ruby on Rails – Part 2 – Getting started with Ruby and Kafka](https://mensfeld.pl/2018/01/kafka-on-rails-using-kafka-with-ruby-on-rails-part-2-getting-started-with-ruby-and-kafka/)
+
 If you want to get started with Kafka and Karafka as fast as possible, then the best idea is to just clone our example repository:
 
 ```bash
@@ -69,7 +82,7 @@ Karafka framework and Karafka team are __not__ related to Kafka streaming servic
 
 First, thank you for considering contributing to Karafka! It's people like you that make the open source community such a great community!
 
-Each pull request must pass all the rspec specs and meet our quality requirements.
+Each pull request must pass all the RSpec specs and meet our quality requirements.
 
 To check if everything is as it should be, we use [Coditsu](https://coditsu.io) that combines multiple linters and code analyzers for both code and documentation. Once you're done with your changes, submit a pull request.
 

data/karafka.gemspec

@@ -17,17 +17,22 @@ Gem::Specification.new do |spec|
   spec.license     = 'MIT'
 
   spec.add_dependency 'activesupport', '>= 4.0'
-  spec.add_dependency 'dry-configurable', '~> 0.7'
-  spec.add_dependency 'dry-inflector', '~> 0.1.1'
-  spec.add_dependency 'dry-monitor', '~> 0.1'
+  spec.add_dependency 'dry-configurable', '~> 0.8'
+  spec.add_dependency 'dry-inflector', '~> 0.1'
+  spec.add_dependency 'dry-monitor', '~> 0.3'
   spec.add_dependency 'dry-validation', '~> 0.11'
   spec.add_dependency 'envlogic', '~> 1.0'
   spec.add_dependency 'multi_json', '>= 1.12'
   spec.add_dependency 'rake', '>= 11.3'
   spec.add_dependency 'require_all', '>= 1.4'
-  spec.add_dependency 'ruby-kafka', '>= 0.5.3'
-  spec.add_dependency 'thor', '~> 0.19'
-  spec.add_dependency 'waterdrop', '~> 1.2'
+  spec.add_dependency 'ruby-kafka', '>= 0.6'
+  spec.add_dependency 'thor', '~> 0.20'
+  spec.add_dependency 'waterdrop', '~> 1.2.4'
+
+  spec.post_install_message = <<~MSG
+    \e[93mWarning:\e[0m If you're using Kafka 0.10, please lock ruby-kafka in your Gemfile to version '0.6.8':
+    gem 'ruby-kafka', '~> 0.6.8'
+  MSG
 
   spec.required_ruby_version = '>= 2.3.0'
 

data/lib/karafka.rb

@@ -6,6 +6,7 @@
   kafka
   envlogic
   thor
+  forwardable
   fileutils
   multi_json
   require_all

data/lib/karafka/attributes_map.rb

@@ -14,15 +14,15 @@ module Karafka
       # @note All other settings will be passed to Kafka.new method invocation.
       #   All elements in this hash are just edge cases
       # @return [Hash] hash with proper sections on what to proxy where in Ruby-Kafka
-      def config_adapter
+      def api_adapter
         {
           consumer: %i[
             session_timeout offset_commit_interval offset_commit_threshold
-            offset_retention_time heartbeat_interval
+            offset_retention_time heartbeat_interval fetcher_max_queue_size
           ],
-          subscription: %i[start_from_beginning max_bytes_per_partition],
-          consuming: %i[min_bytes max_bytes max_wait_time],
-          pausing: %i[pause_timeout],
+          subscribe: %i[start_from_beginning max_bytes_per_partition],
+          consumption: %i[min_bytes max_bytes max_wait_time],
+          pause: %i[pause_timeout],
           # All the options that are under kafka config namespace, but are not used
           # directly with kafka api, but from the Karafka user perspective, they are
           # still related to kafka. They should not be proxied anywhere
@@ -32,7 +32,7 @@ module Karafka
 
       # @return [Array<Symbol>] properties that can be set on a per topic level
       def topic
-        (config_adapter[:subscription] + %i[
+        (api_adapter[:subscribe] + %i[
           backend
           name
           parser
@@ -48,17 +48,19 @@ module Karafka
       #   Thanks to this solution, if any new setting is available for ruby-kafka, we just need
       #   to add it to our configuration class and it will be handled automatically.
       def consumer_group
-        # @note We don't ignore the config_adapter[:ignored] values as they should be ignored
+        # @note We don't ignore the api_adapter[:ignored] values as they should be ignored
         #   only when proxying details go ruby-kafka. We use ignored fields internally in karafka
-        ignored_settings = config_adapter[:subscription]
-        defined_settings = config_adapter.values.flatten
+        ignored_settings = api_adapter[:subscribe]
+        defined_settings = api_adapter.values.flatten
         karafka_settings = %i[batch_fetching]
         # This is a drity and bad hack of dry-configurable to get keys before setting values
         dynamically_proxied = Karafka::Setup::Config
                               ._settings
+                              .settings
                               .find { |s| s.name == :kafka }
                               .value
-                              .instance_variable_get('@klass').settings
+                              .names
+                              .to_a
 
         (defined_settings + dynamically_proxied).uniq + karafka_settings - ignored_settings
       end

data/lib/karafka/base_responder.rb

@@ -163,7 +163,11 @@ module Karafka
     def deliver!
       messages_buffer.each_value do |data_elements|
         data_elements.each do |data, options|
-          producer(options).call(data, options)
+          # We map this topic name, so it will match namespaced/etc topic in Kafka
+          # @note By default will not change topic (if default mapper used)
+          mapped_topic = Karafka::App.config.topic_mapper.outgoing(options[:topic])
+          external_options = options.merge(topic: mapped_topic)
+          producer(options).call(data, external_options)
         end
       end
     end
@@ -189,16 +193,16 @@ module Karafka
       messages_buffer[topic] ||= []
       messages_buffer[topic] << [
         @parser_class.generate(data),
-        # We map this topic name, so it will match namespaced/etc topic in Kafka
-        # @note By default will not change topic (if default mapper used)
-        options.merge(topic: Karafka::App.config.topic_mapper.outgoing(topic))
+        options.merge(topic: topic)
       ]
     end
 
     # @param options [Hash] options for waterdrop
     # @return [Class] WaterDrop producer (sync or async based on the settings)
     def producer(options)
-      options[:async] ? WaterDrop::AsyncProducer : WaterDrop::SyncProducer
+      self.class.topics[
+        options[:topic]
+      ].async? ? WaterDrop::AsyncProducer : WaterDrop::SyncProducer
     end
   end
 end

data/lib/karafka/connection/{config_adapter.rb → api_adapter.rb}

@@ -3,22 +3,22 @@
 module Karafka
   # Namespace for all the things related to Kafka connection
   module Connection
-    # Mapper used to convert our internal settings into ruby-kafka settings
+    # Mapper used to convert our internal settings into ruby-kafka settings based on their
+    # API requirements.
     # Since ruby-kafka has more and more options and there are few "levels" on which
     # we have to apply them (despite the fact, that in Karafka you configure all of it
     # in one place), we have to remap it into what ruby-kafka driver requires
     # @note The good thing about Kafka.new method is that it ignores all options that
     #   do nothing. So we don't have to worry about injecting our internal settings
     #   into the client and breaking stuff
-    module ConfigAdapter
+    module ApiAdapter
       class << self
         # Builds all the configuration settings for Kafka.new method
-        # @param _consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
         # @return [Array<Hash>] Array with all the client arguments including hash with all
         #   the settings required by Kafka.new method
         # @note We return array, so we can inject any arguments we want, in case of changes in the
         #   raw driver
-        def client(_consumer_group)
+        def client
           # This one is a default that takes all the settings except special
           # cases defined in the map
           settings = {
@@ -29,9 +29,9 @@ module Karafka
           kafka_configs.each do |setting_name, setting_value|
             # All options for config adapter should be ignored as we're just interested
             # in what is left, as we want to pass all the options that are "typical"
-            # and not listed in the config_adapter special cases mapping. All the values
-            # from the config_adapter mapping go somewhere else, not to the client directly
-            next if AttributesMap.config_adapter.values.flatten.include?(setting_name)
+            # and not listed in the api_adapter special cases mapping. All the values
+            # from the api_adapter mapping go somewhere else, not to the client directly
+            next if AttributesMap.api_adapter.values.flatten.include?(setting_name)
 
             settings[setting_name] = setting_value
           end
@@ -58,26 +58,54 @@ module Karafka
         # @return [Array<Hash>] Array with all the arguments required by consuming method
         #   including hash with all the settings required by
         #   Kafka::Consumer#consume_each_message and Kafka::Consumer#consume_each_batch method
-        def consuming(consumer_group)
-          settings = {
-            automatically_mark_as_processed: consumer_group.automatically_mark_as_consumed
-          }
-          [sanitize(fetch_for(:consuming, consumer_group, settings))]
+        def consumption(consumer_group)
+          [
+            sanitize(
+              fetch_for(
+                :consumption,
+                consumer_group,
+                automatically_mark_as_processed: consumer_group.automatically_mark_as_consumed
+              )
+            )
+          ]
         end
 
         # Builds all the configuration settings for kafka consumer#subscribe method
         # @param topic [Karafka::Routing::Topic] topic that holds details for a given subscription
         # @return [Hash] hash with all the settings required by kafka consumer#subscribe method
-        def subscription(topic)
-          settings = fetch_for(:subscription, topic)
+        def subscribe(topic)
+          settings = fetch_for(:subscribe, topic)
           [Karafka::App.config.topic_mapper.outgoing(topic.name), sanitize(settings)]
         end
 
         # Builds all the configuration settings required by kafka consumer#pause method
+        # @param topic [String] topic that we want to pause
+        # @param partition [Integer] number partition that we want to pause
         # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
-        # @return [Hash] hash with all the settings required to pause kafka consumer
-        def pausing(consumer_group)
-          { timeout: consumer_group.pause_timeout }
+        # @return [Array] array with all the details required to pause kafka consumer
+        def pause(topic, partition, consumer_group)
+          [
+            Karafka::App.config.topic_mapper.outgoing(topic),
+            partition,
+            { timeout: consumer_group.pause_timeout }
+          ]
+        end
+
+        # Remaps topic details taking the topic mapper feature into consideration.
+        # @param params [Karafka::Params::Params] params instance
+        # @return [Array] array with all the details needed by ruby-kafka to mark message
+        #   as processed
+        # @note When default empty topic mapper is used, no need for any conversion as the
+        #   internal and external format are exactly the same
+        def mark_message_as_processed(params)
+          # Majority of non heroku users don't use custom topic mappers. No need to change
+          # anything when it is a default mapper that does not change anything
+          return [params] if Karafka::App.config.topic_mapper == Karafka::Routing::TopicMapper
+
+          # @note We don't use tap as it is around 13% slower than non-dup version
+          dupped = params.dup
+          dupped['topic'] = Karafka::App.config.topic_mapper.outgoing(params.topic)
+          [dupped]
         end
 
         private
@@ -90,10 +118,10 @@ module Karafka
         def fetch_for(namespace_key, route_layer, preexisting_settings = {})
           kafka_configs.each_key do |setting_name|
             # Ignore settings that are not related to our namespace
-            next unless AttributesMap.config_adapter[namespace_key].include?(setting_name)
+            next unless AttributesMap.api_adapter[namespace_key].include?(setting_name)
             # Ignore settings that are already initialized
             # In case they are in preexisting settings fetched differently
-            next if preexisting_settings.keys.include?(setting_name)
+            next if preexisting_settings.key?(setting_name)
             # Fetch all the settings from a given layer object. Objects can handle the fallback
             # to the kafka settings, so
             preexisting_settings[setting_name] = route_layer.send(setting_name)

data/lib/karafka/connection/builder.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Builder used to construct Kafka client
+    module Builder
+      class << self
+        # Builds a Kafka::Cient instance that we use to work with Kafka cluster
+        # @return [::Kafka::Client] returns a Kafka client
+        def call
+          Kafka.new(*ApiAdapter.client)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/connection/client.rb

@@ -23,7 +23,7 @@ module Karafka
       # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
       # @note This will yield with raw messages - no preprocessing or reformatting.
       def fetch_loop
-        settings = ConfigAdapter.consuming(consumer_group)
+        settings = ApiAdapter.consumption(consumer_group)
 
         if consumer_group.batch_fetching
           kafka_consumer.each_batch(*settings) { |batch| yield(batch.messages) }
@@ -41,16 +41,6 @@ module Karafka
         )
         pause(error.topic, error.partition)
         retry
-        # This is on purpose - see the notes for this method
-        # rubocop:disable RescueException
-      rescue Exception => error
-        # rubocop:enable RescueException
-        Karafka.monitor.instrument(
-          'connection.client.fetch_loop.error',
-          caller: self,
-          error: error
-        )
-        retry
       end
 
       # Gracefuly stops topic consumption
@@ -66,10 +56,7 @@ module Karafka
       # @param topic [String] topic that we want to pause
       # @param partition [Integer] number partition that we want to pause
       def pause(topic, partition)
-        settings = ConfigAdapter.pausing(consumer_group)
-        timeout = settings[:timeout]
-        raise(Errors::InvalidPauseTimeout, timeout) unless timeout.positive?
-        kafka_consumer.pause(topic, partition, settings)
+        kafka_consumer.pause(*ApiAdapter.pause(topic, partition, consumer_group))
       end
 
       # Marks a given message as consumed and commit the offsets
@@ -77,12 +64,20 @@ module Karafka
       #   that offset commit happen asap in case of a crash
       # @param [Karafka::Params::Params] params message that we want to mark as processed
       def mark_as_consumed(params)
-        kafka_consumer.mark_message_as_processed(params)
+        kafka_consumer.mark_message_as_processed(
+          *ApiAdapter.mark_message_as_processed(params)
+        )
         # Trigger an immediate, blocking offset commit in order to minimize the risk of crashing
         # before the automatic triggers have kicked in.
         kafka_consumer.commit_offsets
       end
 
+      # Triggers a non-optional blocking heartbeat that notifies Kafka about the fact, that this
+      # consumer / client is still up and running
+      def trigger_heartbeat
+        kafka_consumer.trigger_heartbeat!
+      end
+
       private
 
       attr_reader :consumer_group
@@ -90,11 +85,13 @@ module Karafka
       # @return [Kafka::Consumer] returns a ready to consume Kafka consumer
       #   that is set up to consume from topics of a given consumer group
       def kafka_consumer
-        @kafka_consumer ||= kafka.consumer(
-          *ConfigAdapter.consumer(consumer_group)
+        # @note We don't cache the connection internally because we cache kafka_consumer that uses
+        #   kafka client object instance
+        @kafka_consumer ||= Builder.call.consumer(
+          *ApiAdapter.consumer(consumer_group)
         ).tap do |consumer|
           consumer_group.topics.each do |topic|
-            consumer.subscribe(*ConfigAdapter.subscription(topic))
+            consumer.subscribe(*ApiAdapter.subscribe(topic))
           end
         end
       rescue Kafka::ConnectionError
@@ -105,13 +102,6 @@ module Karafka
         # down the road
         raise
       end
-
-      # @return [Kafka] returns a Kafka
-      # @note We don't cache it internally because we cache kafka_consumer that uses kafka
-      #   object instance
-      def kafka
-        Kafka.new(*ConfigAdapter.client(consumer_group))
-      end
     end
   end
 end

data/lib/karafka/connection/listener.rb

@@ -46,8 +46,8 @@ module Karafka
       rescue Exception => e
         Karafka.monitor.instrument('connection.listener.fetch_loop.error', caller: self, error: e)
         # rubocop:enable RescueException
-        @client&.stop
-        retry if @client
+        @client.stop
+        sleep(@consumer_group.reconnect_timeout) && retry
       end
 
       # @return [Karafka::Connection::Client] wrapped kafka consuming client for a given topic

data/lib/karafka/consumers/callbacks.rb

@@ -23,7 +23,7 @@ module Karafka
           # @param method_name [Symbol, String] method name or nil if we plan to provide a block
           # @yield A block with a code that should be executed before scheduling
           define_method type do |method_name = nil, &block|
-            set_callback type, :before, method_name ? method_name : block
+            set_callback type, :before, method_name || block
           end
         end
       end

data/lib/karafka/errors.rb

@@ -41,10 +41,10 @@ module Karafka
     # This should never happen and if it does, please contact us
     MissingClient = Class.new(BaseError)
 
-    # Raised when we attemp to pause a partition but the pause timeout is equal to 0
-    InvalidPauseTimeout = Class.new(BaseError)
-
     # Raised when want to hook up to an event that is not registered and supported
     UnregisteredMonitorEvent = Class.new(BaseError)
+
+    # Raised when we've waited enough for shutting down an unresponding process
+    ForcefulShutdown = Class.new(BaseError)
   end
 end

data/lib/karafka/helpers/class_matcher.rb

@@ -42,6 +42,12 @@ module Karafka
       #   matcher.name #=> Super2Responder
       def name
         inflected = @klass.to_s.split('::').last.to_s
+        # We inject the from into the name just in case it is missing as in a situation like
+        # that it would just sanitize the name without adding the "to" postfix.
+        # It could create cases when we want to build for example a responder to a consumer
+        # that does not have the "Consumer" postfix and would do nothing returning the same name.
+        # That would be bad as the matching classes shouldn't be matched to themselves.
+        inflected << @from unless inflected.include?(@from)
         inflected.gsub!(@from, @to)
         inflected.gsub!(CONSTANT_REGEXP, '')
         inflected

data/lib/karafka/params/dsl.rb

@@ -93,7 +93,9 @@ module Karafka
         return self if self['parsed']
         self['parsed'] = true
 
-        merge!(parse(delete('value')))
+        parsed_data = parse(self['value'])
+        delete('value')
+        merge!(parsed_data)
       end
 
       # Includes and extends the base params klass with everything that is needed by Karafka to

data/lib/karafka/patches/dry_configurable.rb

@@ -10,10 +10,8 @@ module Karafka
       # Unfortunately it does not provide an on call proc evaluation, so
       # this feature had to be added here on demand/
       # @param args Any arguments that DryConfigurable::Config accepts
-      def initialize(*args)
-        super
-
-        @config.each_key(&method(:rebuild))
+      def define!(*args)
+        super.tap { @config.each_key(&method(:rebuild)) }
       end
 
       private

data/lib/karafka/process.rb

@@ -34,10 +34,8 @@ module Karafka
 
     # Method catches all HANDLED_SIGNALS and performs appropriate callbacks (if defined)
     # @note If there are no callbacks, this method will just ignore a given signal that was sent
-    # @yield [Block] block of code that we want to execute and supervise
     def supervise
       HANDLED_SIGNALS.each { |signal| trap_signal(signal) }
-      yield
     end
 
     private

data/lib/karafka/responders/builder.rb

@@ -3,7 +3,7 @@
 module Karafka
   # Responders namespace encapsulates all the internal responder implementation parts
   module Responders
-    # Responders builder is used to finding (based on the consumer class name) a responder
+    # Responders builder is used for finding (based on the consumer class name) a responder
     # that match the consumer. We use it when user does not provide a responder inside routing,
     # but he still names responder with the same convention (and namespaces) as consumer
     #

data/lib/karafka/schemas/consumer_group.rb

@@ -7,7 +7,7 @@ module Karafka
       # Valid uri schemas of Kafka broker url
       # The ||= is due to the behavior of require_all that resolves dependencies
       # but someetimes loads things twice
-      URI_SCHEMES ||= %w[kafka kafka+ssl].freeze
+      URI_SCHEMES ||= %w[kafka kafka+ssl plaintext ssl].freeze
 
       # Available sasl scram mechanism of authentication (plus nil)
       SASL_SCRAM_MECHANISMS ||= %w[sha256 sha512].freeze
@@ -31,11 +31,12 @@ module Karafka
       required(:id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
       required(:seed_brokers).filled { each(:broker_schema?) }
       required(:session_timeout).filled { int? | float? }
-      required(:pause_timeout).filled { (int? | float?) & gteq?(0) }
+      required(:pause_timeout) { none? | ((int? | float?) & gteq?(0)) }
       required(:offset_commit_interval) { int? | float? }
       required(:offset_commit_threshold).filled(:int?)
       required(:offset_retention_time) { none?.not > int? }
       required(:heartbeat_interval).filled { (int? | float?) & gteq?(0) }
+      required(:fetcher_max_queue_size).filled(:int?, gt?: 0)
       required(:connect_timeout).filled { (int? | float?) & gt?(0) }
       required(:socket_timeout).filled { (int? | float?) & gt?(0) }
       required(:min_bytes).filled(:int?, gt?: 0)
@@ -68,6 +69,7 @@ module Karafka
       end
 
       optional(:ssl_ca_certs_from_system).maybe(:bool?)
+      optional(:sasl_over_ssl).maybe(:bool?)
 
       # It's not with other encryptions as it has some more rules
       optional(:sasl_scram_mechanism)

data/lib/karafka/server.rb

@@ -19,9 +19,9 @@ module Karafka
 
       # Method which runs app
       def run
-        bind_on_sigint
-        bind_on_sigquit
-        bind_on_sigterm
+        process.on_sigint { stop_supervised }
+        process.on_sigquit { stop_supervised }
+        process.on_sigterm { stop_supervised }
         start_supervised
       end
 
@@ -39,35 +39,24 @@ module Karafka
         Karafka::Process.instance
       end
 
-      # What should happen when we decide to quit with sigint
-      def bind_on_sigint
-        process.on_sigint { stop_supervised }
-      end
-
-      # What should happen when we decide to quit with sigquit
-      def bind_on_sigquit
-        process.on_sigquit { stop_supervised }
-      end
-
-      # What should happen when we decide to quit with sigterm
-      def bind_on_sigterm
-        process.on_sigterm { stop_supervised }
-      end
-
       # Starts Karafka with a supervision
       # @note We don't need to sleep because Karafka::Fetcher is locking and waiting to
       # finish loop (and it won't happen until we explicitily want to stop)
       def start_supervised
-        process.supervise do
-          Karafka::App.run!
-          Karafka::Fetcher.call
-        end
+        process.supervise
+        Karafka::App.run!
+        Karafka::Fetcher.call
       end
 
       # Stops Karafka with a supervision (as long as there is a shutdown timeout)
       # If consumers won't stop in a given timeframe, it will force them to exit
       def stop_supervised
-        Karafka.monitor.instrument('server.stop', {})
+        # Because this is called in the trap context, there is a chance that instrumentation
+        # listeners contain things that aren't allowed from within a trap context.
+        # To bypass that (instead of telling users not to do things they need to)
+        # we spin up a thread to instrument server.stop and server.stop.error and wait until
+        # they're finished
+        Thread.new { Karafka.monitor.instrument('server.stop', {}) }.join
 
         Karafka::App.stop!
         # If there is no shutdown timeout, we don't exit and wait until all the consumers
@@ -82,7 +71,9 @@ module Karafka
           sleep SUPERVISION_SLEEP
         end
 
-        Karafka.monitor.instrument('server.stop.error', {})
+        raise Errors::ForcefulShutdown
+      rescue Errors::ForcefulShutdown => error
+        Thread.new { Karafka.monitor.instrument('server.stop.error', error: error) }.join
         # We're done waiting, lets kill them!
         consumer_threads.each(&:terminate)
 

data/lib/karafka/setup/config.rb

@@ -73,7 +73,7 @@ module Karafka
         # Time that a given partition will be paused from fetching messages, when message
         # consumption fails. It allows us to process other partitions, while the error is being
         # resolved and also "slows" things down, so it prevents from "eating" up all messages and
-        # consuming them with failed code
+        # consuming them with failed code. Use `nil` if you want to pause forever and never retry.
         setting :pause_timeout, 10
         # option offset_commit_interval [Integer] the interval between offset commits,
         #   in seconds.
@@ -85,6 +85,13 @@ module Karafka
         # option heartbeat_interval [Integer] the interval between heartbeats; must be less
         #   than the session window.
         setting :heartbeat_interval, 10
+        # option offset_retention_time [Integer] The length of the retention window, known as
+        #   offset retention time
+        setting :offset_retention_time, nil
+        # option fetcher_max_queue_size [Integer] max number of items in the fetch queue that
+        #   are stored for further processing. Note, that each item in the queue represents a
+        #   response from a single broker
+        setting :fetcher_max_queue_size, 100
         # option max_bytes_per_partition [Integer] the maximum amount of data fetched
         #   from a single partition at a time.
         setting :max_bytes_per_partition, 1_048_576
@@ -111,9 +118,6 @@ module Karafka
         # option reconnect_timeout [Integer] How long should we wait before trying to reconnect to
         # Kafka cluster that went down (in seconds)
         setting :reconnect_timeout, 5
-        # option offset_retention_time [Integer] The length of the retention window, known as
-        #   offset retention time
-        setting :offset_retention_time, nil
         # option connect_timeout [Integer] Sets the number of seconds to wait while connecting to
         # a broker for the first time. When ruby-kafka initializes, it needs to connect to at
         # least one host.
@@ -152,6 +156,8 @@ module Karafka
         setting :sasl_scram_password, nil
         # option sasl_scram_mechanism [String, nil] Scram mechanism, either 'sha256' or 'sha512'
         setting :sasl_scram_mechanism, nil
+        # option sasl_over_ssl [Boolean] whether to enforce SSL with SASL
+        setting :sasl_over_ssl, true
       end
 
       class << self

data/lib/karafka/setup/configurators/water_drop.rb

@@ -13,7 +13,7 @@ module Karafka
           ::WaterDrop.setup do |water_config|
             water_config.deliver = true
 
-            config.to_h.except(:kafka).each do |k, v|
+            config.to_h.reject { |k, _v| k == :kafka }.each do |k, v|
               key_assignment = :"#{k}="
               next unless water_config.respond_to?(key_assignment)
               water_config.public_send(key_assignment, v)

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '1.2.0'
+  VERSION = '1.2.13'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.2.13
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-23 00:00:00.000000000 Z
+date: 2019-05-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -32,42 +32,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '0.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '0.8'
 - !ruby/object:Gem::Dependency
   name: dry-inflector
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.1
+        version: '0.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.1
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: dry-monitor
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.3'
 - !ruby/object:Gem::Dependency
   name: dry-validation
   requirement: !ruby/object:Gem::Requirement
@@ -144,42 +144,42 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.3
+        version: '0.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.3
+        version: '0.6'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.19'
+        version: '0.20'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.19'
+        version: '0.20'
 - !ruby/object:Gem::Dependency
   name: waterdrop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: 1.2.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: 1.2.4
 description: Framework used to simplify Apache Kafka based Ruby applications development
 email:
 - maciej@coditsu.io
@@ -190,6 +190,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".coditsu.yml"
 - ".console_irbrc"
 - ".gitignore"
 - ".rspec"
@@ -222,8 +223,9 @@ files:
 - lib/karafka/cli/info.rb
 - lib/karafka/cli/install.rb
 - lib/karafka/cli/server.rb
+- lib/karafka/connection/api_adapter.rb
+- lib/karafka/connection/builder.rb
 - lib/karafka/connection/client.rb
-- lib/karafka/connection/config_adapter.rb
 - lib/karafka/connection/delegator.rb
 - lib/karafka/connection/listener.rb
 - lib/karafka/consumers/callbacks.rb
@@ -278,7 +280,8 @@ homepage: https://github.com/karafka/karafka
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message: "\e[93mWarning:\e[0m If you're using Kafka 0.10, please lock
+  ruby-kafka in your Gemfile to version '0.6.8':\ngem 'ruby-kafka', '~> 0.6.8'\n"
 rdoc_options: []
 require_paths:
 - lib
@@ -293,8 +296,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.1
 signing_key: 
 specification_version: 4
 summary: Ruby based framework for working with Apache Kafka