karafka 1.3.0 → 1.4.14

data/certs/mensfeld.pem

@@ -1,25 +1,26 @@
 -----BEGIN CERTIFICATE-----
-MIIEODCCAqCgAwIBAgIBATANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDDBhtYWNp
-ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMTkwNzMwMTQ1NDU0WhcNMjAwNzI5MTQ1
-NDU0WjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
-CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQC9fCwtaHZG2SyyNXiH8r0QbJQx/xxl
-dkvwWz9QGJO+O8rEx20FB1Ab+MVkfOscwIv5jWpmk1U9whzDPl1uFtIbgu+sk+Zb
-uQlZyK/DPN6c+/BbBL+RryTBRyvkPLoCVwm7uxc/JZ1n4AI6eF4cCZ2ieZ9QgQbU
-MQs2QPqs9hT50Ez/40GnOdadVfiDDGz+NME2C4ms0BriXwZ1tcRTfJIHe2xjIbbb
-y5qRGfsLKcgMzvLQR24olixyX1MR0s4+Wveq3QL/gBhL4veUcv+UABJA8IJR0kyB
-seHHutusiwZ1v3SjjjW1xLLrc2ARV0mgCb0WaK2T4iA3oFTGLh6Ydz8LNl31KQFv
-94nRd8IhmJxrhQ6dQ/WT9IXoa5S9lfT5lPJeINemH4/6QPABzf9W2IZlCdI9wCdB
-TBaw57MKneGAYZiKjw6OALSy2ltQUCl3RqFl3VP7n8uFy1U987Q5VIIQ3O1UUsQD
-Oe/h+r7GUU4RSPKgPlrwvW9bD/UQ+zF51v8CAwEAAaN3MHUwCQYDVR0TBAIwADAL
-BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFJNIBHdfEUD7TqHqIer2YhWaWhwcMB0GA1Ud
-EQQWMBSBEm1hY2llakBtZW5zZmVsZC5wbDAdBgNVHRIEFjAUgRJtYWNpZWpAbWVu
-c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBAKA4eqko6BTNhlysip6rfBkVTGri
-ZXsL+kRb2hLvsQJS/kLyM21oMlu+LN0aPj3qEFR8mE/YeDD8rLAfruBRTltPNbR7
-xA5eE1gkxY5LfExUtK3b2wPqfmo7mZgfcsMwfYg/tUXw1WpBCnrhAJodpGH6SXmp
-A40qFUZst0vjiOoO+aTblIHPmMJXoZ3K42dTlNKlEiDKUWMRKSgpjjYGEYalFNWI
-hHfCz2r8L2t+dYdMZg1JGbEkq4ADGsAA8ioZIpJd7V4hI17u5TCdi7X5wh/0gN0E
-CgP+nLox3D+l2q0QuQEkayr+auFYkzTCkF+BmEk1D0Ru4mcf3F4CJvEmW4Pzbjqt
-i1tsCWPtJ4E/UUKnKaWKqGbjrjHJ0MuShYzHkodox5IOiCXIQg+1+YSzfXUV6WEK
-KJG/fhg1JV5vVDdVy6x+tv5SQ5ctU0feCsVfESi3rE3zRd+nvzE9HcZ5aXeL1UtJ
-nT5Xrioegu2w1jPyVEgyZgTZC5rvD0nNS5sFNQ==
+MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
+YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
+MB4XDTIyMDgxOTE3MjEzN1oXDTIzMDgxOTE3MjEzN1owPzEQMA4GA1UEAwwHY29u
+dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
+bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAODzeO3L6lxdATzMHKNW
+jFA/GGunoPuylO/BMzy8RiQHh7VIvysAKs0tHhTx3g2D0STDpF+hcQcPELFikiT2
+F+1wOHj/SsrK7VKqfA8+gq04hKc5sQoX2Egf9k3V0YJ3eZ6R/koHkQ8A0TVt0w6F
+ZQckoV4MqnEAx0g/FZN3mnHTlJ3VFLSBqJEIe+S6FZMl92mSv+hTrlUG8VaYxSfN
+lTCvnKk284F6QZq5XIENLRmcDd/3aPBLnLwNnyMyhB+6gK8cUO+CFlDO5tjo/aBA
+rUnl++wGG0JooF1ed0v+evOn9KoMBG6rHewcf79qJbVOscbD8qSAmo+sCXtcFryr
+KRMTB8gNbowJkFRJDEe8tfRy11u1fYzFg/qNO82FJd62rKAw2wN0C29yCeQOPRb1
+Cw9Y4ZwK9VFNEcV9L+3pHTHn2XfuZHtDaG198VweiF6raFO4yiEYccodH/USP0L5
+cbcCFtmu/4HDSxL1ByQXO84A0ybJuk3/+aPUSXe9C9U8fwIDAQABo3cwdTAJBgNV
+HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUSlcEakb7gfn/5E2WY6z73BF/
+iZkwHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
+bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEA1aS+E7RXJ1w9g9mJ
+G0NzFxe64OEuENosNlvYQCbRKGCXAU1qqelYkBQHseRgRKxLICrnypRo9IEobyHa
+vDnJ4r7Tsb34dleqQW2zY/obG+cia3Ym2JsegXWF7dDOzCXJ4FN8MFoT2jHlqLLw
+yrap0YO5zx0GSQ0Dwy8h2n2v2vanMEeCx7iNm3ERgR5WuN5sjzWoz2A/JLEEcK0C
+EnAGKCWAd1fuG8IemDjT1edsd5FyYR4bIX0m+99oDuFZyPiiIbalmyYiSBBp59Yb
+Q0P8zeBi4OfwCZNcxqz0KONmw9JLNv6DgyEAH5xe/4JzhMEgvIRiPj0pHfA7oqQF
+KUNqvD1KlxbEC+bZfE5IZhnqYLdld/Ksqd22FI1RBhiS1Ejfsj99LVIm9cBuZEY2
+Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
+MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
 -----END CERTIFICATE-----

data/config/errors.yml

@@ -37,3 +37,5 @@ en:
         Unknown consumer group
       does_not_exist:
         Given file does not exist or cannot be read
+      does_not_respond_to_call: >
+        needs to respond to a #call method

data/docker-compose.yml

@@ -0,0 +1,17 @@
+version: '2'
+services:
+  zookeeper:
+    image: wurstmeister/zookeeper
+    ports:
+      - "2181:2181"
+  kafka:
+    image: wurstmeister/kafka:1.0.1
+    ports:
+      - "9092:9092"
+    environment:
+      KAFKA_ADVERTISED_HOST_NAME: localhost
+      KAFKA_ADVERTISED_PORT: 9092
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock

data/karafka.gemspec

@@ -11,26 +11,24 @@ Gem::Specification.new do |spec|
   spec.version     = ::Karafka::VERSION
   spec.platform    = Gem::Platform::RUBY
   spec.authors     = ['Maciej Mensfeld', 'Pavlo Vavruk', 'Adam Gwozdowski']
-  spec.email       = %w[maciej@coditsu.io pavlo.vavruk@gmail.com adam99g@gmail.com]
-  spec.homepage    = 'https://github.com/karafka/karafka'
+  spec.email       = %w[maciej@mensfeld.pl pavlo.vavruk@gmail.com adam99g@gmail.com]
+  spec.homepage    = 'https://karafka.io'
   spec.summary     = 'Ruby based framework for working with Apache Kafka'
   spec.description = 'Framework used to simplify Apache Kafka based Ruby applications development'
   spec.license     = 'MIT'
 
-  spec.add_dependency 'dry-configurable', '~> 0.8'
-  spec.add_dependency 'dry-inflector', '~> 0.1'
-  spec.add_dependency 'dry-monitor', '~> 0.3'
-  spec.add_dependency 'dry-validation', '~> 1.2'
+  spec.add_dependency 'concurrent-ruby'
+  spec.add_dependency 'dry-configurable', '~> 0.16'
+  spec.add_dependency 'dry-inflector', '~> 0.2'
+  spec.add_dependency 'dry-monitor', '~> 0.5'
+  spec.add_dependency 'dry-validation', '~> 1.7'
   spec.add_dependency 'envlogic', '~> 1.1'
-  spec.add_dependency 'irb', '~> 1.0'
-  spec.add_dependency 'multi_json', '>= 1.12'
-  spec.add_dependency 'rake', '>= 11.3'
-  spec.add_dependency 'ruby-kafka', '>= 0.7.8'
-  spec.add_dependency 'thor', '~> 0.20'
-  spec.add_dependency 'waterdrop', '~> 1.3.0'
-  spec.add_dependency 'zeitwerk', '~> 2.1'
+  spec.add_dependency 'ruby-kafka', '>= 1.3.0'
+  spec.add_dependency 'thor', '>= 1.1'
+  spec.add_dependency 'waterdrop', '~> 1.4'
+  spec.add_dependency 'zeitwerk', '~> 2.6'
 
-  spec.required_ruby_version = '>= 2.4.0'
+  spec.required_ruby_version = '>= 2.7'
 
   if $PROGRAM_NAME.end_with?('gem')
     spec.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')
@@ -40,5 +38,15 @@ Gem::Specification.new do |spec|
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = %w[lib]
+  spec.post_install_message = <<~MSG
+    WARN: Karafka 1.4 will reach the end of life soon.
+    We highly recommend updating to Karafka 2.0.
+    Visit this page for more details: https://karafka.io/docs/Versions-Lifecycle-and-EOL
+  MSG
+
+  spec.metadata = {
+    'source_code_uri' => 'https://github.com/karafka/karafka',
+    'rubygems_mfa_required' => 'true'
+  }
 end
 # rubocop:enable Metrics/BlockLength

data/lib/karafka/assignment_strategies/round_robin.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Strategies for Kafka partitions assignments
+  module AssignmentStrategies
+    # Standard RoundRobin strategy
+    class RoundRobin < SimpleDelegator
+      def initialize
+        super(Kafka::RoundRobinAssignmentStrategy.new)
+      end
+    end
+  end
+end

data/lib/karafka/attributes_map.rb

@@ -19,6 +19,7 @@ module Karafka
           consumer: %i[
             session_timeout offset_commit_interval offset_commit_threshold
             offset_retention_time heartbeat_interval fetcher_max_queue_size
+            assignment_strategy
           ],
           subscribe: %i[start_from_beginning max_bytes_per_partition],
           consumption: %i[min_bytes max_bytes max_wait_time],
@@ -52,14 +53,8 @@ module Karafka
         ignored_settings = api_adapter[:subscribe]
         defined_settings = api_adapter.values.flatten
         karafka_settings = %i[batch_fetching]
-        # This is a dirty and bad hack of dry-configurable to get keys before setting values
-        dynamically_proxied = Karafka::Setup::Config
-                              ._settings
-                              .settings
-                              .find { |s| s.name == :kafka }
-                              .value
-                              .names
-                              .to_a
+
+        dynamically_proxied = Karafka::Setup::Config.config.kafka.to_h.keys
 
         (defined_settings + dynamically_proxied).uniq + karafka_settings - ignored_settings
       end

data/lib/karafka/cli/base.rb

@@ -43,16 +43,16 @@ module Karafka
         end
 
         # Allows to set description of a given cli command
-        # @param desc [String] Description of a given cli command
-        def desc(desc)
-          @desc ||= desc
+        # @param args [Array] All the arguments that Thor desc method accepts
+        def desc(*args)
+          @desc ||= args
         end
 
         # This method will bind a given Cli command into Karafka Cli
         # This method is a wrapper to way Thor defines its commands
         # @param cli_class [Karafka::Cli] Karafka cli_class
         def bind_to(cli_class)
-          cli_class.desc name, @desc
+          cli_class.desc name, *@desc
 
           (@options || []).each { |option| cli_class.option(*option) }
 

data/lib/karafka/cli/flow.rb

@@ -11,19 +11,22 @@ module Karafka
       def call
         topics.each do |topic|
           any_topics = !topic.responder&.topics.nil?
+          log_messages = []
 
           if any_topics
-            puts "#{topic.name} =>"
+            log_messages << "#{topic.name} =>"
 
             topic.responder.topics.each_value do |responder_topic|
               features = []
               features << (responder_topic.required? ? 'always' : 'conditionally')
 
-              print responder_topic.name, "(#{features.join(', ')})"
+              log_messages << format(responder_topic.name, "(#{features.join(', ')})")
             end
           else
-            puts "#{topic.name} => (nothing)"
+            log_messages << "#{topic.name} => (nothing)"
           end
+
+          Karafka.logger.info(log_messages.join("\n"))
         end
       end
 
@@ -34,11 +37,11 @@ module Karafka
         Karafka::App.consumer_groups.map(&:topics).flatten.sort_by(&:name)
       end
 
-      # Prints a given value with label in a nice way
+      # Formats a given value with label in a nice way
       # @param label [String] label describing value
       # @param value [String] value that should be printed
-      def print(label, value)
-        printf "%-25s %s\n", "  - #{label}:", value
+      def format(label, value)
+        "  - #{label}:                #{value}"
       end
     end
   end

data/lib/karafka/cli/info.rb

@@ -24,7 +24,7 @@ module Karafka
           "Kafka seed brokers: #{config.kafka.seed_brokers}"
         ]
 
-        puts(info.join("\n"))
+        Karafka.logger.info(info.join("\n"))
       end
     end
   end

data/lib/karafka/cli/install.rb

@@ -13,7 +13,9 @@ module Karafka
       INSTALL_DIRS = %w[
         app/consumers
         app/responders
+        app/workers
         config
+        lib
         log
         tmp/pids
       ].freeze
@@ -28,11 +30,12 @@ module Karafka
       # @param args [Array] all the things that Thor CLI accepts
       def initialize(*args)
         super
-        @rails = Bundler::LockfileParser.new(
+        dependencies = Bundler::LockfileParser.new(
           Bundler.read_file(
             Bundler.default_lockfile
           )
-        ).dependencies.key?('rails')
+        ).dependencies
+        @rails = dependencies.key?('railties') || dependencies.key?('rails')
       end
 
       # Install all required things for Karafka application in current directory

data/lib/karafka/cli/missingno.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Karafka
+  class Cli < Thor
+    # Command that gets invoked when no method is provided when running the CLI
+    # It allows us to exit with exit code 1 instead of default 0 to indicate that something
+    #   was missing
+    # @see https://github.com/karafka/karafka/issues/619
+    class Missingno < Base
+      desc 'Hidden command that gets invoked when no command is provided', hide: true
+
+      # Prints an error about the lack of command (nothing selected)
+      def call
+        Karafka.logger.error('No command provided')
+        exit 1
+      end
+    end
+  end
+end

data/lib/karafka/cli/server.rb

@@ -31,14 +31,6 @@ module Karafka
         # part of the topics
         Karafka::Server.consumer_groups = cli.options[:consumer_groups]
 
-        # Remove pidfile on stop, just before the server instance is going to be GCed
-        # We want to delay the moment in which the pidfile is removed as much as we can,
-        # so instead of removing it after the server stops running, we rely on the gc moment
-        # when this object gets removed (it is a bit later), so it is closer to the actual
-        # system process end. We do that, so monitoring and deployment tools that rely on a pid
-        # won't alarm or start new system process up until the current one is finished
-        ObjectSpace.define_finalizer(self, proc { send(:clean) })
-
         Karafka::Server.run
       end
 
@@ -60,6 +52,14 @@ module Karafka
           cli.options[:pid],
           'w'
         ) { |file| file.write(::Process.pid) }
+
+        # Remove pidfile on stop, just before the server instance is going to be GCed
+        # We want to delay the moment in which the pidfile is removed as much as we can,
+        # so instead of removing it after the server stops running, we rely on the gc moment
+        # when this object gets removed (it is a bit later), so it is closer to the actual
+        # system process end. We do that, so monitoring and deployment tools that rely on a pid
+        # won't alarm or start new system process up until the current one is finished
+        ObjectSpace.define_finalizer(self, proc { send(:clean) })
       end
 
       # Removes a pidfile (if exist)

data/lib/karafka/cli.rb

@@ -10,6 +10,8 @@ module Karafka
   class Cli < Thor
     package_name 'Karafka'
 
+    default_task :missingno
+
     class << self
       # Loads all Cli commands into Thor framework
       # This method should be executed before we run Karafka::Cli.start, otherwise we won't
@@ -20,6 +22,12 @@ module Karafka
         end
       end
 
+      # When there is a CLI crash, exit
+      # @return [true]
+      def exit_on_failure?
+        true
+      end
+
       private
 
       # @return [Array<Class>] Array with Cli action classes that can be used as commands
@@ -47,7 +55,7 @@ end
 if ENV['KARAFKA_CONSOLE']
   # Reloads Karafka irb console session
   def reload!
-    puts "Reloading...\n"
+    Karafka.logger.info "Reloading...\n"
     Kernel.exec Karafka::Cli::Console.command
   end
 end

data/lib/karafka/connection/api_adapter.rb

@@ -14,11 +14,12 @@ module Karafka
     module ApiAdapter
       class << self
         # Builds all the configuration settings for Kafka.new method
+        # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
         # @return [Array<Hash>] Array with all the client arguments including hash with all
         #   the settings required by Kafka.new method
         # @note We return array, so we can inject any arguments we want, in case of changes in the
         #   raw driver
-        def client
+        def client(consumer_group)
           # This one is a default that takes all the settings except special
           # cases defined in the map
           settings = {
@@ -26,14 +27,17 @@ module Karafka
             client_id: ::Karafka::App.config.client_id
           }
 
-          kafka_configs.each do |setting_name, setting_value|
+          kafka_configs.each_key do |setting_name|
             # All options for config adapter should be ignored as we're just interested
             # in what is left, as we want to pass all the options that are "typical"
             # and not listed in the api_adapter special cases mapping. All the values
             # from the api_adapter mapping go somewhere else, not to the client directly
             next if AttributesMap.api_adapter.values.flatten.include?(setting_name)
 
-            settings[setting_name] = setting_value
+            # Settings for each consumer group are either defined per consumer group or are
+            # inherited from the global/general settings level, thus we don't have to fetch them
+            # from the kafka settings as they are already on a consumer group level
+            settings[setting_name] = consumer_group.public_send(setting_name)
           end
 
           settings_hash = sanitize(settings)
@@ -44,30 +48,28 @@ module Karafka
 
         # Builds all the configuration settings for kafka#consumer method
         # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
-        # @return [Array<Hash>] array with all the consumer arguments including hash with all
+        # @return [Hash] all the consumer keyword arguments including hash with all
         #   the settings required by Kafka#consumer
         def consumer(consumer_group)
           settings = { group_id: consumer_group.id }
           settings = fetch_for(:consumer, consumer_group, settings)
-          [sanitize(settings)]
+          sanitize(settings)
         end
 
         # Builds all the configuration settings for kafka consumer consume_each_batch and
         #   consume_each_message methods
         # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
-        # @return [Array<Hash>] Array with all the arguments required by consuming method
-        #   including hash with all the settings required by
+        # @return [Hash] hash with all the arguments required by consuming method
+        #   including all the settings required by
         #   Kafka::Consumer#consume_each_message and Kafka::Consumer#consume_each_batch method
         def consumption(consumer_group)
-          [
-            sanitize(
-              fetch_for(
-                :consumption,
-                consumer_group,
-                automatically_mark_as_processed: consumer_group.automatically_mark_as_consumed
-              )
+          sanitize(
+            fetch_for(
+              :consumption,
+              consumer_group,
+              automatically_mark_as_processed: consumer_group.automatically_mark_as_consumed
             )
-          ]
+          )
         end
 
         # Builds all the configuration settings for kafka consumer#subscribe method
@@ -82,17 +84,16 @@ module Karafka
         # @param topic [String] topic that we want to pause
         # @param partition [Integer] number partition that we want to pause
         # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
-        # @return [Array] array with all the details required to pause kafka consumer
+        # @return [Hash] hash with all the details required to pause kafka consumer
         def pause(topic, partition, consumer_group)
-          [
-            Karafka::App.config.topic_mapper.outgoing(topic),
-            partition,
-            {
+          {
+            args: [Karafka::App.config.topic_mapper.outgoing(topic), partition],
+            kwargs: {
               timeout: consumer_group.pause_timeout,
               max_timeout: consumer_group.pause_max_timeout,
               exponential_backoff: consumer_group.pause_exponential_backoff
             }
-          ]
+          }
         end
 
         # Remaps topic details taking the topic mapper feature into consideration.
@@ -105,11 +106,13 @@ module Karafka
           # Majority of users don't use custom topic mappers. No need to change anything when it
           # is a default mapper that does not change anything. Only some cloud providers require
           # topics to be remapped
-          return [params] if Karafka::App.config.topic_mapper.is_a?(Karafka::Routing::TopicMapper)
+          return [params.metadata] if Karafka::App.config.topic_mapper.is_a?(
+            Karafka::Routing::TopicMapper
+          )
 
           # @note We don't use tap as it is around 13% slower than non-dup version
-          dupped = params.dup
-          dupped['topic'] = Karafka::App.config.topic_mapper.outgoing(params.topic)
+          dupped = params.metadata.dup
+          dupped['topic'] = Karafka::App.config.topic_mapper.outgoing(params.metadata.topic)
           [dupped]
         end
 

data/lib/karafka/connection/batch_delegator.rb

@@ -23,7 +23,11 @@ module Karafka
           ) do
             # Due to how ruby-kafka is built, we have the metadata that is stored on the batch
             # level only available for batch consuming
-            consumer.metadata = Params::Builders::Metadata.from_kafka_batch(kafka_batch, topic)
+            consumer.batch_metadata = Params::Builders::BatchMetadata.from_kafka_batch(
+              kafka_batch,
+              topic
+            )
+
             kafka_messages = kafka_batch.messages
 
             # Depending on a case (persisted or not) we might use new consumer instance per

data/lib/karafka/connection/builder.rb

@@ -6,9 +6,16 @@ module Karafka
     module Builder
       class << self
         # Builds a Kafka::Client instance that we use to work with Kafka cluster
+        # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group for which we want
+        #   to have a new Kafka client
         # @return [::Kafka::Client] returns a Kafka client
-        def call
-          Kafka.new(*ApiAdapter.client)
+        def call(consumer_group)
+          settings = ApiAdapter.client(consumer_group)
+
+          Kafka.new(
+            settings[0],
+            **settings[1]
+          )
         end
       end
     end

data/lib/karafka/connection/client.rb

@@ -33,9 +33,9 @@ module Karafka
         settings = ApiAdapter.consumption(consumer_group)
 
         if consumer_group.batch_fetching
-          kafka_consumer.each_batch(*settings) { |batch| yield(batch, :batch) }
+          kafka_consumer.each_batch(**settings) { |batch| yield(batch, :batch) }
         else
-          kafka_consumer.each_message(*settings) { |message| yield(message, :message) }
+          kafka_consumer.each_message(**settings) { |message| yield(message, :message) }
         end
       # @note We catch only the processing errors as any other are considered critical (exceptions)
       #   and should require a client restart with a backoff
@@ -64,7 +64,8 @@ module Karafka
       # @param topic [String] topic that we want to pause
       # @param partition [Integer] number partition that we want to pause
       def pause(topic, partition)
-        kafka_consumer.pause(*ApiAdapter.pause(topic, partition, consumer_group))
+        args, kwargs = ApiAdapter.pause(topic, partition, consumer_group).values_at(:args, :kwargs)
+        kafka_consumer.pause(*args, **kwargs)
       end
 
       # Marks given message as consumed
@@ -97,11 +98,13 @@ module Karafka
       def kafka_consumer
         # @note We don't cache the connection internally because we cache kafka_consumer that uses
         #   kafka client object instance
-        @kafka_consumer ||= Builder.call.consumer(
-          *ApiAdapter.consumer(consumer_group)
+        @kafka_consumer ||= Builder.call(consumer_group).consumer(
+          **ApiAdapter.consumer(consumer_group)
         ).tap do |consumer|
           consumer_group.topics.each do |topic|
-            consumer.subscribe(*ApiAdapter.subscribe(topic))
+            settings = ApiAdapter.subscribe(topic)
+
+            consumer.subscribe(settings[0], **settings[1])
           end
         end
       rescue Kafka::ConnectionError

data/lib/karafka/connection/listener.rb

@@ -47,10 +47,10 @@ module Karafka
           end
         end
         # This is on purpose - see the notes for this method
-        # rubocop:disable RescueException
+        # rubocop:disable Lint/RescueException
       rescue Exception => e
         Karafka.monitor.instrument('connection.listener.fetch_loop.error', caller: self, error: e)
-        # rubocop:enable RescueException
+        # rubocop:enable Lint/RescueException
         # We can stop client without a problem, as it will reinitialize itself when running the
         # `fetch_loop` again
         @client.stop

data/lib/karafka/consumers/batch_metadata.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Consumers
+    # Brings the batch metadata into consumers that support batch_fetching
+    module BatchMetadata
+      attr_accessor :batch_metadata
+    end
+  end
+end

data/lib/karafka/consumers/includer.rb

@@ -16,7 +16,7 @@ module Karafka
 
           bind_backend(consumer, topic)
           bind_params(consumer, topic)
-          bind_metadata(consumer, topic)
+          bind_batch_metadata(consumer, topic)
           bind_responders(consumer, topic)
         end
 
@@ -40,13 +40,14 @@ module Karafka
           consumer.extend(SingleParams)
         end
 
-        # Adds an option to work with metadata for consumer instances that have batch fetching
+        # Adds an option to work with batch metadata for consumer instances that have
+        #   batch fetching enabled
         # @param consumer [Karafka::BaseConsumer] consumer instance
         # @param topic [Karafka::Routing::Topic] topic of a consumer class
-        def bind_metadata(consumer, topic)
+        def bind_batch_metadata(consumer, topic)
           return unless topic.batch_fetching
 
-          consumer.extend(Metadata)
+          consumer.extend(BatchMetadata)
         end
 
         # Adds responders support for topics and consumers with responders defined for them

data/lib/karafka/contracts/consumer_group.rb

@@ -32,6 +32,7 @@ module Karafka
         required(:offset_retention_time).maybe(:integer)
         required(:heartbeat_interval).filled { (int? | float?) & gteq?(0) }
         required(:fetcher_max_queue_size).filled(:int?, gt?: 0)
+        required(:assignment_strategy).value(:any)
         required(:connect_timeout).filled { (int? | float?) & gt?(0) }
         required(:reconnect_timeout).filled { (int? | float?) & gteq?(0) }
         required(:socket_timeout).filled { (int? | float?) & gt?(0) }
@@ -70,13 +71,13 @@ module Karafka
 
       # Uri rule to check if uri is in a Karafka acceptable format
       rule(:seed_brokers) do
-        if value&.is_a?(Array) && !value.all?(&method(:kafka_uri?))
+        if value.is_a?(Array) && !value.all?(&method(:kafka_uri?))
           key.failure(:invalid_broker_schema)
         end
       end
 
       rule(:topics) do
-        if value&.is_a?(Array)
+        if value.is_a?(Array)
           names = value.map { |topic| topic[:name] }
 
           key.failure(:topics_names_not_unique) if names.size != names.uniq.size
@@ -84,7 +85,7 @@ module Karafka
       end
 
       rule(:topics) do
-        if value&.is_a?(Array)
+        if value.is_a?(Array)
           value.each_with_index do |topic, index|
             TOPIC_CONTRACT.call(topic).errors.each do |error|
               key([:topics, index, error.path[0]]).failure(error.text)
@@ -93,6 +94,10 @@ module Karafka
         end
       end
 
+      rule(:assignment_strategy) do
+        key.failure(:does_not_respond_to_call) unless value.respond_to?(:call)
+      end
+
       rule(:ssl_client_cert, :ssl_client_cert_key) do
         if values[:ssl_client_cert] && !values[:ssl_client_cert_key]
           key(:ssl_client_cert_key).failure(:ssl_client_cert_with_ssl_client_cert_key)
@@ -178,9 +183,9 @@ module Karafka
       # @param value [String] potential RSA key value
       # @return [Boolean] is the given string a valid RSA key
       def valid_private_key?(value)
-        OpenSSL::PKey::RSA.new(value)
+        OpenSSL::PKey.read(value)
         true
-      rescue OpenSSL::PKey::RSAError
+      rescue OpenSSL::PKey::PKeyError
         false
       end