karafka 2.0.11 → 2.0.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 00fdec9b351cb2da897d141a0dbb97816d7b0e53d7548b86d1ff48c5a8d5af3a
-  data.tar.gz: 514964e409f4904932ee9e7cc1aa27094820d2cbe875c75ad49c82ce41b8afbf
+  metadata.gz: 0a38f7b176a40f09dc07c58b8abc8b13863f66fc2f3bff4d61fcd889c0bbb213
+  data.tar.gz: 9962f21cb52e566843e4b6994da8ac182752523a937cffa18bd735eefadc409f
 SHA512:
-  metadata.gz: ba9281c9bfdb3a7a2a4c4121c26d1debb4bd39af07a190172a15f34705ee65bd9a1c8ce2e3773ccb3fe749dc83025bf96e0dde31664f1c6541e5923cad87b7a5
-  data.tar.gz: 95f150e80a77f68c16f0ecce849403a7afdcaf50f67bdaa1471074173f8b27b5a6b66d8a5cda8c3dc39106d8153ba843a35c9123883dc803b7bb05ec2c1f47c3
+  metadata.gz: 5b722f8b0baced05f35e907dadb523c129fe12f64066f5efe9bd81fdc0634ccfb824d463baa8f34b7918410c3f6e8a451ceaf5645bb08a8adeaaccff99ec29f1
+  data.tar.gz: d2649dda560f9c171f804da107df33c1a7fdf81c21bb92d33b7b967aee950a042868c566b63fb32cd495d870e4164246f8ee15aca7c169c66fb82f0994b321ff

data/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Karafka framework changelog
 
+## 2.0.13 (2022-10-14)
+- Early exit upon attempts to commit current or earlier offset twice.
+- Add more integration specs covering edge cases.
+- Strip non producer related config when default producer is initialized (#776)
+
+## 2.0.12 (2022-10-06)
+- Commit stored offsets upon rebalance revocation event to reduce number of messages that are re-processed.
+- Support cooperative-sticky rebalance strategy.
+- Replace offset commit after each batch with a per-rebalance commit.
+- User instrumentation to publish internal rebalance errors.
+
 ## 2.0.11 (2022-09-29)
 - Report early on errors related to network and on max poll interval being exceeded to indicate critical problems that will be retries but may mean some underlying problems in the system.
 - Fix support of Ruby 2.7.0 to 2.7.2 (#1045)

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.11)
+    karafka (2.0.13)
       karafka-core (>= 2.0.2, < 3.0.0)
       rdkafka (>= 0.12)
       thor (>= 0.20)
@@ -30,7 +30,7 @@ GEM
       activesupport (>= 5.0)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.0.2)
+    karafka-core (2.0.3)
       concurrent-ruby (>= 1.1)
     mini_portile2 (2.8.0)
     minitest (5.16.3)
@@ -65,7 +65,7 @@ GEM
       karafka-core (>= 2.0.2, < 3.0.0)
       rdkafka (>= 0.10)
       zeitwerk (~> 2.3)
-    zeitwerk (2.6.0)
+    zeitwerk (2.6.1)
 
 PLATFORMS
   x86_64-linux

data/lib/karafka/base_consumer.rb

@@ -149,6 +149,9 @@ module Karafka
     #   processed but rather at the next one. This applies to both sync and async versions of this
     #   method.
     def mark_as_consumed(message)
+      # Ignore earlier offsets than the one we alread committed
+      return true if coordinator.seek_offset > message.offset
+
       unless client.mark_as_consumed(message)
         coordinator.revoke
 
@@ -166,6 +169,9 @@ module Karafka
     # @return [Boolean] true if we were able to mark the offset, false otherwise. False indicates
     #   that we were not able and that we have lost the partition.
     def mark_as_consumed!(message)
+      # Ignore earlier offsets than the one we alread committed
+      return true if coordinator.seek_offset > message.offset
+
       unless client.mark_as_consumed!(message)
         coordinator.revoke
 

data/lib/karafka/connection/listener.rb

@@ -118,10 +118,6 @@ module Karafka
           build_and_schedule_consumption_jobs
 
           wait
-
-          # We don't use the `#commit_offsets!` here for performance reasons. This can be achieved
-          # if needed by using manual offset management.
-          @client.commit_offsets
         end
 
         # If we are stopping we will no longer schedule any jobs despite polling.

data/lib/karafka/connection/rebalance_manager.rb

@@ -14,6 +14,9 @@ module Karafka
     #   that are lost, are those that got revoked but did not get re-assigned back. We do not
     #   expose this concept outside and we normalize to have them revoked, as it is irrelevant
     #   from the rest of the code perspective as only those that are lost are truly revoked.
+    #
+    # @note For cooperative-sticky `#assigned_partitions` holds only the recently assigned
+    #   partitions, not all the partitions that are owned
     class RebalanceManager
       # Empty array for internal usage not to create new objects
       EMPTY_ARRAY = [].freeze

data/lib/karafka/patches/rdkafka/bindings.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Patches
+    module Rdkafka
+      # Binding patches that slightly change how rdkafka operates in certain places
+      module Bindings
+        include ::Rdkafka::Bindings
+
+        # Alias internally
+        RB = ::Rdkafka::Bindings
+
+        class << self
+          # Handle assignments on cooperative rebalance
+          #
+          # @param client_ptr [FFI::Pointer]
+          # @param code [Integer]
+          # @param partitions_ptr [FFI::Pointer]
+          def on_cooperative_rebalance(client_ptr, code, partitions_ptr)
+            case code
+            when RB::RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
+              RB.rd_kafka_incremental_assign(client_ptr, partitions_ptr)
+            when RB::RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS
+              RB.rd_kafka_commit(client_ptr, nil, false)
+              RB.rd_kafka_incremental_unassign(client_ptr, partitions_ptr)
+            else
+              RB.rd_kafka_assign(client_ptr, FFI::Pointer::NULL)
+            end
+          end
+
+          # Handle assignments on a eager rebalance
+          #
+          # @param client_ptr [FFI::Pointer]
+          # @param code [Integer]
+          # @param partitions_ptr [FFI::Pointer]
+          def on_eager_rebalance(client_ptr, code, partitions_ptr)
+            case code
+            when RB::RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
+              RB.rd_kafka_assign(client_ptr, partitions_ptr)
+            when RB::RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS
+              RB.rd_kafka_commit(client_ptr, nil, false)
+              RB.rd_kafka_assign(client_ptr, FFI::Pointer::NULL)
+            else
+              RB.rd_kafka_assign(client_ptr, FFI::Pointer::NULL)
+            end
+          end
+
+          # Trigger Karafka callbacks
+          #
+          # @param code [Integer]
+          # @param opaque [Rdkafka::Opaque]
+          # @param consumer [Rdkafka::Consumer]
+          # @param tpl [Rdkafka::Consumer::TopicPartitionList]
+          def trigger_callbacks(code, opaque, consumer, tpl)
+            case code
+            when RB::RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
+              opaque.call_on_partitions_assigned(consumer, tpl)
+            when RB::RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS
+              opaque.call_on_partitions_revoked(consumer, tpl)
+            end
+          rescue StandardError => e
+            Karafka.monitor.instrument(
+              'error.occurred',
+              caller: self,
+              error: e,
+              type: 'connection.client.rebalance_callback.error'
+            )
+          end
+        end
+
+        # This patch changes few things:
+        # - it commits offsets (if any) upon partition revocation, so less jobs need to be
+        #   reprocessed if they are assigned to a different process
+        # - reports callback errors into the errors instrumentation instead of the logger
+        # - catches only StandardError instead of Exception as we fully control the directly
+        #   executed callbacks
+        #
+        # @see https://docs.confluent.io/2.0.0/clients/librdkafka/classRdKafka_1_1RebalanceCb.html
+        RebalanceCallback = FFI::Function.new(
+          :void, %i[pointer int pointer pointer]
+        ) do |client_ptr, code, partitions_ptr, opaque_ptr|
+          # Patch reference
+          pr = ::Karafka::Patches::Rdkafka::Bindings
+
+          if RB.rd_kafka_rebalance_protocol(client_ptr) == 'COOPERATIVE'
+            pr.on_cooperative_rebalance(client_ptr, code, partitions_ptr)
+          else
+            pr.on_eager_rebalance(client_ptr, code, partitions_ptr)
+          end
+
+          opaque = ::Rdkafka::Config.opaques[opaque_ptr.to_i]
+          return unless opaque
+
+          tpl = ::Rdkafka::Consumer::TopicPartitionList.from_native_tpl(partitions_ptr).freeze
+          consumer = ::Rdkafka::Consumer.new(client_ptr)
+
+          pr.trigger_callbacks(code, opaque, consumer, tpl)
+        end
+      end
+    end
+  end
+end
+
+# We need to replace the original callback with ours.
+# At the moment there is no API in rdkafka-ruby to do so
+::Rdkafka::Bindings.send(
+  :remove_const,
+  'RebalanceCallback'
+)
+
+::Rdkafka::Bindings.const_set(
+  'RebalanceCallback',
+  Karafka::Patches::Rdkafka::Bindings::RebalanceCallback
+)
+
+::Rdkafka::Bindings.attach_function(
+  :rd_kafka_rebalance_protocol,
+  %i[pointer],
+  :string
+)
+
+::Rdkafka::Bindings.attach_function(
+  :rd_kafka_incremental_assign,
+  %i[pointer pointer],
+  :string
+)
+
+::Rdkafka::Bindings.attach_function(
+  :rd_kafka_incremental_unassign,
+  %i[pointer pointer],
+  :string
+)

data/lib/karafka/routing/subscription_group.rb

@@ -37,7 +37,7 @@ module Karafka
       #   topics but they lack the group.id (unless explicitly) provided. To make it compatible
       #   with our routing engine, we inject it before it will go to the consumer
       def kafka
-        kafka = @topics.first.kafka.dup
+        kafka = Setup::AttributesMap.consumer(@topics.first.kafka.dup)
 
         kafka[:'client.id'] ||= Karafka::App.config.client_id
         kafka[:'group.id'] ||= @topics.first.consumer_group.id

data/lib/karafka/setup/attributes_map.rb

@@ -0,0 +1,337 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Setup
+    # To simplify the overall design, in Karafka we define all the rdkafka settings in one scope
+    # under `kafka`. rdkafka though does not like when producer options are passed to the
+    # consumer configuration and issues warnings. This target map is used as a filtering layer, so
+    # only appropriate settings go to both producer and consumer
+    #
+    # It is built based on https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
+    module AttributesMap
+      # List of rdkafka consumer accepted attributes
+      CONSUMER = %i[
+        allow.auto.create.topics
+        api.version.fallback.ms
+        api.version.request
+        api.version.request.timeout.ms
+        auto.commit.enable
+        auto.commit.interval.ms
+        auto.offset.reset
+        background_event_cb
+        bootstrap.servers
+        broker.address.family
+        broker.address.ttl
+        broker.version.fallback
+        builtin.features
+        check.crcs
+        client.id
+        client.rack
+        closesocket_cb
+        connect_cb
+        connections.max.idle.ms
+        consume.callback.max.messages
+        consume_cb
+        coordinator.query.interval.ms
+        debug
+        default_topic_conf
+        enable.auto.commit
+        enable.auto.offset.store
+        enable.partition.eof
+        enable.random.seed
+        enable.sasl.oauthbearer.unsecure.jwt
+        enable.ssl.certificate.verification
+        enabled_events
+        error_cb
+        fetch.error.backoff.ms
+        fetch.max.bytes
+        fetch.message.max.bytes
+        fetch.min.bytes
+        fetch.wait.max.ms
+        group.id
+        group.instance.id
+        group.protocol.type
+        heartbeat.interval.ms
+        interceptors
+        internal.termination.signal
+        isolation.level
+        log.connection.close
+        log.queue
+        log.thread.name
+        log_cb
+        log_level
+        max.in.flight
+        max.in.flight.requests.per.connection
+        max.partition.fetch.bytes
+        max.poll.interval.ms
+        message.copy.max.bytes
+        message.max.bytes
+        metadata.broker.list
+        metadata.max.age.ms
+        oauthbearer_token_refresh_cb
+        offset.store.method
+        offset.store.path
+        offset.store.sync.interval.ms
+        offset_commit_cb
+        opaque
+        open_cb
+        partition.assignment.strategy
+        plugin.library.paths
+        queued.max.messages.kbytes
+        queued.min.messages
+        rebalance_cb
+        receive.message.max.bytes
+        reconnect.backoff.jitter.ms
+        reconnect.backoff.max.ms
+        reconnect.backoff.ms
+        sasl.kerberos.keytab
+        sasl.kerberos.kinit.cmd
+        sasl.kerberos.min.time.before.relogin
+        sasl.kerberos.principal
+        sasl.kerberos.service.name
+        sasl.mechanism
+        sasl.mechanisms
+        sasl.oauthbearer.client.id
+        sasl.oauthbearer.client.secret
+        sasl.oauthbearer.config
+        sasl.oauthbearer.extensions
+        sasl.oauthbearer.method
+        sasl.oauthbearer.scope
+        sasl.oauthbearer.token.endpoint.url
+        sasl.password
+        sasl.username
+        security.protocol
+        session.timeout.ms
+        socket.blocking.max.ms
+        socket.connection.setup.timeout.ms
+        socket.keepalive.enable
+        socket.max.fails
+        socket.nagle.disable
+        socket.receive.buffer.bytes
+        socket.send.buffer.bytes
+        socket.timeout.ms
+        socket_cb
+        ssl.ca.certificate.stores
+        ssl.ca.location
+        ssl.ca.pem
+        ssl.certificate.location
+        ssl.certificate.pem
+        ssl.certificate.verify_cb
+        ssl.cipher.suites
+        ssl.crl.location
+        ssl.curves.list
+        ssl.endpoint.identification.algorithm
+        ssl.engine.id
+        ssl.engine.location
+        ssl.key.location
+        ssl.key.password
+        ssl.key.pem
+        ssl.keystore.location
+        ssl.keystore.password
+        ssl.sigalgs.list
+        ssl_ca
+        ssl_certificate
+        ssl_engine_callback_data
+        ssl_key
+        statistics.interval.ms
+        stats_cb
+        throttle_cb
+        topic.blacklist
+        topic.metadata.propagation.max.ms
+        topic.metadata.refresh.fast.cnt
+        topic.metadata.refresh.fast.interval.ms
+        topic.metadata.refresh.interval.ms
+        topic.metadata.refresh.sparse
+      ].freeze
+
+      # List of rdkafka producer accepted attributes
+      PRODUCER = %i[
+        acks
+        api.version.fallback.ms
+        api.version.request
+        api.version.request.timeout.ms
+        background_event_cb
+        batch.num.messages
+        batch.size
+        bootstrap.servers
+        broker.address.family
+        broker.address.ttl
+        broker.version.fallback
+        builtin.features
+        client.id
+        client.rack
+        closesocket_cb
+        compression.codec
+        compression.level
+        compression.type
+        connect_cb
+        connections.max.idle.ms
+        debug
+        default_topic_conf
+        delivery.report.only.error
+        delivery.timeout.ms
+        dr_cb
+        dr_msg_cb
+        enable.gapless.guarantee
+        enable.idempotence
+        enable.random.seed
+        enable.sasl.oauthbearer.unsecure.jwt
+        enable.ssl.certificate.verification
+        enabled_events
+        error_cb
+        interceptors
+        internal.termination.signal
+        linger.ms
+        log.connection.close
+        log.queue
+        log.thread.name
+        log_cb
+        log_level
+        max.in.flight
+        max.in.flight.requests.per.connection
+        message.copy.max.bytes
+        message.max.bytes
+        message.send.max.retries
+        message.timeout.ms
+        metadata.broker.list
+        metadata.max.age.ms
+        msg_order_cmp
+        oauthbearer_token_refresh_cb
+        opaque
+        open_cb
+        partitioner
+        partitioner_cb
+        plugin.library.paths
+        produce.offset.report
+        queue.buffering.backpressure.threshold
+        queue.buffering.max.kbytes
+        queue.buffering.max.messages
+        queue.buffering.max.ms
+        queuing.strategy
+        receive.message.max.bytes
+        reconnect.backoff.jitter.ms
+        reconnect.backoff.max.ms
+        reconnect.backoff.ms
+        request.required.acks
+        request.timeout.ms
+        retries
+        retry.backoff.ms
+        sasl.kerberos.keytab
+        sasl.kerberos.kinit.cmd
+        sasl.kerberos.min.time.before.relogin
+        sasl.kerberos.principal
+        sasl.kerberos.service.name
+        sasl.mechanism
+        sasl.mechanisms
+        sasl.oauthbearer.client.id
+        sasl.oauthbearer.client.secret
+        sasl.oauthbearer.config
+        sasl.oauthbearer.extensions
+        sasl.oauthbearer.method
+        sasl.oauthbearer.scope
+        sasl.oauthbearer.token.endpoint.url
+        sasl.password
+        sasl.username
+        security.protocol
+        socket.blocking.max.ms
+        socket.connection.setup.timeout.ms
+        socket.keepalive.enable
+        socket.max.fails
+        socket.nagle.disable
+        socket.receive.buffer.bytes
+        socket.send.buffer.bytes
+        socket.timeout.ms
+        socket_cb
+        ssl.ca.certificate.stores
+        ssl.ca.location
+        ssl.ca.pem
+        ssl.certificate.location
+        ssl.certificate.pem
+        ssl.certificate.verify_cb
+        ssl.cipher.suites
+        ssl.crl.location
+        ssl.curves.list
+        ssl.endpoint.identification.algorithm
+        ssl.engine.id
+        ssl.engine.location
+        ssl.key.location
+        ssl.key.password
+        ssl.key.pem
+        ssl.keystore.location
+        ssl.keystore.password
+        ssl.sigalgs.list
+        ssl_ca
+        ssl_certificate
+        ssl_engine_callback_data
+        ssl_key
+        statistics.interval.ms
+        stats_cb
+        sticky.partitioning.linger.ms
+        throttle_cb
+        topic.blacklist
+        topic.metadata.propagation.max.ms
+        topic.metadata.refresh.fast.cnt
+        topic.metadata.refresh.fast.interval.ms
+        topic.metadata.refresh.interval.ms
+        topic.metadata.refresh.sparse
+        transaction.timeout.ms
+        transactional.id
+      ].freeze
+
+      # Location of the file with rdkafka settings list
+      SOURCE = 'https://raw.githubusercontent.com/edenhill/librdkafka/master/CONFIGURATION.md'
+
+      private_constant :SOURCE
+
+      class << self
+        # Filter the provided settings leaving only the once applicable to the consumer
+        # @param kafka_settings [Hash] all kafka settings
+        # @return [Hash] settings applicable to the consumer
+        def consumer(kafka_settings)
+          kafka_settings.slice(*CONSUMER)
+        end
+
+        # Filter the provided settings leaving only the once applicable to the producer
+        # @param kafka_settings [Hash] all kafka settings
+        # @return [Hash] settings applicable to the producer
+        def producer(kafka_settings)
+          kafka_settings.slice(*PRODUCER)
+        end
+
+        # @private
+        # @return [Hash<Symbol, Array<Symbol>>] hash with consumer and producer attributes list
+        #   that is sorted.
+        # @note This method should not be used directly. It is only used to generate appropriate
+        #   options list in case it would change
+        def generate
+          # Not used anywhere else, hence required here
+          require 'open-uri'
+
+          attributes = { consumer: Set.new, producer: Set.new }
+
+          ::URI.parse(SOURCE).open.readlines.each do |line|
+            next unless line.include?('|')
+
+            attribute, attribute_type = line.split('|').map(&:strip)
+
+            case attribute_type
+            when 'C'
+              attributes[:consumer] << attribute
+            when 'P'
+              attributes[:producer] << attribute
+            when '*'
+              attributes[:consumer] << attribute
+              attributes[:producer] << attribute
+            else
+              next
+            end
+          end
+
+          attributes.transform_values!(&:sort)
+          attributes.each_value { |vals| vals.map!(&:to_sym) }
+          attributes
+        end
+      end
+    end
+  end
+end

data/lib/karafka/setup/config.rb

@@ -20,7 +20,7 @@ module Karafka
       }.freeze
 
       # Contains settings that should not be used in production but make life easier in dev
-      DEV_DEFAULTS = {
+      KAFKA_DEV_DEFAULTS = {
         # Will create non-existing topics automatically.
         # Note that the broker needs to be configured with `auto.create.topics.enable=true`
         # While it is not recommended in prod, it simplifies work in dev
@@ -32,7 +32,7 @@ module Karafka
         'topic.metadata.refresh.interval.ms': 5_000
       }.freeze
 
-      private_constant :KAFKA_DEFAULTS, :DEV_DEFAULTS
+      private_constant :KAFKA_DEFAULTS, :KAFKA_DEV_DEFAULTS
 
       # Available settings
 
@@ -172,7 +172,7 @@ module Karafka
 
           return if Karafka::App.env.production?
 
-          DEV_DEFAULTS.each do |key, value|
+          KAFKA_DEV_DEFAULTS.each do |key, value|
             next if config.kafka.key?(key)
 
             config.kafka[key] = value
@@ -185,7 +185,7 @@ module Karafka
           config.producer ||= ::WaterDrop::Producer.new do |producer_config|
             # In some cases WaterDrop updates the config and we don't want our consumer config to
             # be polluted by those updates, that's why we copy
-            producer_config.kafka = config.kafka.dup
+            producer_config.kafka = AttributesMap.producer(config.kafka.dup)
             producer_config.logger = config.logger
           end
 

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.11'
+  VERSION = '2.0.13'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.11
+  version: 2.0.13
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2022-09-29 00:00:00.000000000 Z
+date: 2022-10-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -212,6 +212,7 @@ files:
 - lib/karafka/messages/messages.rb
 - lib/karafka/messages/metadata.rb
 - lib/karafka/messages/seek.rb
+- lib/karafka/patches/rdkafka/bindings.rb
 - lib/karafka/patches/rdkafka/consumer.rb
 - lib/karafka/pro.rb
 - lib/karafka/pro/active_job/consumer.rb
@@ -259,6 +260,7 @@ files:
 - lib/karafka/runner.rb
 - lib/karafka/serialization/json/deserializer.rb
 - lib/karafka/server.rb
+- lib/karafka/setup/attributes_map.rb
 - lib/karafka/setup/config.rb
 - lib/karafka/setup/dsl.rb
 - lib/karafka/status.rb