karafka 2.0.12 → 2.0.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2c407cd113e41314102fc910cf0b35e8081c55bdadae66635afe59491b58e390
-  data.tar.gz: 336dad28cc65218e33a2b3bc42b69a166ff4e8e6a10c65a435e57f75eca5ac90
+  metadata.gz: 0a38f7b176a40f09dc07c58b8abc8b13863f66fc2f3bff4d61fcd889c0bbb213
+  data.tar.gz: 9962f21cb52e566843e4b6994da8ac182752523a937cffa18bd735eefadc409f
 SHA512:
-  metadata.gz: 82ffee28acdaa1d126944426a7b31628380fc04aca24002ea55067f6c80a0b72a885d246a656913ea35e79fa82a9d741c35125e31b75abbbf7fb0022360e3b2b
-  data.tar.gz: b64f9dc4fb2461f3cbe3a774231a661365af5b1be88a6c95b1543521d19d8171514eb5111f88ceb53e309b5ab433f982b9535c2bbb4919e4c07850570424605a
+  metadata.gz: 5b722f8b0baced05f35e907dadb523c129fe12f64066f5efe9bd81fdc0634ccfb824d463baa8f34b7918410c3f6e8a451ceaf5645bb08a8adeaaccff99ec29f1
+  data.tar.gz: d2649dda560f9c171f804da107df33c1a7fdf81c21bb92d33b7b967aee950a042868c566b63fb32cd495d870e4164246f8ee15aca7c169c66fb82f0994b321ff

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Karafka framework changelog
 
+## 2.0.13 (2022-10-14)
+- Early exit upon attempts to commit current or earlier offset twice.
+- Add more integration specs covering edge cases.
+- Strip non producer related config when default producer is initialized (#776)
+
 ## 2.0.12 (2022-10-06)
 - Commit stored offsets upon rebalance revocation event to reduce number of messages that are re-processed.
 - Support cooperative-sticky rebalance strategy.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.12)
+    karafka (2.0.13)
       karafka-core (>= 2.0.2, < 3.0.0)
       rdkafka (>= 0.12)
       thor (>= 0.20)
@@ -30,7 +30,7 @@ GEM
       activesupport (>= 5.0)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.0.2)
+    karafka-core (2.0.3)
       concurrent-ruby (>= 1.1)
     mini_portile2 (2.8.0)
     minitest (5.16.3)

data/lib/karafka/base_consumer.rb

@@ -149,6 +149,9 @@ module Karafka
     #   processed but rather at the next one. This applies to both sync and async versions of this
     #   method.
     def mark_as_consumed(message)
+      # Ignore earlier offsets than the one we alread committed
+      return true if coordinator.seek_offset > message.offset
+
       unless client.mark_as_consumed(message)
         coordinator.revoke
 
@@ -166,6 +169,9 @@ module Karafka
     # @return [Boolean] true if we were able to mark the offset, false otherwise. False indicates
     #   that we were not able and that we have lost the partition.
     def mark_as_consumed!(message)
+      # Ignore earlier offsets than the one we alread committed
+      return true if coordinator.seek_offset > message.offset
+
       unless client.mark_as_consumed!(message)
         coordinator.revoke
 

data/lib/karafka/routing/subscription_group.rb

@@ -37,7 +37,7 @@ module Karafka
       #   topics but they lack the group.id (unless explicitly) provided. To make it compatible
       #   with our routing engine, we inject it before it will go to the consumer
       def kafka
-        kafka = @topics.first.kafka.dup
+        kafka = Setup::AttributesMap.consumer(@topics.first.kafka.dup)
 
         kafka[:'client.id'] ||= Karafka::App.config.client_id
         kafka[:'group.id'] ||= @topics.first.consumer_group.id

data/lib/karafka/setup/attributes_map.rb

@@ -0,0 +1,337 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Setup
+    # To simplify the overall design, in Karafka we define all the rdkafka settings in one scope
+    # under `kafka`. rdkafka though does not like when producer options are passed to the
+    # consumer configuration and issues warnings. This target map is used as a filtering layer, so
+    # only appropriate settings go to both producer and consumer
+    #
+    # It is built based on https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
+    module AttributesMap
+      # List of rdkafka consumer accepted attributes
+      CONSUMER = %i[
+        allow.auto.create.topics
+        api.version.fallback.ms
+        api.version.request
+        api.version.request.timeout.ms
+        auto.commit.enable
+        auto.commit.interval.ms
+        auto.offset.reset
+        background_event_cb
+        bootstrap.servers
+        broker.address.family
+        broker.address.ttl
+        broker.version.fallback
+        builtin.features
+        check.crcs
+        client.id
+        client.rack
+        closesocket_cb
+        connect_cb
+        connections.max.idle.ms
+        consume.callback.max.messages
+        consume_cb
+        coordinator.query.interval.ms
+        debug
+        default_topic_conf
+        enable.auto.commit
+        enable.auto.offset.store
+        enable.partition.eof
+        enable.random.seed
+        enable.sasl.oauthbearer.unsecure.jwt
+        enable.ssl.certificate.verification
+        enabled_events
+        error_cb
+        fetch.error.backoff.ms
+        fetch.max.bytes
+        fetch.message.max.bytes
+        fetch.min.bytes
+        fetch.wait.max.ms
+        group.id
+        group.instance.id
+        group.protocol.type
+        heartbeat.interval.ms
+        interceptors
+        internal.termination.signal
+        isolation.level
+        log.connection.close
+        log.queue
+        log.thread.name
+        log_cb
+        log_level
+        max.in.flight
+        max.in.flight.requests.per.connection
+        max.partition.fetch.bytes
+        max.poll.interval.ms
+        message.copy.max.bytes
+        message.max.bytes
+        metadata.broker.list
+        metadata.max.age.ms
+        oauthbearer_token_refresh_cb
+        offset.store.method
+        offset.store.path
+        offset.store.sync.interval.ms
+        offset_commit_cb
+        opaque
+        open_cb
+        partition.assignment.strategy
+        plugin.library.paths
+        queued.max.messages.kbytes
+        queued.min.messages
+        rebalance_cb
+        receive.message.max.bytes
+        reconnect.backoff.jitter.ms
+        reconnect.backoff.max.ms
+        reconnect.backoff.ms
+        sasl.kerberos.keytab
+        sasl.kerberos.kinit.cmd
+        sasl.kerberos.min.time.before.relogin
+        sasl.kerberos.principal
+        sasl.kerberos.service.name
+        sasl.mechanism
+        sasl.mechanisms
+        sasl.oauthbearer.client.id
+        sasl.oauthbearer.client.secret
+        sasl.oauthbearer.config
+        sasl.oauthbearer.extensions
+        sasl.oauthbearer.method
+        sasl.oauthbearer.scope
+        sasl.oauthbearer.token.endpoint.url
+        sasl.password
+        sasl.username
+        security.protocol
+        session.timeout.ms
+        socket.blocking.max.ms
+        socket.connection.setup.timeout.ms
+        socket.keepalive.enable
+        socket.max.fails
+        socket.nagle.disable
+        socket.receive.buffer.bytes
+        socket.send.buffer.bytes
+        socket.timeout.ms
+        socket_cb
+        ssl.ca.certificate.stores
+        ssl.ca.location
+        ssl.ca.pem
+        ssl.certificate.location
+        ssl.certificate.pem
+        ssl.certificate.verify_cb
+        ssl.cipher.suites
+        ssl.crl.location
+        ssl.curves.list
+        ssl.endpoint.identification.algorithm
+        ssl.engine.id
+        ssl.engine.location
+        ssl.key.location
+        ssl.key.password
+        ssl.key.pem
+        ssl.keystore.location
+        ssl.keystore.password
+        ssl.sigalgs.list
+        ssl_ca
+        ssl_certificate
+        ssl_engine_callback_data
+        ssl_key
+        statistics.interval.ms
+        stats_cb
+        throttle_cb
+        topic.blacklist
+        topic.metadata.propagation.max.ms
+        topic.metadata.refresh.fast.cnt
+        topic.metadata.refresh.fast.interval.ms
+        topic.metadata.refresh.interval.ms
+        topic.metadata.refresh.sparse
+      ].freeze
+
+      # List of rdkafka producer accepted attributes
+      PRODUCER = %i[
+        acks
+        api.version.fallback.ms
+        api.version.request
+        api.version.request.timeout.ms
+        background_event_cb
+        batch.num.messages
+        batch.size
+        bootstrap.servers
+        broker.address.family
+        broker.address.ttl
+        broker.version.fallback
+        builtin.features
+        client.id
+        client.rack
+        closesocket_cb
+        compression.codec
+        compression.level
+        compression.type
+        connect_cb
+        connections.max.idle.ms
+        debug
+        default_topic_conf
+        delivery.report.only.error
+        delivery.timeout.ms
+        dr_cb
+        dr_msg_cb
+        enable.gapless.guarantee
+        enable.idempotence
+        enable.random.seed
+        enable.sasl.oauthbearer.unsecure.jwt
+        enable.ssl.certificate.verification
+        enabled_events
+        error_cb
+        interceptors
+        internal.termination.signal
+        linger.ms
+        log.connection.close
+        log.queue
+        log.thread.name
+        log_cb
+        log_level
+        max.in.flight
+        max.in.flight.requests.per.connection
+        message.copy.max.bytes
+        message.max.bytes
+        message.send.max.retries
+        message.timeout.ms
+        metadata.broker.list
+        metadata.max.age.ms
+        msg_order_cmp
+        oauthbearer_token_refresh_cb
+        opaque
+        open_cb
+        partitioner
+        partitioner_cb
+        plugin.library.paths
+        produce.offset.report
+        queue.buffering.backpressure.threshold
+        queue.buffering.max.kbytes
+        queue.buffering.max.messages
+        queue.buffering.max.ms
+        queuing.strategy
+        receive.message.max.bytes
+        reconnect.backoff.jitter.ms
+        reconnect.backoff.max.ms
+        reconnect.backoff.ms
+        request.required.acks
+        request.timeout.ms
+        retries
+        retry.backoff.ms
+        sasl.kerberos.keytab
+        sasl.kerberos.kinit.cmd
+        sasl.kerberos.min.time.before.relogin
+        sasl.kerberos.principal
+        sasl.kerberos.service.name
+        sasl.mechanism
+        sasl.mechanisms
+        sasl.oauthbearer.client.id
+        sasl.oauthbearer.client.secret
+        sasl.oauthbearer.config
+        sasl.oauthbearer.extensions
+        sasl.oauthbearer.method
+        sasl.oauthbearer.scope
+        sasl.oauthbearer.token.endpoint.url
+        sasl.password
+        sasl.username
+        security.protocol
+        socket.blocking.max.ms
+        socket.connection.setup.timeout.ms
+        socket.keepalive.enable
+        socket.max.fails
+        socket.nagle.disable
+        socket.receive.buffer.bytes
+        socket.send.buffer.bytes
+        socket.timeout.ms
+        socket_cb
+        ssl.ca.certificate.stores
+        ssl.ca.location
+        ssl.ca.pem
+        ssl.certificate.location
+        ssl.certificate.pem
+        ssl.certificate.verify_cb
+        ssl.cipher.suites
+        ssl.crl.location
+        ssl.curves.list
+        ssl.endpoint.identification.algorithm
+        ssl.engine.id
+        ssl.engine.location
+        ssl.key.location
+        ssl.key.password
+        ssl.key.pem
+        ssl.keystore.location
+        ssl.keystore.password
+        ssl.sigalgs.list
+        ssl_ca
+        ssl_certificate
+        ssl_engine_callback_data
+        ssl_key
+        statistics.interval.ms
+        stats_cb
+        sticky.partitioning.linger.ms
+        throttle_cb
+        topic.blacklist
+        topic.metadata.propagation.max.ms
+        topic.metadata.refresh.fast.cnt
+        topic.metadata.refresh.fast.interval.ms
+        topic.metadata.refresh.interval.ms
+        topic.metadata.refresh.sparse
+        transaction.timeout.ms
+        transactional.id
+      ].freeze
+
+      # Location of the file with rdkafka settings list
+      SOURCE = 'https://raw.githubusercontent.com/edenhill/librdkafka/master/CONFIGURATION.md'
+
+      private_constant :SOURCE
+
+      class << self
+        # Filter the provided settings leaving only the once applicable to the consumer
+        # @param kafka_settings [Hash] all kafka settings
+        # @return [Hash] settings applicable to the consumer
+        def consumer(kafka_settings)
+          kafka_settings.slice(*CONSUMER)
+        end
+
+        # Filter the provided settings leaving only the once applicable to the producer
+        # @param kafka_settings [Hash] all kafka settings
+        # @return [Hash] settings applicable to the producer
+        def producer(kafka_settings)
+          kafka_settings.slice(*PRODUCER)
+        end
+
+        # @private
+        # @return [Hash<Symbol, Array<Symbol>>] hash with consumer and producer attributes list
+        #   that is sorted.
+        # @note This method should not be used directly. It is only used to generate appropriate
+        #   options list in case it would change
+        def generate
+          # Not used anywhere else, hence required here
+          require 'open-uri'
+
+          attributes = { consumer: Set.new, producer: Set.new }
+
+          ::URI.parse(SOURCE).open.readlines.each do |line|
+            next unless line.include?('|')
+
+            attribute, attribute_type = line.split('|').map(&:strip)
+
+            case attribute_type
+            when 'C'
+              attributes[:consumer] << attribute
+            when 'P'
+              attributes[:producer] << attribute
+            when '*'
+              attributes[:consumer] << attribute
+              attributes[:producer] << attribute
+            else
+              next
+            end
+          end
+
+          attributes.transform_values!(&:sort)
+          attributes.each_value { |vals| vals.map!(&:to_sym) }
+          attributes
+        end
+      end
+    end
+  end
+end

data/lib/karafka/setup/config.rb

@@ -20,7 +20,7 @@ module Karafka
       }.freeze
 
       # Contains settings that should not be used in production but make life easier in dev
-      DEV_DEFAULTS = {
+      KAFKA_DEV_DEFAULTS = {
         # Will create non-existing topics automatically.
         # Note that the broker needs to be configured with `auto.create.topics.enable=true`
         # While it is not recommended in prod, it simplifies work in dev
@@ -32,7 +32,7 @@ module Karafka
         'topic.metadata.refresh.interval.ms': 5_000
       }.freeze
 
-      private_constant :KAFKA_DEFAULTS, :DEV_DEFAULTS
+      private_constant :KAFKA_DEFAULTS, :KAFKA_DEV_DEFAULTS
 
       # Available settings
 
@@ -172,7 +172,7 @@ module Karafka
 
           return if Karafka::App.env.production?
 
-          DEV_DEFAULTS.each do |key, value|
+          KAFKA_DEV_DEFAULTS.each do |key, value|
             next if config.kafka.key?(key)
 
             config.kafka[key] = value
@@ -185,7 +185,7 @@ module Karafka
           config.producer ||= ::WaterDrop::Producer.new do |producer_config|
             # In some cases WaterDrop updates the config and we don't want our consumer config to
             # be polluted by those updates, that's why we copy
-            producer_config.kafka = config.kafka.dup
+            producer_config.kafka = AttributesMap.producer(config.kafka.dup)
             producer_config.logger = config.logger
           end
 

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.12'
+  VERSION = '2.0.13'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.12
+  version: 2.0.13
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2022-10-06 00:00:00.000000000 Z
+date: 2022-10-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -260,6 +260,7 @@ files:
 - lib/karafka/runner.rb
 - lib/karafka/serialization/json/deserializer.rb
 - lib/karafka/server.rb
+- lib/karafka/setup/attributes_map.rb
 - lib/karafka/setup/config.rb
 - lib/karafka/setup/dsl.rb
 - lib/karafka/status.rb