karafka 2.4.3 → 2.4.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c37b855609a9d20ca0b5cf13875d30ccb71b4072b3f07886f10445ce761c6e1e
-  data.tar.gz: 852fa3340f0dd1cbc2286823a2451c635568440c3e753e7235a72dc85289fcf5
+  metadata.gz: fe76e139b2007e11b0cb9393bc94e2b5b9cad5dc40d69b5ad2bfa34a0555e2b8
+  data.tar.gz: e288c63648d17388f577a45b51267e30fceb4726def0a42c40a9133de27d93f8
 SHA512:
-  metadata.gz: 373b460007cefdb039b5f2fd00abdc722f42c5bd8f324dad121c7e293264ca4a63cb63cb4fa5057e0f2ff856fe6c28073c04d2ffb21250dd534408ee511807c9
-  data.tar.gz: 595e408578801d949bb2b1e8d3337e8535473b80c7a1db34e56822214de9fd2083a594433e264ed8675c2292cd3aa73bacaef44938ad7a4cd743292097fd4d8f
+  metadata.gz: b5e46439a216e66c728ceb74070832407d39b1ebe06d21c858f3e3a0e8e599df15b749318131aa82a5bc994e6dd7ed3094f44639e706a57231f031f0e52186b6
+  data.tar.gz: dc44b6a2f0e5d21bdcbe156f0cc94c324d586c74e9fb757454157ac014897fc3cbd683acb19e6397d668f67a9c68e97d683521f1ceecdbf338e24f3d497c688d

data/.github/workflows/ci.yml

@@ -148,6 +148,11 @@ jobs:
 
           bundle config set without 'tools benchmarks docs'
 
+      - name: Fix directory permissions for Bundler
+        run: |
+          chmod -R o-w /opt/hostedtoolcache/Ruby/3*/x64/lib/ruby/gems/3*/gems
+          chmod +t /opt/hostedtoolcache/Ruby/3*/x64/lib/ruby/gems/3*/gems
+
       - name: Bundle install
         run: |
           bundle config set without development

data/.ruby-version

@@ -1 +1 @@
-3.3.3
+3.3.4

data/CHANGELOG.md

@@ -1,5 +1,27 @@
 # Karafka framework changelog
 
+## 2.4.5 (2024-07-18)
+- [Change] Inject `client.id` when building subscription group and not during the initial setup.
+- [Fix] Mitigate `confluentinc/librdkafka/issues/4783` by injecting dynamic client id when using `cooperative-sticky` strategy.
+
+### Change Note
+
+`client.id` is technically a low-importance value that should not (aside from this error) impact operations. This is why it is not considered a breaking change. This change may be reverted when the original issue is fixed in librdkafka.
+
+## 2.4.4 (2024-07-04)
+- [Enhancement] Allow for offset storing from the Filtering API.
+- [Enhancement] Print more extensive error info on forceful shutdown.
+- [Enhancement] Include `original_key` in the DLQ dispatch headers.
+- [Enhancement] Support embedding mode control management from the trap context.
+- [Enhancement] Make sure, that the listener thread is stopped before restarting.
+- [Fix] Do not block on hanging listener shutdown when invoking forceful shutdown.
+- [Fix] Static membership fencing error is not propagated explicitly enough.
+- [Fix] Make sure DLQ dispatches raw headers and not deserialized headers (same as payload).
+- [Fix] Fix a typo where `ms` in logger listener would not have space before it.
+- [Maintenance] Require `karafka-core` `>=` `2.4.3`.
+- [Maintenance] Allow for usage of `karafka-rdkafka` `~` `0.16` to support librdkafka `2.4.0`.
+- [Maintenance] Lower the precision reporting to 100 microseconds in the logger listener.
+
 ## 2.4.3 (2024-06-12)
 - [Enhancement] Allow for customization of Virtual Partitions reducer for enhanced parallelization.
 - [Enhancement] Add more error codes to early report on polling issues (kidlab)

data/Gemfile.lock

@@ -1,9 +1,9 @@
 PATH
   remote: .
   specs:
-    karafka (2.4.3)
+    karafka (2.4.5)
       base64 (~> 0.2)
-      karafka-core (>= 2.4.0, < 2.5.0)
+      karafka-core (>= 2.4.3, < 2.5.0)
       waterdrop (>= 2.7.3, < 3.0.0)
       zeitwerk (~> 2.3)
 
@@ -26,12 +26,12 @@ GEM
     base64 (0.2.0)
     bigdecimal (3.1.8)
     byebug (11.1.3)
-    concurrent-ruby (1.3.1)
+    concurrent-ruby (1.3.3)
     connection_pool (2.4.1)
     diff-lcs (1.5.1)
     docile (1.4.0)
     drb (2.2.1)
-    erubi (1.12.0)
+    erubi (1.13.0)
     factory_bot (6.4.6)
       activesupport (>= 5.0.0)
     ffi (1.17.0)
@@ -39,13 +39,13 @@ GEM
       activesupport (>= 6.1)
     i18n (1.14.5)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.4.0)
-      karafka-rdkafka (>= 0.15.0, < 0.16.0)
-    karafka-rdkafka (0.15.1)
+    karafka-core (2.4.3)
+      karafka-rdkafka (>= 0.15.0, < 0.17.0)
+    karafka-rdkafka (0.16.0)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    karafka-testing (2.4.3)
+    karafka-testing (2.4.4)
       karafka (>= 2.4.0, < 2.5.0)
       waterdrop (>= 2.7.0)
     karafka-web (0.9.1)
@@ -55,12 +55,12 @@ GEM
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
     mini_portile2 (2.8.7)
-    minitest (5.23.1)
+    minitest (5.24.0)
     mutex_m (0.2.0)
     ostruct (0.6.0)
-    rack (3.0.11)
+    rack (3.1.4)
     rake (13.2.1)
-    roda (3.80.0)
+    roda (3.81.0)
       rack
     rspec (3.13.0)
       rspec-core (~> 3.13.0)
@@ -68,7 +68,7 @@ GEM
       rspec-mocks (~> 3.13.0)
     rspec-core (3.13.0)
       rspec-support (~> 3.13.0)
-    rspec-expectations (3.13.0)
+    rspec-expectations (3.13.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-mocks (3.13.1)
@@ -81,14 +81,14 @@ GEM
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
-    tilt (2.3.0)
+    tilt (2.4.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     waterdrop (2.7.3)
       karafka-core (>= 2.4.0, < 3.0.0)
       karafka-rdkafka (>= 0.15.1)
       zeitwerk (~> 2.3)
-    zeitwerk (2.6.15)
+    zeitwerk (2.6.16)
 
 PLATFORMS
   ruby
@@ -106,4 +106,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.5.11
+   2.5.14

data/docker-compose.yml

@@ -3,7 +3,7 @@ version: '2'
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.6.1
+    image: confluentinc/cp-kafka:7.6.2
 
     ports:
       - 9092:9092

data/karafka.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
   DESC
 
   spec.add_dependency 'base64', '~> 0.2'
-  spec.add_dependency 'karafka-core', '>= 2.4.0', '< 2.5.0'
+  spec.add_dependency 'karafka-core', '>= 2.4.3', '< 2.5.0'
   spec.add_dependency 'waterdrop', '>= 2.7.3', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 

data/lib/karafka/connection/client.rb

@@ -559,6 +559,9 @@ module Karafka
           early_report = true
         when :cluster_authorization_failed # 31
           early_report = true
+        # This can happen for many reasons, including issues with static membership being fenced
+        when :fatal # -150
+          early_report = true
         # @see
         # https://github.com/confluentinc/confluent-kafka-dotnet/issues/1366#issuecomment-821842990
         # This will be raised each time poll detects a non-existing topic. When auto creation is

data/lib/karafka/embedded.rb

@@ -2,11 +2,28 @@
 
 module Karafka
   # Allows to start and stop Karafka as part of a different process
+  # Following limitations and restrictions apply:
+  #
+  # - `#start` cannot be called from a trap context - non blocking
+  # - `#quiet` - can be called from a trap context - non blocking
+  # - `#stop` - can be called from a trap context - blocking
   module Embedded
     class << self
+      # Lock for ensuring we do not control embedding in parallel
+      MUTEX = Mutex.new
+
+      private_constant :MUTEX
+
       # Starts Karafka without supervision and without ownership of signals in a background thread
       # so it won't interrupt other things running
       def start
+        MUTEX.synchronize do
+          # Prevent from double-starting
+          return if @started
+
+          @started = true
+        end
+
         Thread.new do
           Thread.current.name = 'karafka.embedded'
 
@@ -19,9 +36,32 @@ module Karafka
       #
       # @note This method is blocking because we want to wait until Karafka is stopped with final
       #   process shutdown
+      #
+      # @note This method **is** safe to run from a trap context.
       def stop
-        # Stop needs to be blocking to wait for all the things to finalize
-        Karafka::Server.stop
+        # Prevent from double stopping
+        unless @stopping
+          Thread.new do
+            Thread.current.name = 'karafka.embedded.stopping'
+
+            stop = false
+
+            # We spawn a new thread because `#stop` may be called from a trap context
+            MUTEX.synchronize do
+              break if @stopping
+
+              @stopping = true
+              stop = true
+            end
+
+            next unless stop
+
+            Karafka::Server.stop
+          end
+        end
+
+        # Since we want to have this blocking, we wait for the background thread
+        sleep(0.1) until Karafka::App.terminated?
       end
 
       # Quiets Karafka upon any event
@@ -29,6 +69,8 @@ module Karafka
       # @note This method is not blocking and will not wait for Karafka to fully quiet.
       # It will trigger the quiet procedure but won't wait.
       #
+      # @note This method **can** be called from a trap context.
+      #
       # @note Please keep in mind you need to `#stop` to actually stop the server anyhow.
       def quiet
         Karafka::Server.quiet

data/lib/karafka/helpers/async.rb

@@ -22,7 +22,16 @@ module Karafka
         def included(base)
           base.extend ::Forwardable
 
-          base.def_delegators :@thread, :join, :terminate, :alive?, :name
+          base.def_delegators :@thread, :join, :terminate, :name
+        end
+      end
+
+      # @return [Boolean] true if thread is present and is running, false otherwise
+      def alive?
+        MUTEX.synchronize do
+          return false unless @thread
+
+          @thread.alive?
         end
       end
 

data/lib/karafka/instrumentation/logger_listener.rb

@@ -41,7 +41,7 @@ module Karafka
         return unless log_polling?
 
         listener = event[:caller]
-        time = event[:time]
+        time = event[:time].round(2)
         messages_count = event[:messages_buffer].size
 
         message = "[#{listener.id}] Polled #{messages_count} messages in #{time}ms"
@@ -69,14 +69,14 @@ module Karafka
       # @param event [Karafka::Core::Monitoring::Event] event details including payload
       def on_worker_processed(event)
         job = event[:job]
-        time = event[:time]
+        time = event[:time].round(2)
         job_type = job.class.to_s.split('::').last
         consumer = job.executor.topic.consumer
         topic = job.executor.topic.name
         partition = job.executor.partition
         info <<~MSG.tr("\n", ' ').strip!
           [#{job.id}] #{job_type} job for #{consumer}
-          on #{topic}/#{partition} finished in #{time}ms
+          on #{topic}/#{partition} finished in #{time} ms
         MSG
       end
 
@@ -306,7 +306,24 @@ module Karafka
           fatal "Runner crashed due to an error: #{error}"
           fatal details
         when 'app.stopping.error'
-          error 'Forceful Karafka server stop'
+          # Counts number of workers and listeners that were still active when forcing the
+          # shutdown. Please note, that unless all listeners are closed, workers will not finalize
+          # their operations as well.
+          # We need to check if listeners and workers are assigned as during super early stages of
+          # boot they are not.
+          listeners = Server.listeners ? Server.listeners.count(&:active?) : 0
+          workers = Server.workers ? Server.workers.count(&:alive?) : 0
+
+          message = <<~MSG.tr("\n", ' ').strip!
+            Forceful Karafka server stop with:
+            #{workers} active workers and
+            #{listeners} active listeners
+          MSG
+
+          error message
+        when 'app.forceful_stopping.error'
+          error "Forceful shutdown error occurred: #{error}"
+          error details
         when 'librdkafka.error'
           error "librdkafka internal error occurred: #{error}"
           error details

data/lib/karafka/pro/connection/manager.rb

@@ -216,6 +216,10 @@ module Karafka
               next unless multi_part_sgs_families.include?(sg_listener.subscription_group.name)
               # Skip already active connections
               next unless sg_listener.pending? || sg_listener.stopped?
+              # Ensure that the listener thread under which we operate is already stopped and
+              # is not dangling. While not likely to happen, this may protect against a
+              # case where a shutdown critical crash would case a restart of the same listener
+              next if sg_listener.alive?
 
               touch(sg_listener.subscription_group.id)
               sg_listener.start!

data/lib/karafka/pro/processing/coordinators/filters_applier.rb

@@ -89,6 +89,23 @@ module Karafka
             applied.map(&:cursor).compact.min_by(&:offset)
           end
 
+          # @return [Boolean] did any of the filters requested offset storage during filter
+          #   application
+          def mark_as_consumed?
+            # We can manage filtering offset only when user wanted that and there is a cursor
+            # to use
+            applied.any?(&:mark_as_consumed?) && cursor
+          end
+
+          # @return [Symbol] `:mark_as_consumed` or `:mark_as_consumed!`
+          def marking_method
+            candidates = applied.map(&:marking_method)
+
+            return :mark_as_consumed! if candidates.include?(:mark_as_consumed!)
+
+            :mark_as_consumed
+          end
+
           private
 
           # @return [Boolean] is filtering active

data/lib/karafka/pro/processing/filters/base.rb

@@ -54,6 +54,18 @@ module Karafka
           def timeout
             0
           end
+
+          # @return [Boolean] should we use the cursor value to mark as consumed. If any of the
+          #   filters returns true, we return lowers applicable cursor value (if any)
+          def mark_as_consumed?
+            false
+          end
+
+          # @return [Symbol] `:mark_as_consumed` or `:mark_as_consumed!`. Applicable only if
+          #   marking is requested
+          def marking_method
+            :mark_as_consumed
+          end
         end
       end
     end

data/lib/karafka/pro/processing/filters/delayer.rb

@@ -33,7 +33,7 @@ module Karafka
 
             # Time on message is in seconds with ms precision, so we need to convert the ttl that
             # is in ms to this format
-            border = ::Time.now.utc - @delay / 1_000.to_f
+            border = ::Time.now.utc - @delay / 1_000.0
 
             messages.delete_if do |message|
               too_young = message.timestamp > border
@@ -52,7 +52,7 @@ module Karafka
           def timeout
             return 0 unless @cursor
 
-            timeout = (@delay / 1_000.to_f) - (::Time.now.utc - @cursor.timestamp)
+            timeout = (@delay / 1_000.0) - (::Time.now.utc - @cursor.timestamp)
 
             timeout <= 0 ? 0 : timeout * 1_000
           end

data/lib/karafka/pro/processing/strategies/dlq/default.rb

@@ -154,11 +154,12 @@ module Karafka
                 topic: topic.dead_letter_queue.topic,
                 key: original_partition,
                 payload: skippable_message.raw_payload,
-                headers: skippable_message.headers.merge(
+                headers: skippable_message.raw_headers.merge(
                   'original_topic' => topic.name,
                   'original_partition' => original_partition,
                   'original_offset' => skippable_message.offset.to_s,
                   'original_consumer_group' => topic.consumer_group.id,
+                  'original_key' => skippable_message.raw_key.to_s,
                   'original_attempts' => attempt.to_s
                 )
               }

data/lib/karafka/pro/processing/strategies/ftr/default.rb

@@ -66,6 +66,19 @@ module Karafka
               # and this should not happen
               throttle_timeout = filter.timeout
 
+              # If user requested marking when applying filter, we mark. We may be in the user
+              # flow but even then this is not a problem. Older offsets will be ignored since
+              # we do not force the offset update (expected) and newer are on the user to control.
+              # This can be primarily used when filtering large quantities of data to mark on the
+              # idle runs, so lag reporting is aware that those messages were not consumed but also
+              # are no longer relevant
+              if filter.mark_as_consumed?
+                send(
+                  filter.marking_method,
+                  filter.cursor
+                )
+              end
+
               case filter.action
               when :skip
                 nil

data/lib/karafka/routing/subscription_group.rb

@@ -121,8 +121,8 @@ module Karafka
         kafka = Setup::AttributesMap.consumer(@topics.first.kafka.dup)
 
         inject_group_instance_id(kafka)
+        inject_client_id(kafka)
 
-        kafka[:'client.id'] ||= client_id
         kafka[:'group.id'] ||= @consumer_group.id
         kafka[:'auto.offset.reset'] ||= @topics.first.initial_offset
         # Karafka manages the offsets based on the processing state, thus we do not rely on the
@@ -132,6 +132,26 @@ module Karafka
         kafka
       end
 
+      # Sets (if needed) the client.id attribute
+      #
+      # @param kafka [Hash] kafka level config
+      def inject_client_id(kafka)
+        # If client id is set directly on librdkafka level, we do nothing and just go with what
+        # end user has configured
+        return if kafka.key?(:'client.id')
+
+        # This mitigates an issue for multiplexing and potentially other cases when running
+        # multiple karafka processes on one machine, where librdkafka goes into an infinite
+        # loop when using cooperative-sticky and upscaling.
+        #
+        # @see https://github.com/confluentinc/librdkafka/issues/4783
+        kafka[:'client.id'] = if kafka[:'partition.assignment.strategy'] == 'cooperative-sticky'
+                                "#{client_id}/#{Time.now.to_f}/#{SecureRandom.hex[0..9]}"
+                              else
+                                client_id
+                              end
+      end
+
       # If we use static group memberships, there can be a case, where same instance id would
       # be set on many subscription groups as the group instance id from Karafka perspective is
       # set per config. Each instance even if they are subscribed to different topics needs to

data/lib/karafka/server.rb

@@ -3,6 +3,12 @@
 module Karafka
   # Karafka consuming server class
   class Server
+    # How long should we wait on the listeners forceful shutdown when they are stuck beyond the
+    # shutdown timeout before forcing a bypass
+    FORCEFUL_SHUTDOWN_WAIT = 5
+
+    private_constant :FORCEFUL_SHUTDOWN_WAIT
+
     class << self
       # Set of consuming threads. Each consumer thread contains a single consumer
       attr_accessor :listeners
@@ -105,9 +111,23 @@ module Karafka
         # We're done waiting, lets kill them!
         workers.each(&:terminate)
         listeners.active.each(&:terminate)
+
         # We always need to shutdown clients to make sure we do not force the GC to close consumer.
         # This can cause memory leaks and crashes.
-        listeners.each(&:shutdown)
+        # We run it in a separate thread in case this would hang and we ignore it after the time
+        # we assigned to it and force shutdown as we prefer to stop the process rather than wait
+        # indefinitely even with risk of VM crash as this is a last resort.
+        Thread.new do
+          listeners.each(&:shutdown)
+        rescue StandardError => e
+          # If anything wrong happened during shutdown, we also want to record it
+          Karafka.monitor.instrument(
+            'error.occurred',
+            caller: self,
+            error: e,
+            type: 'app.forceful_stopping.error'
+          )
+        end.join(FORCEFUL_SHUTDOWN_WAIT)
 
         # We also do not forcefully terminate everything when running in the embedded mode,
         # otherwise we would overwrite the shutdown process of the process that started Karafka

data/lib/karafka/setup/attributes_map.rb

@@ -52,7 +52,9 @@ module Karafka
         fetch.wait.max.ms
         group.id
         group.instance.id
+        group.protocol
         group.protocol.type
+        group.remote.assignor
         heartbeat.interval.ms
         interceptors
         internal.termination.signal

data/lib/karafka/setup/config.rb

@@ -385,9 +385,6 @@ module Karafka
             config.kafka[key] = value
           end
 
-          # Use Karafka client_id as kafka client id if not set
-          config.kafka[:'client.id'] ||= config.client_id
-
           return if Karafka::App.env.production?
 
           KAFKA_DEV_DEFAULTS.each do |key, value|

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.4.3'
+  VERSION = '2.4.5'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.4.3
+  version: 2.4.5
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-06-12 00:00:00.000000000 Z
+date: 2024-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: base64
@@ -57,7 +57,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.0
+        version: 2.4.3
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.5.0
@@ -67,7 +67,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.0
+        version: 2.4.3
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.5.0