karafka 2.4.3 → 2.4.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c37b855609a9d20ca0b5cf13875d30ccb71b4072b3f07886f10445ce761c6e1e
-  data.tar.gz: 852fa3340f0dd1cbc2286823a2451c635568440c3e753e7235a72dc85289fcf5
+  metadata.gz: 1b7333a7812643bc128c721e411449dfd477adfa52968c7c291900e622cdcc04
+  data.tar.gz: 7062798152f0d4e097e5c4fcd228798103a680441579bbc1918c8bf5a65ce98b
 SHA512:
-  metadata.gz: 373b460007cefdb039b5f2fd00abdc722f42c5bd8f324dad121c7e293264ca4a63cb63cb4fa5057e0f2ff856fe6c28073c04d2ffb21250dd534408ee511807c9
-  data.tar.gz: 595e408578801d949bb2b1e8d3337e8535473b80c7a1db34e56822214de9fd2083a594433e264ed8675c2292cd3aa73bacaef44938ad7a4cd743292097fd4d8f
+  metadata.gz: bd24bd450ac1cda02d18d7d487317139988b943ecc92c606cb713f13c819db044d6feae978873d26445e1ebf8d8823bedfd7efae58bc53c8213be9ef56a84c1d
+  data.tar.gz: 0f2f4b50ea90cd76f6275e0cdc9eeada5e5cd30bf6c18a219209dfe4b717d90ef88d7c6b68797e9eb0615c83ba03ee738d41164014ce74aaf27970ef07ba5974

data/.github/workflows/ci.yml

@@ -148,6 +148,11 @@ jobs:
 
           bundle config set without 'tools benchmarks docs'
 
+      - name: Fix directory permissions for Bundler
+        run: |
+          chmod -R o-w /opt/hostedtoolcache/Ruby/3*/x64/lib/ruby/gems/3*/gems
+          chmod +t /opt/hostedtoolcache/Ruby/3*/x64/lib/ruby/gems/3*/gems
+
       - name: Bundle install
         run: |
           bundle config set without development

data/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Karafka framework changelog
 
+## 2.4.4 (2024-07-04)
+- [Enhancement] Allow for offset storing from the Filtering API.
+- [Enhancement] Print more extensive error info on forceful shutdown.
+- [Enhancement] Include `original_key` in the DLQ dispatch headers.
+- [Enhancement] Support embedding mode control management from the trap context.
+- [Enhancement] Make sure, that the listener thread is stopped before restarting.
+- [Fix] Do not block on hanging listener shutdown when invoking forceful shutdown.
+- [Fix] Static membership fencing error is not propagated explicitly enough.
+- [Fix] Make sure DLQ dispatches raw headers and not deserialized headers (same as payload).
+- [Fix] Fix a typo where `ms` in logger listener would not have space before it.
+- [Maintenance] Require `karafka-core` `>=` `2.4.3`.
+- [Maintenance] Allow for usage of `karafka-rdkafka` `~` `0.16` to support librdkafka `2.4.0`.
+- [Maintenance] Lower the precision reporting to 100 microseconds in the logger listener.
+
 ## 2.4.3 (2024-06-12)
 - [Enhancement] Allow for customization of Virtual Partitions reducer for enhanced parallelization.
 - [Enhancement] Add more error codes to early report on polling issues (kidlab)

data/Gemfile.lock

@@ -1,9 +1,9 @@
 PATH
   remote: .
   specs:
-    karafka (2.4.3)
+    karafka (2.4.4)
       base64 (~> 0.2)
-      karafka-core (>= 2.4.0, < 2.5.0)
+      karafka-core (>= 2.4.3, < 2.5.0)
       waterdrop (>= 2.7.3, < 3.0.0)
       zeitwerk (~> 2.3)
 
@@ -26,12 +26,12 @@ GEM
     base64 (0.2.0)
     bigdecimal (3.1.8)
     byebug (11.1.3)
-    concurrent-ruby (1.3.1)
+    concurrent-ruby (1.3.3)
     connection_pool (2.4.1)
     diff-lcs (1.5.1)
     docile (1.4.0)
     drb (2.2.1)
-    erubi (1.12.0)
+    erubi (1.13.0)
     factory_bot (6.4.6)
       activesupport (>= 5.0.0)
     ffi (1.17.0)
@@ -39,13 +39,13 @@ GEM
       activesupport (>= 6.1)
     i18n (1.14.5)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.4.0)
-      karafka-rdkafka (>= 0.15.0, < 0.16.0)
-    karafka-rdkafka (0.15.1)
+    karafka-core (2.4.3)
+      karafka-rdkafka (>= 0.15.0, < 0.17.0)
+    karafka-rdkafka (0.16.0)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    karafka-testing (2.4.3)
+    karafka-testing (2.4.4)
       karafka (>= 2.4.0, < 2.5.0)
       waterdrop (>= 2.7.0)
     karafka-web (0.9.1)
@@ -55,12 +55,12 @@ GEM
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
     mini_portile2 (2.8.7)
-    minitest (5.23.1)
+    minitest (5.24.0)
     mutex_m (0.2.0)
     ostruct (0.6.0)
-    rack (3.0.11)
+    rack (3.1.4)
     rake (13.2.1)
-    roda (3.80.0)
+    roda (3.81.0)
       rack
     rspec (3.13.0)
       rspec-core (~> 3.13.0)
@@ -68,7 +68,7 @@ GEM
       rspec-mocks (~> 3.13.0)
     rspec-core (3.13.0)
       rspec-support (~> 3.13.0)
-    rspec-expectations (3.13.0)
+    rspec-expectations (3.13.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-mocks (3.13.1)
@@ -81,14 +81,14 @@ GEM
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
-    tilt (2.3.0)
+    tilt (2.4.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     waterdrop (2.7.3)
       karafka-core (>= 2.4.0, < 3.0.0)
       karafka-rdkafka (>= 0.15.1)
       zeitwerk (~> 2.3)
-    zeitwerk (2.6.15)
+    zeitwerk (2.6.16)
 
 PLATFORMS
   ruby
@@ -106,4 +106,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.5.11
+   2.5.14

data/karafka.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
   DESC
 
   spec.add_dependency 'base64', '~> 0.2'
-  spec.add_dependency 'karafka-core', '>= 2.4.0', '< 2.5.0'
+  spec.add_dependency 'karafka-core', '>= 2.4.3', '< 2.5.0'
   spec.add_dependency 'waterdrop', '>= 2.7.3', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 

data/lib/karafka/connection/client.rb

@@ -559,6 +559,9 @@ module Karafka
           early_report = true
         when :cluster_authorization_failed # 31
           early_report = true
+        # This can happen for many reasons, including issues with static membership being fenced
+        when :fatal # -150
+          early_report = true
         # @see
         # https://github.com/confluentinc/confluent-kafka-dotnet/issues/1366#issuecomment-821842990
         # This will be raised each time poll detects a non-existing topic. When auto creation is

data/lib/karafka/embedded.rb

@@ -2,11 +2,28 @@
 
 module Karafka
   # Allows to start and stop Karafka as part of a different process
+  # Following limitations and restrictions apply:
+  #
+  # - `#start` cannot be called from a trap context - non blocking
+  # - `#quiet` - can be called from a trap context - non blocking
+  # - `#stop` - can be called from a trap context - blocking
   module Embedded
     class << self
+      # Lock for ensuring we do not control embedding in parallel
+      MUTEX = Mutex.new
+
+      private_constant :MUTEX
+
       # Starts Karafka without supervision and without ownership of signals in a background thread
       # so it won't interrupt other things running
       def start
+        MUTEX.synchronize do
+          # Prevent from double-starting
+          return if @started
+
+          @started = true
+        end
+
         Thread.new do
           Thread.current.name = 'karafka.embedded'
 
@@ -19,9 +36,32 @@ module Karafka
       #
       # @note This method is blocking because we want to wait until Karafka is stopped with final
       #   process shutdown
+      #
+      # @note This method **is** safe to run from a trap context.
       def stop
-        # Stop needs to be blocking to wait for all the things to finalize
-        Karafka::Server.stop
+        # Prevent from double stopping
+        unless @stopping
+          Thread.new do
+            Thread.current.name = 'karafka.embedded.stopping'
+
+            stop = false
+
+            # We spawn a new thread because `#stop` may be called from a trap context
+            MUTEX.synchronize do
+              break if @stopping
+
+              @stopping = true
+              stop = true
+            end
+
+            next unless stop
+
+            Karafka::Server.stop
+          end
+        end
+
+        # Since we want to have this blocking, we wait for the background thread
+        sleep(0.1) until Karafka::App.terminated?
       end
 
       # Quiets Karafka upon any event
@@ -29,6 +69,8 @@ module Karafka
       # @note This method is not blocking and will not wait for Karafka to fully quiet.
       # It will trigger the quiet procedure but won't wait.
       #
+      # @note This method **can** be called from a trap context.
+      #
       # @note Please keep in mind you need to `#stop` to actually stop the server anyhow.
       def quiet
         Karafka::Server.quiet

data/lib/karafka/helpers/async.rb

@@ -22,7 +22,16 @@ module Karafka
         def included(base)
           base.extend ::Forwardable
 
-          base.def_delegators :@thread, :join, :terminate, :alive?, :name
+          base.def_delegators :@thread, :join, :terminate, :name
+        end
+      end
+
+      # @return [Boolean] true if thread is present and is running, false otherwise
+      def alive?
+        MUTEX.synchronize do
+          return false unless @thread
+
+          @thread.alive?
         end
       end
 

data/lib/karafka/instrumentation/logger_listener.rb

@@ -41,7 +41,7 @@ module Karafka
         return unless log_polling?
 
         listener = event[:caller]
-        time = event[:time]
+        time = event[:time].round(2)
         messages_count = event[:messages_buffer].size
 
         message = "[#{listener.id}] Polled #{messages_count} messages in #{time}ms"
@@ -69,14 +69,14 @@ module Karafka
       # @param event [Karafka::Core::Monitoring::Event] event details including payload
       def on_worker_processed(event)
         job = event[:job]
-        time = event[:time]
+        time = event[:time].round(2)
         job_type = job.class.to_s.split('::').last
         consumer = job.executor.topic.consumer
         topic = job.executor.topic.name
         partition = job.executor.partition
         info <<~MSG.tr("\n", ' ').strip!
           [#{job.id}] #{job_type} job for #{consumer}
-          on #{topic}/#{partition} finished in #{time}ms
+          on #{topic}/#{partition} finished in #{time} ms
         MSG
       end
 
@@ -306,7 +306,24 @@ module Karafka
           fatal "Runner crashed due to an error: #{error}"
           fatal details
         when 'app.stopping.error'
-          error 'Forceful Karafka server stop'
+          # Counts number of workers and listeners that were still active when forcing the
+          # shutdown. Please note, that unless all listeners are closed, workers will not finalize
+          # their operations as well.
+          # We need to check if listeners and workers are assigned as during super early stages of
+          # boot they are not.
+          listeners = Server.listeners ? Server.listeners.count(&:active?) : 0
+          workers = Server.workers ? Server.workers.count(&:alive?) : 0
+
+          message = <<~MSG.tr("\n", ' ').strip!
+            Forceful Karafka server stop with:
+            #{workers} active workers and
+            #{listeners} active listeners
+          MSG
+
+          error message
+        when 'app.forceful_stopping.error'
+          error "Forceful shutdown error occurred: #{error}"
+          error details
         when 'librdkafka.error'
           error "librdkafka internal error occurred: #{error}"
           error details

data/lib/karafka/pro/connection/manager.rb

@@ -216,6 +216,10 @@ module Karafka
               next unless multi_part_sgs_families.include?(sg_listener.subscription_group.name)
               # Skip already active connections
               next unless sg_listener.pending? || sg_listener.stopped?
+              # Ensure that the listener thread under which we operate is already stopped and
+              # is not dangling. While not likely to happen, this may protect against a
+              # case where a shutdown critical crash would case a restart of the same listener
+              next if sg_listener.alive?
 
               touch(sg_listener.subscription_group.id)
               sg_listener.start!

data/lib/karafka/pro/processing/coordinators/filters_applier.rb

@@ -89,6 +89,23 @@ module Karafka
             applied.map(&:cursor).compact.min_by(&:offset)
           end
 
+          # @return [Boolean] did any of the filters requested offset storage during filter
+          #   application
+          def mark_as_consumed?
+            # We can manage filtering offset only when user wanted that and there is a cursor
+            # to use
+            applied.any?(&:mark_as_consumed?) && cursor
+          end
+
+          # @return [Symbol] `:mark_as_consumed` or `:mark_as_consumed!`
+          def marking_method
+            candidates = applied.map(&:marking_method)
+
+            return :mark_as_consumed! if candidates.include?(:mark_as_consumed!)
+
+            :mark_as_consumed
+          end
+
           private
 
           # @return [Boolean] is filtering active

data/lib/karafka/pro/processing/filters/base.rb

@@ -54,6 +54,18 @@ module Karafka
           def timeout
             0
           end
+
+          # @return [Boolean] should we use the cursor value to mark as consumed. If any of the
+          #   filters returns true, we return lowers applicable cursor value (if any)
+          def mark_as_consumed?
+            false
+          end
+
+          # @return [Symbol] `:mark_as_consumed` or `:mark_as_consumed!`. Applicable only if
+          #   marking is requested
+          def marking_method
+            :mark_as_consumed
+          end
         end
       end
     end

data/lib/karafka/pro/processing/filters/delayer.rb

@@ -33,7 +33,7 @@ module Karafka
 
             # Time on message is in seconds with ms precision, so we need to convert the ttl that
             # is in ms to this format
-            border = ::Time.now.utc - @delay / 1_000.to_f
+            border = ::Time.now.utc - @delay / 1_000.0
 
             messages.delete_if do |message|
               too_young = message.timestamp > border
@@ -52,7 +52,7 @@ module Karafka
           def timeout
             return 0 unless @cursor
 
-            timeout = (@delay / 1_000.to_f) - (::Time.now.utc - @cursor.timestamp)
+            timeout = (@delay / 1_000.0) - (::Time.now.utc - @cursor.timestamp)
 
             timeout <= 0 ? 0 : timeout * 1_000
           end

data/lib/karafka/pro/processing/strategies/dlq/default.rb

@@ -154,11 +154,12 @@ module Karafka
                 topic: topic.dead_letter_queue.topic,
                 key: original_partition,
                 payload: skippable_message.raw_payload,
-                headers: skippable_message.headers.merge(
+                headers: skippable_message.raw_headers.merge(
                   'original_topic' => topic.name,
                   'original_partition' => original_partition,
                   'original_offset' => skippable_message.offset.to_s,
                   'original_consumer_group' => topic.consumer_group.id,
+                  'original_key' => skippable_message.raw_key.to_s,
                   'original_attempts' => attempt.to_s
                 )
               }

data/lib/karafka/pro/processing/strategies/ftr/default.rb

@@ -66,6 +66,19 @@ module Karafka
               # and this should not happen
               throttle_timeout = filter.timeout
 
+              # If user requested marking when applying filter, we mark. We may be in the user
+              # flow but even then this is not a problem. Older offsets will be ignored since
+              # we do not force the offset update (expected) and newer are on the user to control.
+              # This can be primarily used when filtering large quantities of data to mark on the
+              # idle runs, so lag reporting is aware that those messages were not consumed but also
+              # are no longer relevant
+              if filter.mark_as_consumed?
+                send(
+                  filter.marking_method,
+                  filter.cursor
+                )
+              end
+
               case filter.action
               when :skip
                 nil

data/lib/karafka/server.rb

@@ -3,6 +3,12 @@
 module Karafka
   # Karafka consuming server class
   class Server
+    # How long should we wait on the listeners forceful shutdown when they are stuck beyond the
+    # shutdown timeout before forcing a bypass
+    FORCEFUL_SHUTDOWN_WAIT = 5
+
+    private_constant :FORCEFUL_SHUTDOWN_WAIT
+
     class << self
       # Set of consuming threads. Each consumer thread contains a single consumer
       attr_accessor :listeners
@@ -105,9 +111,23 @@ module Karafka
         # We're done waiting, lets kill them!
         workers.each(&:terminate)
         listeners.active.each(&:terminate)
+
         # We always need to shutdown clients to make sure we do not force the GC to close consumer.
         # This can cause memory leaks and crashes.
-        listeners.each(&:shutdown)
+        # We run it in a separate thread in case this would hang and we ignore it after the time
+        # we assigned to it and force shutdown as we prefer to stop the process rather than wait
+        # indefinitely even with risk of VM crash as this is a last resort.
+        Thread.new do
+          listeners.each(&:shutdown)
+        rescue StandardError => e
+          # If anything wrong happened during shutdown, we also want to record it
+          Karafka.monitor.instrument(
+            'error.occurred',
+            caller: self,
+            error: e,
+            type: 'app.forceful_stopping.error'
+          )
+        end.join(FORCEFUL_SHUTDOWN_WAIT)
 
         # We also do not forcefully terminate everything when running in the embedded mode,
         # otherwise we would overwrite the shutdown process of the process that started Karafka

data/lib/karafka/setup/attributes_map.rb

@@ -52,7 +52,9 @@ module Karafka
         fetch.wait.max.ms
         group.id
         group.instance.id
+        group.protocol
         group.protocol.type
+        group.remote.assignor
         heartbeat.interval.ms
         interceptors
         internal.termination.signal

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.4.3'
+  VERSION = '2.4.4'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.4.3
+  version: 2.4.4
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-06-12 00:00:00.000000000 Z
+date: 2024-07-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: base64
@@ -57,7 +57,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.0
+        version: 2.4.3
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.5.0
@@ -67,7 +67,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.0
+        version: 2.4.3
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.5.0