karafka 2.2.11 → 2.2.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '08439276b10ee121dabe7ef6496334d7aef3cf6a8a49fdc723e6333a147d57d7'
-  data.tar.gz: f1022962556a4e3397ec256bdd63e7ced077cadabddf72849473eef8ec190186
+  metadata.gz: 2ee648826503a1b1841a97e368ec2894a16eadc3509270d2a5dbbbe9ee703b3a
+  data.tar.gz: 915285e224ab6dcaa4b2f75e2b6aa52f4f6eb0f613b8b06efe78e2ef4ff3f514
 SHA512:
-  metadata.gz: 75f6a68aba0fa013bcdbcd618c9186f5b2e8870723aaef87bbfb8cb745c4a33862efac55c2a46938b7ad843f1f5e6640ebe381861c4365f459df8f115288cf2d
-  data.tar.gz: 6e495a3376f1c9650039534260c3d21ea697b77104ad2b9d7393b1ae8301cc29a116a6757efb9ead13716932ec7f6b188ab2404f3f0d86a68942f2c9972a5dc6
+  metadata.gz: 1f7f109c533a98a46306be62a2172432f0d18af7003e401d3a894aa356bc2cae2622ba4c323bfdd230a66f0ae544a7cfb61ee0168b396e2e809e408a657eecb6
+  data.tar.gz: d10de0ca361236c35bed27ca3c5db13e9e245805412f85c2d8d4e6a140fe088025403be7a65e1d97831613f02032bfe3fb2194c5ec7f6a880bc7ddc67a112813

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Karafka framework changelog
 
+## 2.2.12 (2023-11-09)
+- [Improvement] Rewrite the polling engine to update statistics and error callbacks despite longer non LRJ processing or long `max_wait_time` setups. This change provides stability to the statistics and background error emitting making them time-reliable.
+- [Improvement] Auto-update Inline Insights if new insights are present for all consumers and not only LRJ (OSS and Pro).
+- [Improvement] Alias `#insights` with `#inline_insights` and `#insights?` with `#inline_insights?`
+
 ## 2.2.11 (2023-11-03)
 - [Improvement] Allow marking as consumed in the user `#synchronize` block.
 - [Improvement] Make whole Pro VP marking as consumed concurrency safe for both async and sync scenarios.

data/Gemfile.lock

@@ -1,9 +1,9 @@
 PATH
   remote: .
   specs:
-    karafka (2.2.11)
-      karafka-core (>= 2.2.6, < 2.3.0)
-      waterdrop (>= 2.6.10, < 3.0.0)
+    karafka (2.2.12)
+      karafka-core (>= 2.2.7, < 2.3.0)
+      waterdrop (>= 2.6.11, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
@@ -39,10 +39,10 @@ GEM
       activesupport (>= 6.1)
     i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.2.6)
+    karafka-core (2.2.7)
       concurrent-ruby (>= 1.1)
-      karafka-rdkafka (>= 0.13.8, < 0.15.0)
-    karafka-rdkafka (0.13.8)
+      karafka-rdkafka (>= 0.13.9, < 0.15.0)
+    karafka-rdkafka (0.13.9)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)

data/config/locales/errors.yml

@@ -26,6 +26,7 @@ en:
       internal.active_job.consumer_class: cannot be nil
       internal.status_format: needs to be present
       internal.process_format: needs to be present
+      internal.tick_interval_format: needs to be an integer bigger or equal to 1000
       internal.routing.builder_format: needs to be present
       internal.routing.subscription_groups_builder_format: needs to be present
       internal.connection.proxy.query_watermark_offsets.timeout_format: needs to be an integer bigger than 0

data/docker-compose.yml

@@ -3,7 +3,7 @@ version: '2'
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.5.1
+    image: confluentinc/cp-kafka:7.5.2
 
     ports:
       - 9092:9092

data/karafka.gemspec

@@ -21,8 +21,8 @@ Gem::Specification.new do |spec|
     without having to focus on things that are not your business domain.
   DESC
 
-  spec.add_dependency 'karafka-core', '>= 2.2.6', '< 2.3.0'
-  spec.add_dependency 'waterdrop', '>= 2.6.10', '< 3.0.0'
+  spec.add_dependency 'karafka-core', '>= 2.2.7', '< 2.3.0'
+  spec.add_dependency 'waterdrop', '>= 2.6.11', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   if $PROGRAM_NAME.end_with?('gem')

data/lib/karafka/connection/client.rb

@@ -43,11 +43,13 @@ module Karafka
         @closed = false
         @subscription_group = subscription_group
         @buffer = RawMessagesBuffer.new
+        @tick_interval = ::Karafka::App.config.internal.tick_interval
         @rebalance_manager = RebalanceManager.new(@subscription_group.id)
         @rebalance_callback = Instrumentation::Callbacks::Rebalance.new(
           @subscription_group.id,
           @subscription_group.consumer_group.id
         )
+        @events_poller = Helpers::IntervalRunner.new { events_poll }
         @kafka = build_consumer
         # There are few operations that can happen in parallel from the listener threads as well
         # as from the workers. They are not fully thread-safe because they may be composed out of
@@ -64,6 +66,8 @@ module Karafka
 
       # Fetches messages within boundaries defined by the settings (time, size, topics, etc).
       #
+      # Also periodically runs the events polling to trigger events callbacks.
+      #
       # @return [Karafka::Connection::MessagesBuffer] messages buffer that holds messages per topic
       #   partition
       # @note This method should not be executed from many threads at the same time
@@ -73,38 +77,46 @@ module Karafka
         @buffer.clear
         @rebalance_manager.clear
 
+        events_poll
+
         loop do
           time_poll.start
 
           # Don't fetch more messages if we do not have any time left
           break if time_poll.exceeded?
-          # Don't fetch more messages if we've fetched max as we've wanted
+          # Don't fetch more messages if we've fetched max that we've wanted
           break if @buffer.size >= @subscription_group.max_messages
 
           # Fetch message within our time boundaries
-          message = poll(time_poll.remaining)
+          response = poll(time_poll.remaining)
 
           # Put a message to the buffer if there is one
-          @buffer << message if message
+          @buffer << response if response && response != :tick_time
 
           # Upon polling rebalance manager might have been updated.
           # If partition revocation happens, we need to remove messages from revoked partitions
           # as well as ensure we do not have duplicated due to the offset reset for partitions
           # that we got assigned
+          #
           # We also do early break, so the information about rebalance is used as soon as possible
           if @rebalance_manager.changed?
+            # Since rebalances do not occur often, we can run events polling as well without
+            # any throttling
+            events_poll
             remove_revoked_and_duplicated_messages
             break
           end
 
+          @events_poller.call
+
           # Track time spent on all of the processing and polling
           time_poll.checkpoint
 
           # Finally once we've (potentially) removed revoked, etc, if no messages were returned
-          # we can break.
+          # and it was not an early poll exist, we can break.
           # Worth keeping in mind, that the rebalance manager might have been updated despite no
           # messages being returned during a poll
-          break unless message
+          break unless response
         end
 
         @buffer
@@ -299,22 +311,38 @@ module Karafka
       def reset
         close
 
+        @events_poller.reset
         @closed = false
         @paused_tpls.clear
         @kafka = build_consumer
       end
 
-      # Runs a single poll ignoring all the potential errors
+      # Runs a single poll on the main queue and consumer queue ignoring all the potential errors
       # This is used as a keep-alive in the shutdown stage and any errors that happen here are
       # irrelevant from the shutdown process perspective
       #
-      # This is used only to trigger rebalance callbacks
+      # This is used only to trigger rebalance callbacks and other callbacks
       def ping
+        events_poll(100)
         poll(100)
       rescue Rdkafka::RdkafkaError
         nil
       end
 
+      # Triggers the rdkafka main queue events by consuming this queue. This is not the consumer
+      # consumption queue but the one with:
+      #   - error callbacks
+      #   - stats callbacks
+      #   - OAUTHBEARER token refresh callbacks
+      #
+      # @param timeout [Integer] number of milliseconds to wait on events or 0 not to wait.
+      #
+      # @note It is non-blocking when timeout 0 and will not wait if queue empty. It costs up to
+      #   2ms when no callbacks are triggered.
+      def events_poll(timeout = 0)
+        @kafka.events_poll(timeout)
+      end
+
       private
 
       # When we cannot store an offset, it means we no longer own the partition
@@ -464,18 +492,52 @@ module Karafka
         @kafka.position(tpl).to_h.fetch(topic).first.offset || -1
       end
 
-      # Performs a single poll operation and handles retries and error
+      # Performs a single poll operation and handles retries and errors
+      #
+      # Keep in mind, that this timeout will be limited by a tick interval value, because we cannot
+      # block on a single poll longer than that. Otherwise our events polling would not be able to
+      # run frequently enough. This means, that even if you provide big value, it will not block
+      # for that long. This is anyhow compensated by the `#batch_poll` that can run for extended
+      # period of time but will run events polling frequently while waiting for the requested total
+      # time.
       #
-      # @param timeout [Integer] timeout for a single poll
-      # @return [Rdkafka::Consumer::Message, nil] fetched message or nil if nothing polled
+      # @param timeout [Integer] timeout for a single poll.
+      # @return [Rdkafka::Consumer::Message, nil, Symbol] fetched message, nil if nothing polled
+      #   within the time we had or symbol indicating the early return reason
       def poll(timeout)
         time_poll ||= TimeTrackers::Poll.new(timeout)
 
         return nil if time_poll.exceeded?
 
         time_poll.start
+        remaining = time_poll.remaining
+
+        # We should not run a single poll longer than the tick frequency. Otherwise during a single
+        # `#batch_poll` we would not be able to run `#events_poll` often enough effectively
+        # blocking events from being handled.
+        poll_tick = timeout > @tick_interval ? @tick_interval : timeout
+
+        result = @kafka.poll(poll_tick)
+
+        # If we've got a message, we can return it
+        return result if result
+
+        time_poll.checkpoint
+
+        # We need to check if we have used all the allocated time as depending on the outcome, the
+        # batch loop behavior will differ. Using all time means, that we had nothing to do as no
+        # messages were present but if we did not exceed total time, it means we can still try
+        # polling again as we are withing user expected max wait time
+        used = remaining - time_poll.remaining
+
+        # In case we did not use enough time, it means that an internal event occured that means
+        # that something has changed without messages being published. For example a rebalance.
+        # In cases like this we finish early as well
+        return nil if used < poll_tick
 
-        @kafka.poll(timeout)
+        # If we did not exceed total time allocated, it means that we finished because of the
+        # tick interval time limitations and not because time run out without any data
+        time_poll.exceeded? ? nil : :tick_time
       rescue ::Rdkafka::RdkafkaError => e
         early_report = false
 
@@ -535,6 +597,10 @@ module Karafka
         ::Rdkafka::Config.logger = ::Karafka::App.config.logger
         config = ::Rdkafka::Config.new(@subscription_group.kafka)
         config.consumer_rebalance_listener = @rebalance_callback
+        # We want to manage the events queue independently from the messages queue. Thanks to that
+        # we can ensure, that we get statistics and errors often enough even when not polling
+        # new messages. This allows us to report statistics while data is still being processed
+        config.consumer_poll_set = false
 
         consumer = config.consumer
         @name = consumer.name

data/lib/karafka/connection/listener.rb

@@ -14,6 +14,12 @@ module Karafka
       # @return [String] id of this listener
       attr_reader :id
 
+      # How long to wait in the initial events poll. Increases chances of having the initial events
+      # immediately available
+      INITIAL_EVENTS_POLL_TIMEOUT = 100
+
+      private_constant :INITIAL_EVENTS_POLL_TIMEOUT
+
       # @param consumer_group_coordinator [Karafka::Connection::ConsumerGroupCoordinator]
       # @param subscription_group [Karafka::Routing::SubscriptionGroup]
       # @param jobs_queue [Karafka::Processing::JobsQueue] queue where we should push work
@@ -32,6 +38,7 @@ module Karafka
         @partitioner = proc_config.partitioner_class.new(subscription_group)
         # We reference scheduler here as it is much faster than fetching this each time
         @scheduler = proc_config.scheduler
+        @events_poller = Helpers::IntervalRunner.new { @client.events_poll }
         # We keep one buffer for messages to preserve memory and not allocate extra objects
         # We can do this that way because we always first schedule jobs using messages before we
         # fetch another batch.
@@ -84,6 +91,15 @@ module Karafka
       #   Kafka connections / Internet connection issues / Etc. Business logic problems should not
       #   propagate this far.
       def fetch_loop
+        # Run the initial events fetch to improve chances of having metrics and initial callbacks
+        # triggers on start.
+        #
+        # In theory this may slow down the initial boot but we limit it up to 100ms, so it should
+        # not have a big initial impact. It may not be enough but Karafka does not give the boot
+        # warranties of statistics or other callbacks being immediately available, hence this is
+        # a fair trade-off
+        @client.events_poll(INITIAL_EVENTS_POLL_TIMEOUT)
+
         # Run the main loop as long as we are not stopping or moving into quiet mode
         until Karafka::App.done?
           Karafka.monitor.instrument(
@@ -287,7 +303,7 @@ module Karafka
 
       # Waits for all the jobs from a given subscription group to finish before moving forward
       def wait
-        @jobs_queue.wait(@subscription_group.id)
+        @jobs_queue.wait(@subscription_group.id) { @events_poller.call }
       end
 
       # Waits without blocking the polling
@@ -318,6 +334,7 @@ module Karafka
         # resetting.
         @jobs_queue.wait(@subscription_group.id)
         @jobs_queue.clear(@subscription_group.id)
+        @events_poller.reset
         @client.reset
         @coordinators.reset
         @executors = Processing::ExecutorsBuffer.new(@client, @subscription_group)

data/lib/karafka/contracts/config.rb

@@ -46,6 +46,9 @@ module Karafka
       nested(:internal) do
         required(:status) { |val| !val.nil? }
         required(:process) { |val| !val.nil? }
+        # In theory this could be less than a second, however this would impact the maximum time
+        # of a single consumer queue poll, hence we prevent it
+        required(:tick_interval) { |val| val.is_a?(Integer) && val >= 1_000 }
 
         nested(:connection) do
           nested(:proxy) do

data/lib/karafka/helpers/interval_runner.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Helpers
+    # Object responsible for running given code with a given interval. It won't run given code
+    # more often than with a given interval.
+    #
+    # This allows us to execute certain code only once in a while.
+    #
+    # This can be used when we have code that could be invoked often due to it being in loops
+    # or other places but would only slow things down if would run with each tick.
+    class IntervalRunner
+      include Karafka::Core::Helpers::Time
+
+      # @param interval [Integer] interval in ms for running the provided code. Defaults to the
+      #   `internal.tick_interval` value
+      # @param block [Proc] block of code we want to run once in a while
+      def initialize(interval: ::Karafka::App.config.internal.tick_interval, &block)
+        @block = block
+        @interval = interval
+        @last_called_at = monotonic_now - @interval
+      end
+
+      # Runs the requested code if it was not executed previously recently
+      def call
+        return if monotonic_now - @last_called_at < @interval
+
+        @last_called_at = monotonic_now
+
+        @block.call
+      end
+
+      # Resets the runner, so next `#call` will run the underlying code
+      def reset
+        @last_called_at = monotonic_now - @interval
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb

@@ -137,15 +137,7 @@ module Karafka
           def push_tags
             return unless Karafka.logger.respond_to?(:push_tags)
 
-            # Older versions of ddtrace do not have the `#log_correlation` method, so we fallback
-            # to the older method for tags
-            tags = if client.respond_to?(:log_correlation)
-                     client.log_correlation
-                   else
-                     client.active_correlation.to_s
-                   end
-
-            Karafka.logger.push_tags(tags)
+            Karafka.logger.push_tags(client.log_correlation)
           end
 
           # Pops datadog's tags from the logger

data/lib/karafka/processing/inline_insights/consumer.rb

@@ -35,6 +35,8 @@ module Karafka
 
         alias statistics insights
         alias statistics? insights?
+        alias inline_insights insights
+        alias inline_insights? insights?
       end
     end
   end

data/lib/karafka/processing/jobs_queue.rb

@@ -21,9 +21,12 @@ module Karafka
         # We cannot use a single semaphore as it could potentially block in listeners that should
         # process with their data and also could unlock when a given group needs to remain locked
         @semaphores = Concurrent::Map.new do |h, k|
-          h.compute_if_absent(k) { Queue.new }
+          # Ruby prior to 3.2 did not have queue with a timeout on `#pop`, that is why for those
+          # versions we use our custom queue wrapper
+          h.compute_if_absent(k) { RUBY_VERSION < '3.2' ? TimedQueue.new : Queue.new }
         end
 
+        @tick_interval = ::Karafka::App.config.internal.tick_interval
         @in_processing = Hash.new { |h, k| h[k] = [] }
 
         @mutex = Mutex.new
@@ -118,11 +121,19 @@ module Karafka
       #   jobs from a given group are completed
       #
       # @param group_id [String] id of the group in which jobs we're interested.
+      # @yieldparam [Block] block we want to run before each pop (in case of Ruby pre 3.2) or
+      #   before each pop and on every tick interval.
+      #   This allows us to run extra code that needs to be executed even when we are waiting on
+      #   the work to be finished.
       # @note This method is blocking.
       def wait(group_id)
         # Go doing other things while we cannot process and wait for anyone to finish their work
         # and re-check the wait status
-        @semaphores[group_id].pop while wait?(group_id)
+        while wait?(group_id)
+          yield if block_given?
+
+          @semaphores[group_id].pop(timeout: @tick_interval / 1_000.0)
+        end
       end
 
       # - `busy` - number of jobs that are currently being processed (active work)

data/lib/karafka/processing/timed_queue.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Minimal queue with timeout for Ruby 3.1 and lower.
+    #
+    # It is needed because only since 3.2, Ruby has a timeout on `#pop`
+    class TimedQueue
+      include Karafka::Core::Helpers::Time
+
+      def initialize
+        @queue = Queue.new
+        @mutex = Thread::Mutex.new
+        @resource = Thread::ConditionVariable.new
+      end
+
+      # Adds element to the queue
+      #
+      # @param obj [Object] pushes an element onto the queue
+      def push(obj)
+        @mutex.synchronize do
+          @queue << obj
+          @resource.broadcast
+        end
+      end
+
+      alias << push
+
+      # No timeout means waiting up to 31 years
+      #
+      # @param timeout [Integer] max number of seconds to wait on the pop
+      # @return [Object] element inserted on the array or `nil` on timeout
+      #
+      # @note We use timeout in seconds because this is how Ruby 3.2+ works and we want to have
+      #   the same API for newer and older Ruby versions
+      def pop(timeout: 10_000_000_000)
+        deadline = monotonic_now + timeout * 1000
+
+        @mutex.synchronize do
+          loop do
+            return @queue.pop unless @queue.empty?
+            return @queue.pop if @queue.closed?
+
+            to_wait = (deadline - monotonic_now) / 1_000.0
+
+            return nil if to_wait <= 0
+
+            @resource.wait(@mutex, to_wait)
+          end
+        end
+      end
+
+      # Closes the internal queue and releases the lock
+      def close
+        @mutex.synchronize do
+          @queue.close
+          @resource.broadcast
+        end
+      end
+    end
+  end
+end

data/lib/karafka/setup/config.rb

@@ -152,6 +152,17 @@ module Karafka
         #   instances
         setting :process, default: Process.new
 
+        # Interval of "ticking". This is used to define the maximum time between consecutive
+        # polling of the main rdkafka queue. It should match also the `statistics.interval.ms`
+        # smallest value defined in any of the per-kafka settings, so metrics are published with
+        # the desired frequency. It is set to 5 seconds because `statistics.interval.ms` is also
+        # set to five seconds.
+        #
+        # It is NOT allowed to set it to a value less than 1 seconds because it could cause polling
+        # not to have enough time to run. This (not directly) defines also a single poll
+        # max timeout as to allow for frequent enough events polling
+        setting :tick_interval, default: 5_000
+
         # Namespace for CLI related settings
         setting :cli do
           # option contract [Object] cli setup validation contract (in the context of options and

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.2.11'
+  VERSION = '2.2.12'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.2.11
+  version: 2.2.12
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-11-03 00:00:00.000000000 Z
+date: 2023-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.6
+        version: 2.2.7
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.3.0
@@ -53,7 +53,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.6
+        version: 2.2.7
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.3.0
@@ -63,7 +63,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.6.10
+        version: 2.6.11
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -73,7 +73,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.6.10
+        version: 2.6.11
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -181,6 +181,7 @@ files:
 - lib/karafka/errors.rb
 - lib/karafka/helpers/async.rb
 - lib/karafka/helpers/colorize.rb
+- lib/karafka/helpers/interval_runner.rb
 - lib/karafka/helpers/multi_delegator.rb
 - lib/karafka/instrumentation/callbacks/error.rb
 - lib/karafka/instrumentation/callbacks/rebalance.rb
@@ -374,6 +375,7 @@ files:
 - lib/karafka/processing/strategies/dlq_mom.rb
 - lib/karafka/processing/strategies/mom.rb
 - lib/karafka/processing/strategy_selector.rb
+- lib/karafka/processing/timed_queue.rb
 - lib/karafka/processing/worker.rb
 - lib/karafka/processing/workers_batch.rb
 - lib/karafka/railtie.rb