karafka 2.0.38 → 2.0.39

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1b9653385cf5a3b1e27eae06d53b9761c9a1f265252f721773258459eb3df1e7
-  data.tar.gz: c0af983ab0539e8463bf2612068a6b261de1325078c3e8600b0d6df0f596d100
+  metadata.gz: 12fe8a47dc0ab16b0f7783424cd1aa043c2d2b228b4f4164f1cecefe604269d9
+  data.tar.gz: 9fa3bae282770dd67503c41ef4b73a27a38bfcff3bf472ddd63753d14d03614f
 SHA512:
-  metadata.gz: d9000a8f71d7fff762db5f567956f6ea68e436b428014c509ae233730c9f75fd6ac311e51b0022999dfdce64362c86dab6912ce549378d9def231e5749961140
-  data.tar.gz: f980261b5ada2f46efbf919aac86ab63da5bccce26639b9e7d98c07c6012cc3c727189a548627687092ee2802aca8df3d5459bcdcc8d9d29b35f2d6da92a64fc
+  metadata.gz: 9e6536c90a411a0b42337f73c00d9f454028366f42eabb1b7f40902181bcbcfd43258741d6fc51c6e29046b9ee1f8598755440d28a00ca96104a61a8095c20c2
+  data.tar.gz: be75dd1bfa744187f770f2e1f0deeedfba4f3fb1b824d6bab91f056f96e60a33498429e35ea22841404def0935e584db0df4289d2818631ace2e597d28785960

data/.github/workflows/ci.yml

@@ -62,7 +62,7 @@ jobs:
         run: \curl -sSL https://api.coditsu.io/run/ci | bash
 
   specs:
-    timeout-minutes: 30
+    timeout-minutes: 45
     runs-on: ubuntu-latest
     needs: diffend
     strategy:

data/.ruby-version

@@ -1 +1 @@
-3.2.1
+3.2.2

data/CHANGELOG.md

@@ -1,5 +1,33 @@
 # Karafka framework changelog
 
+## 2.0.39 (2023-04-11)
+- **[Feature]** Provide ability to throttle/limit number of messages processed in a time unit (#1203)
+- **[Feature]** Provide Delayed Topics (#1000)
+- **[Feature]** Provide ability to expire messages (expiring topics)
+- **[Feature]** Provide ability to apply filters after messages are polled and before enqueued. This is a generic filter API for any usage.
+- [Improvement] When using ActiveJob with Virtual Partitions, Karafka will stop if collectively VPs are failing. This minimizes number of jobs that will be collectively re-processed.
+- [Improvement] `#retrying?` method has been added to consumers to provide ability to check, that we're reprocessing data after a failure. This is useful for branching out processing based on errors.
+- [Improvement] Track active_job_id in instrumentation (#1372)
+- [Improvement] Introduce new housekeeping job type called `Idle` for non-consumption execution flows.
+- [Improvement] Change how a manual offset management works with Long-Running Jobs. Use the last message offset to move forward instead of relying on the last message marked as consumed for a scenario where no message is marked.
+- [Improvement] Prioritize in Pro non-consumption jobs execution over consumption despite LJF. This will ensure, that housekeeping as well as other non-consumption events are not saturated when running a lot of work.
+- [Improvement] Normalize the DLQ behaviour with MoM. Always pause on dispatch for all the strategies.
+- [Improvement] Improve the manual offset management and DLQ behaviour when no markings occur for OSS.
+- [Improvement] Do not early stop ActiveJob work running under virtual partitions to prevent extensive reprocessing.
+- [Improvement] Drastically increase number of scenarios covered by integration specs (OSS and Pro).
+- [Improvement] Introduce a `Coordinator#synchronize` lock for cross virtual partitions operations.
+- [Fix] Do not resume partition that is not paused.
+- [Fix] Fix `LoggerListener` cases where logs would not include caller id (when available)
+- [Fix] Fix not working benchmark tests.
+- [Fix] Fix a case where when using manual offset management with a user pause would ignore the pause and seek to the next message.
+- [Fix] Fix a case where dead letter queue would go into an infinite loop on message with first ever offset if the first ever offset would not recover.
+- [Fix] Make sure to resume always for all LRJ strategies on revocation.
+- [Refactor] Make sure that coordinator is topic aware. Needed for throttling, delayed processing and expired jobs.
+- [Refactor] Put Pro strategies into namespaces to better organize multiple combinations.
+- [Refactor] Do not rely on messages metadata for internal topic and partition operations like `#seek` so they can run independently from the consumption flow.
+- [Refactor] Hold a single topic/partition reference on a coordinator instead of in executor, coordinator and consumer.
+- [Refactor] Move `#mark_as_consumed` and `#mark_as_consumed!`into `Strategies::Default` to be able to introduce marking for virtual partitions.
+
 ## 2.0.38 (2023-03-27)
 - [Improvement] Introduce `Karafka::Admin#read_watermark_offsets` to get low and high watermark offsets values.
 - [Improvement] Track active_job_id in instrumentation (#1372)

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.38)
+    karafka (2.0.39)
       karafka-core (>= 2.0.12, < 3.0.0)
       thor (>= 0.20)
       waterdrop (>= 2.4.10, < 3.0.0)
@@ -48,7 +48,7 @@ GEM
     rspec-expectations (3.12.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
-    rspec-mocks (3.12.3)
+    rspec-mocks (3.12.5)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
     rspec-support (3.12.0)
@@ -61,7 +61,7 @@ GEM
     thor (1.2.1)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.5.0)
+    waterdrop (2.5.1)
       karafka-core (>= 2.0.12, < 3.0.0)
       zeitwerk (~> 2.3)
     zeitwerk (2.6.7)
@@ -79,4 +79,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.4.7
+   2.4.10

data/bin/integrations

@@ -25,7 +25,7 @@ ROOT_PATH = Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../
 # we limit it. Locally we can run a lot of those, as many of them have sleeps and do not use a lot
 # of CPU. Locally we also cannot go beyond certain limit due to how often and how many topics we
 # create in Kafka. With an overloaded system, we start getting timeouts.
-CONCURRENCY = ENV.key?('CI') ? 4 : Etc.nprocessors * 2
+CONCURRENCY = ENV.key?('CI') ? 5 : Etc.nprocessors * 3
 
 # How may bytes do we want to keep from the stdout in the buffer for when we need to print it
 MAX_BUFFER_OUTPUT = 51_200

data/config/locales/errors.yml

@@ -72,10 +72,3 @@ en:
     test:
       missing: needs to be present
       id_format: needs to be a String
-
-    pro_topic:
-      virtual_partitions.partitioner_respond_to_call: needs to be defined and needs to respond to `#call`
-      virtual_partitions.max_partitions_format: needs to be equal or more than 1
-      manual_offset_management_not_with_virtual_partitions: cannot be used together with Virtual Partitions
-      long_running_job.active_format: needs to be either true or false
-      dead_letter_queue_not_with_virtual_partitions: cannot be used together with Virtual Partitions

data/config/locales/pro_errors.yml

@@ -3,10 +3,28 @@ en:
     topic:
       virtual_partitions.partitioner_respond_to_call: needs to be defined and needs to respond to `#call`
       virtual_partitions.max_partitions_format: needs to be equal or more than 1
+
       manual_offset_management_not_with_virtual_partitions: cannot be used together with Virtual Partitions
+
       long_running_job.active_format: needs to be either true or false
+
       dead_letter_queue_with_virtual_partitions: when using Dead Letter Queue with Virtual Partitions, at least one retry is required.
 
+      throttling.active_format: needs to be either true or false
+      throttling.limit_format: needs to be equal or more than 1
+      throttling.interval_format: needs to be equal or more than 1
+
+      filtering.active_missing: needs to be present
+      filtering.factory_format: 'needs to respond to #call'
+      filtering.factories_format: 'needs to contain only factories responding to #call'
+      filtering.active_format: 'needs to be boolean'
+
+      expiring.ttl_format: 'needs to be equal or more than 0 and an integer'
+      expiring.active_format: 'needs to be boolean'
+
+      delaying.delay_format: 'needs to be equal or more than 0 and an integer'
+      delaying.active_format: 'needs to be boolean'
+
     config:
       encryption.active_format: 'needs to be either true or false'
       encryption.public_key_invalid: 'is not a valid public RSA key'

data/lib/karafka/base_consumer.rb

@@ -7,11 +7,13 @@ module Karafka
     # Allow for consumer instance tagging for instrumentation
     include ::Karafka::Core::Taggable
 
+    extend Forwardable
+
+    def_delegators :@coordinator, :topic, :partition
+
     # @return [String] id of the current consumer
     attr_reader :id
     # @return [Karafka::Routing::Topic] topic to which a given consumer is subscribed
-    attr_accessor :topic
-    # @return [Karafka::Messages::Messages] current messages batch
     attr_accessor :messages
     # @return [Karafka::Connection::Client] kafka connection client
     attr_accessor :client
@@ -97,6 +99,20 @@ module Karafka
       )
     end
 
+    # Trigger method for running on idle runs without messages
+    #
+    # @private
+    def on_idle
+      handle_idle
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.idle.error'
+      )
+    end
+
     # Trigger method for running on partition revocation.
     #
     # @private
@@ -143,51 +159,6 @@ module Karafka
     # some teardown procedures (closing file handler, etc).
     def shutdown; end
 
-    # Marks message as consumed in an async way.
-    #
-    # @param message [Messages::Message] last successfully processed message.
-    # @return [Boolean] true if we were able to mark the offset, false otherwise. False indicates
-    #   that we were not able and that we have lost the partition.
-    #
-    # @note We keep track of this offset in case we would mark as consumed and got error when
-    #   processing another message. In case like this we do not pause on the message we've already
-    #   processed but rather at the next one. This applies to both sync and async versions of this
-    #   method.
-    def mark_as_consumed(message)
-      # Ignore earlier offsets than the one we alread committed
-      return true if coordinator.seek_offset > message.offset
-
-      unless client.mark_as_consumed(message)
-        coordinator.revoke
-
-        return false
-      end
-
-      coordinator.seek_offset = message.offset + 1
-
-      true
-    end
-
-    # Marks message as consumed in a sync way.
-    #
-    # @param message [Messages::Message] last successfully processed message.
-    # @return [Boolean] true if we were able to mark the offset, false otherwise. False indicates
-    #   that we were not able and that we have lost the partition.
-    def mark_as_consumed!(message)
-      # Ignore earlier offsets than the one we alread committed
-      return true if coordinator.seek_offset > message.offset
-
-      unless client.mark_as_consumed!(message)
-        coordinator.revoke
-
-        return false
-      end
-
-      coordinator.seek_offset = message.offset + 1
-
-      true
-    end
-
     # Pauses processing on a given offset for the current topic partition
     #
     # After given partition is resumed, it will continue processing from the given offset
@@ -201,8 +172,8 @@ module Karafka
       timeout ? coordinator.pause_tracker.pause(timeout) : coordinator.pause_tracker.pause
 
       client.pause(
-        messages.metadata.topic,
-        messages.metadata.partition,
+        topic.name,
+        partition,
         offset
       )
 
@@ -213,8 +184,8 @@ module Karafka
         'consumer.consuming.pause',
         caller: self,
         manual: manual_pause,
-        topic: messages.metadata.topic,
-        partition: messages.metadata.partition,
+        topic: topic.name,
+        partition: partition,
         offset: offset,
         timeout: coordinator.pause_tracker.current_timeout,
         attempt: coordinator.pause_tracker.attempt
@@ -223,6 +194,8 @@ module Karafka
 
     # Resumes processing of the current topic partition
     def resume
+      return unless coordinator.pause_tracker.paused?
+
       # This is sufficient to expire a partition pause, as with it will be resumed by the listener
       # thread before the next poll.
       coordinator.pause_tracker.expire
@@ -234,8 +207,8 @@ module Karafka
     def seek(offset)
       client.seek(
         Karafka::Messages::Seek.new(
-          messages.metadata.topic,
-          messages.metadata.partition,
+          topic.name,
+          partition,
           offset
         )
       )
@@ -248,6 +221,13 @@ module Karafka
       coordinator.revoked?
     end
 
+    # @return [Boolean] are we retrying processing after an error. This can be used to provide a
+    #   different flow after there is an error, for example for resources cleanup, small manual
+    #   backoff or different instrumentation tracking.
+    def retrying?
+      coordinator.pause_tracker.attempt.positive?
+    end
+
     # Pauses the processing from the last offset to retry on given message
     # @private
     def retry_after_pause
@@ -258,8 +238,8 @@ module Karafka
       Karafka.monitor.instrument(
         'consumer.consuming.retry',
         caller: self,
-        topic: messages.metadata.topic,
-        partition: messages.metadata.partition,
+        topic: topic.name,
+        partition: partition,
         offset: coordinator.seek_offset,
         timeout: coordinator.pause_tracker.current_timeout,
         attempt: coordinator.pause_tracker.attempt

data/lib/karafka/connection/listener.rb

@@ -25,7 +25,7 @@ module Karafka
         @consumer_group_coordinator = consumer_group_coordinator
         @subscription_group = subscription_group
         @jobs_queue = jobs_queue
-        @coordinators = Processing::CoordinatorsBuffer.new
+        @coordinators = Processing::CoordinatorsBuffer.new(subscription_group.topics)
         @client = Client.new(@subscription_group)
         @executors = Processing::ExecutorsBuffer.new(@client, subscription_group)
         @jobs_builder = proc_config.jobs_builder
@@ -234,7 +234,7 @@ module Karafka
       def build_and_schedule_shutdown_jobs
         jobs = []
 
-        @executors.each do |_, _, executor|
+        @executors.each do |executor|
           job = @jobs_builder.shutdown(executor)
           job.before_enqueue
           jobs << job
@@ -263,20 +263,25 @@ module Karafka
 
         @messages_buffer.each do |topic, partition, messages|
           coordinator = @coordinators.find_or_create(topic, partition)
-
           # Start work coordination for this topic partition
           coordinator.start(messages)
 
-          @partitioner.call(topic, messages, coordinator) do |group_id, partition_messages|
-            # Count the job we're going to create here
-            coordinator.increment
-            executor = @executors.find_or_create(topic, partition, group_id)
-            job = @jobs_builder.consume(executor, partition_messages, coordinator)
-            job.before_enqueue
-            jobs << job
+          # We do not increment coordinator for idle job because it's not a user related one
+          # and it will not go through a standard lifecycle. Same applies to revoked and shutdown
+          if messages.empty?
+            executor = @executors.find_or_create(topic, partition, 0, coordinator)
+            jobs << @jobs_builder.idle(executor)
+          else
+            @partitioner.call(topic, messages, coordinator) do |group_id, partition_messages|
+              executor = @executors.find_or_create(topic, partition, group_id, coordinator)
+              coordinator.increment
+              jobs << @jobs_builder.consume(executor, partition_messages)
+            end
           end
         end
 
+        jobs.each(&:before_enqueue)
+
         @scheduler.schedule_consumption(@jobs_queue, jobs)
       end
 

data/lib/karafka/errors.rb

@@ -46,8 +46,5 @@ module Karafka
 
     # This should never happen. Please open an issue if it does.
     StrategyNotFoundError = Class.new(BaseError)
-
-    # This should never happen. Please open an issue if it does.
-    SkipMessageNotFoundError = Class.new(BaseError)
   end
 end

data/lib/karafka/instrumentation/logger_listener.rb

@@ -170,13 +170,51 @@ module Karafka
       #
       # @param event [Karafka::Core::Monitoring::Event] event details including payload
       def on_dead_letter_queue_dispatched(event)
+        consumer = event[:caller]
+        topic = consumer.topic.name
         message = event[:message]
         offset = message.offset
-        topic = event[:caller].topic.name
-        dlq_topic = event[:caller].topic.dead_letter_queue.topic
+        dlq_topic = consumer.topic.dead_letter_queue.topic
         partition = message.partition
 
-        info "Dispatched message #{offset} from #{topic}/#{partition} to DLQ topic: #{dlq_topic}"
+        info <<~MSG.tr("\n", ' ').strip!
+          [#{consumer.id}] Dispatched message #{offset}
+          from #{topic}/#{partition}
+          to DLQ topic: #{dlq_topic}
+        MSG
+      end
+
+      # Logs info about throttling event
+      #
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_filtering_throttled(event)
+        consumer = event[:caller]
+        topic = consumer.topic.name
+        # Here we get last message before throttle
+        message = event[:message]
+        partition = message.partition
+        offset = message.offset
+
+        info <<~MSG.tr("\n", ' ').strip!
+          [#{consumer.id}] Throttled and will resume
+          from message #{offset}
+          on #{topic}/#{partition}
+        MSG
+      end
+
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_filtering_seek(event)
+        consumer = event[:caller]
+        topic = consumer.topic.name
+        # Message to which we seek
+        message = event[:message]
+        partition = message.partition
+        offset = message.offset
+
+        info <<~MSG.tr("\n", ' ').strip!
+          [#{consumer.id}] Post-filtering seeking to message #{offset}
+          on #{topic}/#{partition}
+        MSG
       end
 
       # There are many types of errors that can occur in many places, but we provide a single
@@ -203,6 +241,9 @@ module Karafka
         when 'consumer.after_consume.error'
           error "Consumer after consume failed due to an error: #{error}"
           error details
+        when 'consumer.idle.error'
+          error "Consumer idle failed due to an error: #{error}"
+          error details
         when 'consumer.shutdown.error'
           error "Consumer on shutdown failed due to an error: #{error}"
           error details

data/lib/karafka/instrumentation/notifications.rb

@@ -39,6 +39,7 @@ module Karafka
         consumer.consumed
         consumer.consuming.pause
         consumer.consuming.retry
+        consumer.idle
         consumer.revoke
         consumer.revoked
         consumer.shutting_down
@@ -46,6 +47,9 @@ module Karafka
 
         dead_letter_queue.dispatched
 
+        filtering.throttled
+        filtering.seek
+
         process.notice_signal
 
         statistics.emitted

data/lib/karafka/pro/active_job/consumer.rb

@@ -29,7 +29,16 @@ module Karafka
             # If for any reason we've lost this partition, not worth iterating over new messages
             # as they are no longer ours
             break if revoked?
-            break if Karafka::App.stopping?
+
+            # We cannot early stop when running virtual partitions because the intermediate state
+            # would force us not to commit the offsets. This would cause extensive
+            # double-processing
+            break if Karafka::App.stopping? && !topic.virtual_partitions?
+
+            # Break if we already know, that one of virtual partitions has failed and we will
+            # be restarting processing all together after all VPs are done. This will minimize
+            # number of jobs that will be re-processed
+            break if topic.virtual_partitions? && failing?
 
             consume_job(message)
 

data/lib/karafka/pro/processing/coordinator.rb

@@ -17,6 +17,8 @@ module Karafka
       # Pro coordinator that provides extra orchestration methods useful for parallel processing
       # within the same partition
       class Coordinator < ::Karafka::Processing::Coordinator
+        attr_reader :filter
+
         # @param args [Object] anything the base coordinator accepts
         def initialize(*args)
           super
@@ -24,6 +26,7 @@ module Karafka
           @executed = []
           @flow_lock = Mutex.new
           @collapser = Collapser.new
+          @filter = FiltersApplier.new(self)
         end
 
         # Starts the coordination process
@@ -34,10 +37,10 @@ module Karafka
 
           @collapser.refresh!(messages.first.offset)
 
-          @mutex.synchronize do
-            @executed.clear
-            @last_message = messages.last
-          end
+          @filter.apply!(messages)
+
+          @executed.clear
+          @last_message = messages.last
         end
 
         # Sets the consumer failure status and additionally starts the collapse until
@@ -54,6 +57,12 @@ module Karafka
           @collapser.collapsed?
         end
 
+        # @return [Boolean] did any of the filters apply any logic that would cause use to run
+        #   the filtering flow
+        def filtered?
+          @filter.applied?
+        end
+
         # @return [Boolean] is the coordinated work finished or not
         def finished?
           @running_jobs.zero?

data/lib/karafka/pro/processing/filters/base.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Filters
+        # Base for all the filters.
+        # All filters (including custom) need to use this API.
+        #
+        # Due to the fact, that filters can limit data in such a way, that we need to pause or
+        # seek (throttling for example), the api is not just "remove some things from batch" but
+        # also provides ways to control the post-filtering operations that may be needed.
+        class Base
+          # @return [Karafka::Messages::Message, nil] the message that we want to use as a cursor
+          #   one to pause or seek or nil if not applicable.
+          attr_reader :cursor
+
+          include Karafka::Core::Helpers::Time
+
+          def initialize
+            @applied = false
+            @cursor = nil
+          end
+
+          # @param messages [Array<Karafka::Messages::Message>] array with messages. Please keep
+          #   in mind, this may already be partial due to execution of previous filters.
+          def apply!(messages)
+            raise NotImplementedError, 'Implement in a subclass'
+          end
+
+          # @return [Symbol] filter post-execution action on consumer. Either `:skip`, `:pause` or
+          #   `:seek`.
+          def action
+            :skip
+          end
+
+          # @return [Boolean] did this filter change messages in any way
+          def applied?
+            @applied
+          end
+
+          # @return [Integer] default timeout for pausing (if applicable)
+          def timeout
+            0
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/filters/delayer.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Filters
+        # A filter that allows us to delay processing by pausing until time is right.
+        class Delayer < Base
+          # @param delay [Integer] ms delay / minimum age of each message we want to process
+          def initialize(delay)
+            super()
+
+            @delay = delay
+          end
+
+          # Removes too old messages
+          #
+          # @param messages [Array<Karafka::Messages::Message>]
+          def apply!(messages)
+            @applied = false
+            @cursor = nil
+
+            # Time on message is in seconds with ms precision, so we need to convert the ttl that
+            # is in ms to this format
+            border = ::Time.now.utc - @delay / 1_000.to_f
+
+            messages.delete_if do |message|
+              too_young = message.timestamp > border
+
+              if too_young
+                @applied = true
+
+                @cursor ||= message
+              end
+
+              @applied
+            end
+          end
+
+          # @return [Integer] timeout delay in ms
+          def timeout
+            return 0 unless @cursor
+
+            timeout = (@delay / 1_000.to_f) - (::Time.now.utc - @cursor.timestamp)
+
+            timeout <= 0 ? 0 : timeout * 1_000
+          end
+
+          # @return [Symbol] action to take on post-filtering
+          def action
+            return :skip unless applied?
+
+            timeout <= 0 ? :seek : :pause
+          end
+        end
+      end
+    end
+  end
+end