karafka 2.0.0 → 2.1.0

data/lib/karafka/processing/coordinator.rb

@@ -10,60 +10,105 @@ module Karafka
     #   listener thread, but we go with thread-safe by default for all not to worry about potential
     #   future mistakes.
     class Coordinator
-      # @return [Karafka::TimeTrackers::Pause]
-      attr_reader :pause_tracker
+      attr_reader :pause_tracker, :seek_offset, :topic, :partition
 
+      # @param topic [Karafka::Routing::Topic]
+      # @param partition [Integer]
       # @param pause_tracker [Karafka::TimeTrackers::Pause] pause tracker for given topic partition
-      def initialize(pause_tracker)
+      def initialize(topic, partition, pause_tracker)
+        @topic = topic
+        @partition = partition
         @pause_tracker = pause_tracker
         @revoked = false
         @consumptions = {}
         @running_jobs = 0
+        @manual_pause = false
         @mutex = Mutex.new
+        @marked = false
+        @failure = false
       end
 
       # Starts the coordinator for given consumption jobs
-      # @param _messages [Array<Karafka::Messages::Message>] batch of message for which we are
+      # @param messages [Array<Karafka::Messages::Message>] batch of message for which we are
       #   going to coordinate work. Not used with regular coordinator.
-      def start(_messages)
-        @mutex.synchronize do
-          @running_jobs = 0
-          # We need to clear the consumption results hash here, otherwise we could end up storing
-          # consumption results of consumer instances we no longer control
-          @consumptions.clear
+      def start(messages)
+        @failure = false
+        @running_jobs = 0
+        # We need to clear the consumption results hash here, otherwise we could end up storing
+        # consumption results of consumer instances we no longer control
+        @consumptions.clear
+
+        # When starting to run, no pause is expected and no manual pause as well
+        @manual_pause = false
+
+        # We set it on the first encounter and never again, because then the offset setting
+        # should be up to the consumers logic (our or the end user)
+        # Seek offset needs to be always initialized as for case where manual offset management
+        # is turned on, we need to have reference to the first offset even in case of running
+        # multiple batches without marking any messages as consumed. Rollback needs to happen to
+        # the last place we know of or the last message + 1 that was marked
+        #
+        # It is however worth keeping in mind, that this may need to be used with `#marked?` to
+        # make sure that the first offset is an offset that has been marked.
+        @seek_offset ||= messages.first.offset
+      end
+
+      # @param offset [Integer] message offset
+      def seek_offset=(offset)
+        synchronize do
+          @marked = true
+          @seek_offset = offset
         end
       end
 
       # Increases number of jobs that we handle with this coordinator
       def increment
-        @mutex.synchronize { @running_jobs += 1 }
+        synchronize { @running_jobs += 1 }
       end
 
       # Decrements number of jobs we handle at the moment
       def decrement
-        @mutex.synchronize do
+        synchronize do
           @running_jobs -= 1
 
           return @running_jobs unless @running_jobs.negative?
 
           # This should never happen. If it does, something is heavily out of sync. Please reach
           # out to us if you encounter this
-          raise Karafka::Errors::InvalidCoordinatorState, 'Was zero before decrementation'
+          raise Karafka::Errors::InvalidCoordinatorStateError, 'Was zero before decrementation'
         end
       end
 
-      # @param consumer [Object] karafka consumer (normal or pro)
-      # @return [Karafka::Processing::Result] result object which we can use to indicate
-      #   consumption processing state.
-      def consumption(consumer)
-        @mutex.synchronize do
-          @consumptions[consumer] ||= Processing::Result.new
+      # Is all the consumption done and finished successfully for this coordinator
+      # We do not say we're successful until all work is done, because running work may still
+      # crash.
+      def success?
+        synchronize do
+          @running_jobs.zero? && @consumptions.values.all?(&:success?)
         end
       end
 
-      # Is all the consumption done and finished successfully for this coordinator
-      def success?
-        @mutex.synchronize { @running_jobs.zero? && @consumptions.values.all?(&:success?) }
+      # Mark given consumption on consumer as successful
+      # @param consumer [Karafka::BaseConsumer] consumer that finished successfully
+      def success!(consumer)
+        synchronize do
+          consumption(consumer).success!
+        end
+      end
+
+      # Mark given consumption on consumer as failed
+      # @param consumer [Karafka::BaseConsumer] consumer that failed
+      # @param error [StandardError] error that occurred
+      def failure!(consumer, error)
+        synchronize do
+          @failure = true
+          consumption(consumer).failure!(error)
+        end
+      end
+
+      # @return [Boolean] true if any of work we were running failed
+      def failure?
+        @failure
       end
 
       # Marks given coordinator for processing group as revoked
@@ -76,13 +121,47 @@ module Karafka
       # listener loop dispatching the revocation job. It is ok, as effectively nothing will be
       # processed until revocation jobs are done.
       def revoke
-        @mutex.synchronize { @revoked = true }
+        synchronize { @revoked = true }
       end
 
       # @return [Boolean] is the partition we are processing revoked or not
       def revoked?
         @revoked
       end
+
+      # @return [Boolean] was the new seek offset assigned at least once. This is needed because
+      #   by default we assign seek offset of a first message ever, however this is insufficient
+      #   for DLQ in a scenario where the first message would be broken. We would never move
+      #   out of it and would end up in an endless loop.
+      def marked?
+        @marked
+      end
+
+      # Store in the coordinator info, that this pause was done manually by the end user and not
+      # by the system itself
+      def manual_pause
+        @manual_pause = true
+      end
+
+      # @return [Boolean] are we in a pause that was initiated by the user
+      def manual_pause?
+        @pause_tracker.paused? && @manual_pause
+      end
+
+      # Allows to run synchronized (locked) code that can operate in between virtual partitions
+      # @param block [Proc] code we want to run in the synchronized mode
+      def synchronize(&block)
+        @mutex.synchronize(&block)
+      end
+
+      private
+
+      # @param consumer [Object] karafka consumer (normal or pro)
+      # @return [Karafka::Processing::Result] result object which we can use to indicate
+      #   consumption processing state.
+      def consumption(consumer)
+        @consumptions[consumer] ||= Processing::Result.new
+      end
     end
   end
 end

data/lib/karafka/processing/coordinators_buffer.rb

@@ -9,16 +9,20 @@ module Karafka
     # @note This buffer operates only from the listener loop, thus we do not have to make it
     #   thread-safe.
     class CoordinatorsBuffer
-      def initialize
+      # @param topics [Karafka::Routing::Topics]
+      def initialize(topics)
         @pauses_manager = Connection::PausesManager.new
         @coordinator_class = ::Karafka::App.config.internal.processing.coordinator_class
         @coordinators = Hash.new { |h, k| h[k] = {} }
+        @topics = topics
       end
 
       # @param topic [String] topic name
       # @param partition [Integer] partition number
       def find_or_create(topic, partition)
         @coordinators[topic][partition] ||= @coordinator_class.new(
+          @topics.find(topic),
+          partition,
           @pauses_manager.fetch(topic, partition)
         )
       end

data/lib/karafka/processing/executor.rb

@@ -12,6 +12,10 @@ module Karafka
     # @note Executors are not removed after partition is revoked. They are not that big and will
     #   be re-used in case of a re-claim
     class Executor
+      extend Forwardable
+
+      def_delegators :@coordinator, :topic, :partition
+
       # @return [String] unique id that we use to ensure, that we use for state tracking
       attr_reader :id
 
@@ -21,45 +25,46 @@ module Karafka
       # @return [Karafka::Messages::Messages] messages batch
       attr_reader :messages
 
-      # Topic accessibility may be needed for the jobs builder to be able to build a proper job
-      # based on the topic settings defined by the end user
-      #
-      # @return [Karafka::Routing::Topic] topic of this executor
-      attr_reader :topic
+      # @return [Karafka::Processing::Coordinator] coordinator for this executor
+      attr_reader :coordinator
 
       # @param group_id [String] id of the subscription group to which the executor belongs
       # @param client [Karafka::Connection::Client] kafka client
-      # @param topic [Karafka::Routing::Topic] topic for which this executor will run
-      def initialize(group_id, client, topic)
-        @id = SecureRandom.uuid
+      # @param coordinator [Karafka::Processing::Coordinator]
+      def initialize(group_id, client, coordinator)
+        @id = SecureRandom.hex(6)
         @group_id = group_id
         @client = client
-        @topic = topic
+        @coordinator = coordinator
       end
 
-      # Builds the consumer instance, builds messages batch and sets all that is needed to run the
-      # user consumption logic
+      # Allows us to prepare the consumer in the listener thread prior to the job being send to
+      # the queue. It also allows to run some code that is time sensitive and cannot wait in the
+      # queue as it could cause starvation.
       #
       # @param messages [Array<Karafka::Messages::Message>]
-      # @param received_at [Time] the moment we've received the batch (actually the moment we've)
-      #   enqueued it, but good enough
-      # @param coordinator [Karafka::Processing::Coordinator] coordinator for processing management
-      def before_consume(messages, received_at, coordinator)
+      def before_enqueue(messages)
         # Recreate consumer with each batch if persistence is not enabled
         # We reload the consumers with each batch instead of relying on some external signals
         # when needed for consistency. That way devs may have it on or off and not in this
         # middle state, where re-creation of a consumer instance would occur only sometimes
         @consumer = nil unless ::Karafka::App.config.consumer_persistence
 
-        consumer.coordinator = coordinator
-
         # First we build messages batch...
         consumer.messages = Messages::Builders::Messages.call(
           messages,
-          @topic,
-          received_at
+          topic,
+          partition,
+          # the moment we've received the batch or actually the moment we've enqueued it,
+          # but good enough
+          Time.now
         )
 
+        consumer.on_before_enqueue
+      end
+
+      # Runs setup and warm-up code in the worker prior to running the consumption
+      def before_consume
         consumer.on_before_consume
       end
 
@@ -74,6 +79,23 @@ module Karafka
         consumer.on_after_consume
       end
 
+      # Runs consumer idle operations
+      # This may include house-keeping or other state management changes that can occur but that
+      # not mean there are any new messages available for the end user to process
+      def idle
+        # Initializes the messages set in case idle operation would happen before any processing
+        # This prevents us from having no messages object at all as the messages object and
+        # its metadata may be used for statistics
+        consumer.messages ||= Messages::Builders::Messages.call(
+          [],
+          topic,
+          partition,
+          Time.now
+        )
+
+        consumer.on_idle
+      end
+
       # Runs the controller `#revoked` method that should be triggered when a given consumer is
       # no longer needed due to partitions reassignment.
       #
@@ -106,10 +128,18 @@ module Karafka
       # @return [Object] cached consumer instance
       def consumer
         @consumer ||= begin
-          consumer = @topic.consumer_class.new
-          consumer.topic = @topic
+          topic = @coordinator.topic
+
+          strategy = ::Karafka::App.config.internal.processing.strategy_selector.find(topic)
+
+          consumer = topic.consumer_class.new
+          # We use singleton class as the same consumer class may be used to process different
+          # topics with different settings
+          consumer.singleton_class.include(strategy)
           consumer.client = @client
           consumer.producer = ::Karafka::App.producer
+          consumer.coordinator = @coordinator
+
           consumer
         end
       end

data/lib/karafka/processing/executors_buffer.rb

@@ -20,14 +20,13 @@ module Karafka
       # @param topic [String] topic name
       # @param partition [Integer] partition number
       # @param parallel_key [String] parallel group key
+      # @param coordinator [Karafka::Processing::Coordinator]
       # @return [Executor] consumer executor
-      def find_or_create(topic, partition, parallel_key)
-        ktopic = find_topic(topic)
-
-        @buffer[ktopic][partition][parallel_key] ||= Executor.new(
+      def find_or_create(topic, partition, parallel_key, coordinator)
+        @buffer[topic][partition][parallel_key] ||= Executor.new(
           @subscription_group.id,
           @client,
-          ktopic
+          coordinator
         )
       end
 
@@ -37,9 +36,7 @@ module Karafka
       # @param topic [String] topic name
       # @param partition [Integer] partition number
       def revoke(topic, partition)
-        ktopic = find_topic(topic)
-
-        @buffer[ktopic][partition].clear
+        @buffer[topic][partition].clear
       end
 
       # Finds all the executors available for a given topic partition
@@ -48,9 +45,7 @@ module Karafka
       # @param partition [Integer] partition number
       # @return [Array<Executor>] executors in use for this topic + partition
       def find_all(topic, partition)
-        ktopic = find_topic(topic)
-
-        @buffer[ktopic][partition].values
+        @buffer[topic][partition].values
       end
 
       # Iterates over all available executors and yields them together with topic and partition
@@ -59,11 +54,10 @@ module Karafka
       # @yieldparam [Integer] partition number
       # @yieldparam [Executor] given executor
       def each
-        @buffer.each do |ktopic, partitions|
-          partitions.each do |partition, executors|
-            executors.each do |_parallel_key, executor|
-              # We skip the parallel key here as it does not serve any value when iterating
-              yield(ktopic, partition, executor)
+        @buffer.each do |_, partitions|
+          partitions.each do |_, executors|
+            executors.each do |_, executor|
+              yield(executor)
             end
           end
         end
@@ -73,16 +67,6 @@ module Karafka
       def clear
         @buffer.clear
       end
-
-      private
-
-      # Finds topic based on its name
-      #
-      # @param topic [String] topic we're looking for
-      # @return [Karafka::Routing::Topic] topic we're interested in
-      def find_topic(topic)
-        @subscription_group.topics.find(topic) || raise(Errors::TopicNotFoundError, topic)
-      end
     end
   end
 end

data/lib/karafka/processing/jobs/base.rb

@@ -2,7 +2,7 @@
 
 module Karafka
   module Processing
-    # Namespace for all the jobs that are suppose to run in workers.
+    # Namespace for all the jobs that are supposed to run in workers.
     module Jobs
       # Base class for all the jobs types that are suppose to run in workers threads.
       # Each job can have 3 main entry-points: `#before_call`, `#call` and `#after_call`
@@ -10,7 +10,7 @@ module Karafka
       class Base
         extend Forwardable
 
-        # @note Since one job has always one executer, we use the jobs id and group id as reference
+        # @note Since one job has always one executor, we use the jobs id and group id as reference
         def_delegators :executor, :id, :group_id
 
         attr_reader :executor
@@ -22,6 +22,10 @@ module Karafka
           @non_blocking = false
         end
 
+        # When redefined can run any code prior to the job being enqueued
+        # @note This will run in the listener thread and not in the worker
+        def before_enqueue; end
+
         # When redefined can run any code that should run before executing the proper code
         def before_call; end
 

data/lib/karafka/processing/jobs/consume.rb

@@ -12,19 +12,22 @@ module Karafka
         # @param executor [Karafka::Processing::Executor] executor that is suppose to run a given
         #   job
         # @param messages [Karafka::Messages::Messages] karafka messages batch
-        # @param coordinator [Karafka::Processing::Coordinator] processing coordinator
         # @return [Consume]
-        def initialize(executor, messages, coordinator)
+        def initialize(executor, messages)
           @executor = executor
           @messages = messages
-          @coordinator = coordinator
-          @created_at = Time.now
           super()
         end
 
+        # Runs all the preparation code on the executor that needs to happen before the job is
+        # enqueued.
+        def before_enqueue
+          executor.before_enqueue(@messages)
+        end
+
         # Runs the before consumption preparations on the executor
         def before_call
-          executor.before_consume(@messages, @created_at, @coordinator)
+          executor.before_consume
         end
 
         # Runs the given executor

data/lib/karafka/processing/jobs/idle.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Jobs
+      # Type of job that we may use to run some extra handling that happens without the user
+      # related lifecycle event like consumption, revocation, etc.
+      class Idle < Base
+        # @param executor [Karafka::Processing::Executor] executor that is suppose to run a given
+        #   job on an active consumer
+        # @return [Shutdown]
+        def initialize(executor)
+          @executor = executor
+          super()
+        end
+
+        # Run the idle work via the executor
+        def call
+          executor.idle
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/jobs_builder.rb

@@ -7,10 +7,9 @@ module Karafka
     class JobsBuilder
       # @param executor [Karafka::Processing::Executor]
       # @param messages [Karafka::Messages::Messages] messages batch to be consumed
-      # @param coordinator [Karafka::Processing::Coordinator]
       # @return [Karafka::Processing::Jobs::Consume] consumption job
-      def consume(executor, messages, coordinator)
-        Jobs::Consume.new(executor, messages, coordinator)
+      def consume(executor, messages)
+        Jobs::Consume.new(executor, messages)
       end
 
       # @param executor [Karafka::Processing::Executor]

data/lib/karafka/processing/jobs_queue.rb

@@ -13,15 +13,19 @@ module Karafka
       # @return [Karafka::Processing::JobsQueue]
       def initialize
         @queue = Queue.new
-        # Those queues will act as a semaphores internally. Since we need an indicator for waiting
+        # Those queues will act as semaphores internally. Since we need an indicator for waiting
         # we could use Thread.pass but this is expensive. Instead we can just lock until any
         # of the workers finishes their work and we can re-check. This means that in the worse
         # scenario, we will context switch 10 times per poll instead of getting this thread
         # scheduled by Ruby hundreds of thousands of times per group.
         # We cannot use a single semaphore as it could potentially block in listeners that should
         # process with their data and also could unlock when a given group needs to remain locked
-        @semaphores = Hash.new { |h, k| h[k] = Queue.new }
+        @semaphores = Concurrent::Map.new do |h, k|
+          h.compute_if_absent(k) { Queue.new }
+        end
+
         @in_processing = Hash.new { |h, k| h[k] = [] }
+
         @mutex = Mutex.new
       end
 
@@ -47,9 +51,9 @@ module Karafka
           raise(Errors::JobsQueueSynchronizationError, job.group_id) if group.include?(job)
 
           group << job
-        end
 
-        @queue << job
+          @queue << job
+        end
       end
 
       # @return [Jobs::Base, nil] waits for a job from the main queue and returns it once available
@@ -96,7 +100,7 @@ module Karafka
           return if @queue.closed?
 
           @queue.close
-          @semaphores.values.each(&:close)
+          @semaphores.each_value(&:close)
         end
       end
 
@@ -105,7 +109,9 @@ module Karafka
       # @return [Boolean] tell us if we have anything in the processing (or for processing) from
       # a given group.
       def empty?(group_id)
-        @in_processing[group_id].empty?
+        @mutex.synchronize do
+          @in_processing[group_id].empty?
+        end
       end
 
       # Blocks when there are things in the queue in a given group and waits until all the blocking
@@ -119,13 +125,13 @@ module Karafka
         @semaphores[group_id].pop while wait?(group_id)
       end
 
-      # - `processing` - number of jobs that are currently being processed (active work)
+      # - `busy` - number of jobs that are currently being processed (active work)
       # - `enqueued` - number of jobs in the queue that are waiting to be picked up by a worker
       #
       # @return [Hash] hash with basic usage statistics of this queue.
       def statistics
         {
-          processing: size - @queue.size,
+          busy: size - @queue.size,
           enqueued: @queue.size
         }.freeze
       end

data/lib/karafka/processing/partitioner.rb

@@ -12,9 +12,11 @@ module Karafka
 
       # @param _topic [String] topic name
       # @param messages [Array<Karafka::Messages::Message>] karafka messages
+      # @param _coordinator [Karafka::Processing::Coordinator] processing coordinator that will
+      #   be used with those messages
       # @yieldparam [Integer] group id
       # @yieldparam [Array<Karafka::Messages::Message>] karafka messages
-      def call(_topic, messages)
+      def call(_topic, messages, _coordinator)
         yield(0, messages)
       end
     end

data/lib/karafka/processing/result.rb

@@ -6,8 +6,11 @@ module Karafka
     # It allows to indicate if given thing moved from success to a failure or the other way around
     # Useful for tracking consumption state
     class Result
+      attr_reader :cause
+
       def initialize
         @success = true
+        @cause = false
       end
 
       # @return [Boolean]
@@ -18,11 +21,21 @@ module Karafka
       # Marks state as successful
       def success!
         @success = true
+        # We set cause to false so the previous error that occurred does not leak when error is
+        # no longer present
+        @cause = false
       end
 
       # Marks state as failure
-      def failure!
+      # @param cause [StandardError] error that occurred and caused failure
+      def failure!(cause)
         @success = false
+        @cause = cause
+      end
+
+      # @return [Boolean] true if processing failed
+      def failure?
+        !@success
       end
     end
   end

data/lib/karafka/processing/strategies/aj_dlq_mom.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Strategies
+      # ActiveJob strategy to cooperate with the DLQ.
+      #
+      # While AJ is uses MOM by default because it delegates the offset management to the AJ
+      # consumer. With DLQ however there is an extra case for skipping broken jobs with offset
+      # marking due to ordered processing.
+      module AjDlqMom
+        include DlqMom
+
+        # Apply strategy when only when using AJ with MOM and DLQ
+        FEATURES = %i[
+          active_job
+          dead_letter_queue
+          manual_offset_management
+        ].freeze
+
+        # How should we post-finalize consumption.
+        def handle_after_consume
+          return if revoked?
+
+          if coordinator.success?
+            # Do NOT commit offsets, they are comitted after each job in the AJ consumer.
+            coordinator.pause_tracker.reset
+          elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+            retry_after_pause
+          else
+            coordinator.pause_tracker.reset
+            skippable_message, = find_skippable_message
+            dispatch_to_dlq(skippable_message)
+            # We can commit the offset here because we know that we skip it "forever" and
+            # since AJ consumer commits the offset after each job, we also know that the
+            # previous job was successful
+            mark_as_consumed(skippable_message)
+            pause(coordinator.seek_offset, nil, false)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/strategies/aj_mom.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Strategies
+      # ActiveJob enabled
+      # Manual offset management enabled
+      #
+      # This is the default AJ strategy since AJ cannot be used without MOM
+      module AjMom
+        include Mom
+
+        # Apply strategy when only when using AJ with MOM
+        FEATURES = %i[
+          active_job
+          manual_offset_management
+        ].freeze
+      end
+    end
+  end
+end