karafka 2.0.0.beta2 → 2.0.0.beta5

data/lib/karafka/processing/coordinator.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Basic coordinator that allows us to provide coordination objects into consumers.
+    #
+    # This is a wrapping layer to simplify management of work to be handled around consumption.
+    #
+    # @note This coordinator needs to be thread safe. Some operations are performed only in the
+    #   listener thread, but we go with thread-safe by default for all not to worry about potential
+    #   future mistakes.
+    class Coordinator
+      # @return [Karafka::TimeTrackers::Pause]
+      attr_reader :pause_tracker
+
+      # @param pause_tracker [Karafka::TimeTrackers::Pause] pause tracker for given topic partition
+      def initialize(pause_tracker)
+        @pause_tracker = pause_tracker
+        @revoked = false
+        @consumptions = {}
+        @running_jobs = 0
+        @mutex = Mutex.new
+      end
+
+      # Starts the coordinator for given consumption jobs
+      def start
+        @mutex.synchronize do
+          @running_jobs = 0
+          # We need to clear the consumption results hash here, otherwise we could end up storing
+          # consumption results of consumer instances we no longer control
+          @consumptions.clear
+        end
+      end
+
+      # Increases number of jobs that we handle with this coordinator
+      def increment
+        @mutex.synchronize { @running_jobs += 1 }
+      end
+
+      # Decrements number of jobs we handle at the moment
+      def decrement
+        @mutex.synchronize do
+          @running_jobs -= 1
+
+          return @running_jobs unless @running_jobs.negative?
+
+          raise Karafka::Errors::InvalidCoordinatorState, @running_jobs
+        end
+      end
+
+      # @param consumer [Object] karafka consumer (normal or pro)
+      # @return [Karafka::Processing::Result] result object which we can use to indicate
+      #   consumption processing state.
+      def consumption(consumer)
+        @mutex.synchronize do
+          @consumptions[consumer] ||= Processing::Result.new
+        end
+      end
+
+      # Is all the consumption done and finished successfully for this coordinator
+      def success?
+        @mutex.synchronize { @running_jobs.zero? && @consumptions.values.all?(&:success?) }
+      end
+
+      # Marks given coordinator for processing group as revoked
+      #
+      # This is invoked in two places:
+      #   - from the main listener loop when we detect revoked partitions
+      #   - from the consumer in case checkpointing fails
+      #
+      # This means, we can end up having consumer being aware that it was revoked prior to the
+      # listener loop dispatching the revocation job. It is ok, as effectively nothing will be
+      # processed until revocation jobs are done.
+      def revoke
+        @mutex.synchronize { @revoked = true }
+      end
+
+      # @return [Boolean] is the partition we are processing revoked or not
+      def revoked?
+        @revoked
+      end
+    end
+  end
+end

data/lib/karafka/processing/coordinators_buffer.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Buffer used to build and store coordinators per topic partition
+    #
+    # It provides direct pauses access for revocation
+    #
+    # @note This buffer operates only from the listener loop, thus we do not have to make it
+    #   thread-safe.
+    class CoordinatorsBuffer
+      def initialize
+        @pauses_manager = Connection::PausesManager.new
+        @coordinator_class = ::Karafka::App.config.internal.processing.coordinator_class
+        @coordinators = Hash.new { |h, k| h[k] = {} }
+      end
+
+      # @param topic [String] topic name
+      # @param partition [Integer] partition number
+      def find_or_create(topic, partition)
+        @coordinators[topic][partition] ||= @coordinator_class.new(
+          @pauses_manager.fetch(topic, partition)
+        )
+      end
+
+      # Resumes processing of partitions for which pause time has ended.
+      # @param block we want to run for resumed topic partitions
+      # @yieldparam [String] topic name
+      # @yieldparam [Integer] partition number
+      def resume(&block)
+        @pauses_manager.resume(&block)
+      end
+
+      # @param topic [String] topic name
+      # @param partition [Integer] partition number
+      def revoke(topic, partition)
+        @pauses_manager.revoke(topic, partition)
+
+        # The fact that we delete here does not change the fact that the executor still holds the
+        # reference to this coordinator. We delete it here, as we will no longer process any
+        # new stuff with it and we may need a new coordinator if we regain this partition, but the
+        # coordinator may still be in use
+        coordinator = @coordinators[topic].delete(partition)
+
+        return unless coordinator
+
+        coordinator.revoke
+      end
+
+      # Clears coordinators and re-created the pauses manager
+      # This should be used only for critical errors recovery
+      def reset
+        @pauses_manager = Connection::PausesManager.new
+        @coordinators.clear
+      end
+    end
+  end
+end

data/lib/karafka/processing/executor.rb

@@ -18,18 +18,23 @@ module Karafka
       # @return [String] subscription group id to which a given executor belongs
       attr_reader :group_id
 
+      # @return [Karafka::Messages::Messages] messages batch
       attr_reader :messages
 
+      # Topic accessibility may be needed for the jobs builder to be able to build a proper job
+      # based on the topic settings defined by the end user
+      #
+      # @return [Karafka::Routing::Topic] topic of this executor
+      attr_reader :topic
+
       # @param group_id [String] id of the subscription group to which the executor belongs
       # @param client [Karafka::Connection::Client] kafka client
       # @param topic [Karafka::Routing::Topic] topic for which this executor will run
-      # @param pause_tracker [Karafka::TimeTrackers::Pause] fetch pause tracker for pausing
-      def initialize(group_id, client, topic, pause_tracker)
+      def initialize(group_id, client, topic)
         @id = SecureRandom.uuid
         @group_id = group_id
         @client = client
         @topic = topic
-        @pause_tracker = pause_tracker
       end
 
       # Builds the consumer instance, builds messages batch and sets all that is needed to run the
@@ -38,13 +43,16 @@ module Karafka
       # @param messages [Array<Karafka::Messages::Message>]
       # @param received_at [Time] the moment we've received the batch (actually the moment we've)
       #   enqueued it, but good enough
-      def prepare(messages, received_at)
+      # @param coordinator [Karafka::Processing::Coordinator] coordinator for processing management
+      def before_consume(messages, received_at, coordinator)
         # Recreate consumer with each batch if persistence is not enabled
         # We reload the consumers with each batch instead of relying on some external signals
         # when needed for consistency. That way devs may have it on or off and not in this
         # middle state, where re-creation of a consumer instance would occur only sometimes
         @consumer = nil unless ::Karafka::App.config.consumer_persistence
 
+        consumer.coordinator = coordinator
+
         # First we build messages batch...
         consumer.messages = Messages::Builders::Messages.call(
           messages,
@@ -52,7 +60,7 @@ module Karafka
           received_at
         )
 
-        consumer.on_prepared
+        consumer.on_before_consume
       end
 
       # Runs consumer data processing against given batch and handles failures and errors.
@@ -61,6 +69,11 @@ module Karafka
         consumer.on_consume
       end
 
+      # Runs consumer after consumption code
+      def after_consume
+        consumer.on_after_consume if @consumer
+      end
+
       # Runs the controller `#revoked` method that should be triggered when a given consumer is
       # no longer needed due to partitions reassignment.
       #
@@ -69,9 +82,12 @@ module Karafka
       #
       # @note We run it only when consumer was present, because presence indicates, that at least
       #   a single message has been consumed.
+      #
+      # @note We do not reset the consumer but we indicate need for recreation instead, because
+      #   after the revocation, there still may be `#after_consume` running that needs a given
+      #   consumer instance.
       def revoked
         consumer.on_revoked if @consumer
-        @consumer = nil
       end
 
       # Runs the controller `#shutdown` method that should be triggered when a given consumer is
@@ -83,7 +99,6 @@ module Karafka
         # There is a case, where the consumer no longer exists because it was revoked, in case like
         # that we do not build a new instance and shutdown should not be triggered.
         consumer.on_shutdown if @consumer
-        @consumer = nil
       end
 
       private
@@ -91,10 +106,9 @@ module Karafka
       # @return [Object] cached consumer instance
       def consumer
         @consumer ||= begin
-          consumer = @topic.consumer.new
+          consumer = @topic.consumer_class.new
           consumer.topic = @topic
           consumer.client = @client
-          consumer.pause_tracker = @pause_tracker
           consumer.producer = ::Karafka::App.producer
           consumer
         end

data/lib/karafka/processing/executors_buffer.rb

@@ -11,30 +11,48 @@ module Karafka
       def initialize(client, subscription_group)
         @subscription_group = subscription_group
         @client = client
-        @buffer = Hash.new { |h, k| h[k] = {} }
+        # We need two layers here to keep track of topics, partitions and processing groups
+        @buffer = Hash.new { |h, k| h[k] = Hash.new { |h2, k2| h2[k2] = {} } }
       end
 
+      # Finds or creates an executor based on the provided details
+      #
       # @param topic [String] topic name
       # @param partition [Integer] partition number
-      # @param pause [TimeTrackers::Pause] pause corresponding with provided topic and partition
+      # @param parallel_key [String] parallel group key
       # @return [Executor] consumer executor
-      def fetch(
-        topic,
-        partition,
-        pause
-      )
-        ktopic = @subscription_group.topics.find(topic)
+      def find_or_create(topic, partition, parallel_key)
+        ktopic = find_topic(topic)
 
-        ktopic || raise(Errors::TopicNotFoundError, topic)
-
-        @buffer[ktopic][partition] ||= Executor.new(
+        @buffer[ktopic][partition][parallel_key] ||= Executor.new(
           @subscription_group.id,
           @client,
-          ktopic,
-          pause
+          ktopic
         )
       end
 
+      # Revokes executors of a given topic partition, so they won't be used anymore for incoming
+      # messages
+      #
+      # @param topic [String] topic name
+      # @param partition [Integer] partition number
+      def revoke(topic, partition)
+        ktopic = find_topic(topic)
+
+        @buffer[ktopic][partition].clear
+      end
+
+      # Finds all the executors available for a given topic partition
+      #
+      # @param topic [String] topic name
+      # @param partition [Integer] partition number
+      # @return [Array<Executor>] executors in use for this topic + partition
+      def find_all(topic, partition)
+        ktopic = find_topic(topic)
+
+        @buffer[ktopic][partition].values
+      end
+
       # Iterates over all available executors and yields them together with topic and partition
       # info
       # @yieldparam [Routing::Topic] karafka routing topic object
@@ -42,8 +60,11 @@ module Karafka
       # @yieldparam [Executor] given executor
       def each
         @buffer.each do |ktopic, partitions|
-          partitions.each do |partition, executor|
-            yield(ktopic, partition, executor)
+          partitions.each do |partition, executors|
+            executors.each do |_parallel_key, executor|
+              # We skip the parallel key here as it does not serve any value when iterating
+              yield(ktopic, partition, executor)
+            end
           end
         end
       end
@@ -52,6 +73,16 @@ module Karafka
       def clear
         @buffer.clear
       end
+
+      private
+
+      # Finds topic based on its name
+      #
+      # @param topic [String] topic we're looking for
+      # @return [Karafka::Routing::Topic] topic we're interested in
+      def find_topic(topic)
+        @subscription_group.topics.find(topic) || raise(Errors::TopicNotFoundError, topic)
+      end
     end
   end
 end

data/lib/karafka/processing/jobs/base.rb

@@ -5,7 +5,7 @@ module Karafka
     # Namespace for all the jobs that are suppose to run in workers.
     module Jobs
       # Base class for all the jobs types that are suppose to run in workers threads.
-      # Each job can have 3 main entry-points: `#prepare`, `#call` and `#teardown`
+      # Each job can have 3 main entry-points: `#before_call`, `#call` and `#after_call`
       # Only `#call` is required.
       class Base
         extend Forwardable
@@ -23,10 +23,15 @@ module Karafka
         end
 
         # When redefined can run any code that should run before executing the proper code
-        def prepare; end
+        def before_call; end
+
+        # The main entry-point of a job
+        def call
+          raise NotImplementedError, 'Please implement in a subclass'
+        end
 
         # When redefined can run any code that should run after executing the proper code
-        def teardown; end
+        def after_call; end
 
         # @return [Boolean] is this a non-blocking job
         #

data/lib/karafka/processing/jobs/consume.rb

@@ -12,23 +12,30 @@ module Karafka
         # @param executor [Karafka::Processing::Executor] executor that is suppose to run a given
         #   job
         # @param messages [Karafka::Messages::Messages] karafka messages batch
+        # @param coordinator [Karafka::Processing::Coordinator] processing coordinator
         # @return [Consume]
-        def initialize(executor, messages)
+        def initialize(executor, messages, coordinator)
           @executor = executor
           @messages = messages
+          @coordinator = coordinator
           @created_at = Time.now
           super()
         end
 
-        # Runs the preparations on the executor
-        def prepare
-          executor.prepare(@messages, @created_at)
+        # Runs the before consumption preparations on the executor
+        def before_call
+          executor.before_consume(@messages, @created_at, @coordinator)
         end
 
         # Runs the given executor
         def call
           executor.consume
         end
+
+        # Runs any error handling and other post-consumption stuff on the executor
+        def after_call
+          executor.after_consume
+        end
       end
     end
   end

data/lib/karafka/processing/jobs_builder.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Class responsible for deciding what type of job should we build to run a given command and
+    # for building a proper job for it.
+    class JobsBuilder
+      # @param executor [Karafka::Processing::Executor]
+      # @param messages [Karafka::Messages::Messages] messages batch to be consumed
+      # @param coordinator [Karafka::Processing::Coordinator]
+      # @return [Karafka::Processing::Jobs::Consume] consumption job
+      def consume(executor, messages, coordinator)
+        Jobs::Consume.new(executor, messages, coordinator)
+      end
+
+      # @param executor [Karafka::Processing::Executor]
+      # @return [Karafka::Processing::Jobs::Revoked] revocation job
+      def revoked(executor)
+        Jobs::Revoked.new(executor)
+      end
+
+      # @param executor [Karafka::Processing::Executor]
+      # @return [Karafka::Processing::Jobs::Shutdown] shutdown job
+      def shutdown(executor)
+        Jobs::Shutdown.new(executor)
+      end
+    end
+  end
+end

data/lib/karafka/processing/result.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # A simple object that allows us to keep track of processing state.
+    # It allows to indicate if given thing moved from success to a failure or the other way around
+    # Useful for tracking consumption state
+    class Result
+      def initialize
+        @success = true
+      end
+
+      # @return [Boolean]
+      def success?
+        @success
+      end
+
+      # Marks state as successful
+      def success!
+        @success = true
+      end
+
+      # Marks state as failure
+      def failure!
+        @success = false
+      end
+    end
+  end
+end

data/lib/karafka/processing/scheduler.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # FIFO scheduler for messages coming from various topics and partitions
+    class Scheduler
+      # Schedules jobs in the fifo order
+      #
+      # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
+      # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
+      def schedule_consumption(queue, jobs_array)
+        jobs_array.each do |job|
+          queue << job
+        end
+      end
+
+      # Both revocation and shutdown jobs can also run in fifo by default
+      alias schedule_revocation schedule_consumption
+      alias schedule_shutdown schedule_consumption
+    end
+  end
+end

data/lib/karafka/processing/worker.rb

@@ -19,9 +19,13 @@ module Karafka
     class Worker
       include Helpers::Async
 
+      # @return [String] id of this worker
+      attr_reader :id
+
       # @param jobs_queue [JobsQueue]
       # @return [Worker]
       def initialize(jobs_queue)
+        @id = SecureRandom.uuid
         @jobs_queue = jobs_queue
       end
 
@@ -43,19 +47,23 @@ module Karafka
         job = @jobs_queue.pop
 
         if job
-          job.prepare
+          Karafka.monitor.instrument('worker.process', caller: self, job: job)
+
+          Karafka.monitor.instrument('worker.processed', caller: self, job: job) do
+            job.before_call
 
-          # If a job is marked as non blocking, we can run a tick in the job queue and if there
-          # are no other blocking factors, the job queue will be unlocked.
-          # If this does not run, all the things will be blocking and job queue won't allow to
-          # pass it until done.
-          @jobs_queue.tick(job.group_id) if job.non_blocking?
+            # If a job is marked as non blocking, we can run a tick in the job queue and if there
+            # are no other blocking factors, the job queue will be unlocked.
+            # If this does not run, all the things will be blocking and job queue won't allow to
+            # pass it until done.
+            @jobs_queue.tick(job.group_id) if job.non_blocking?
 
-          job.call
+            job.call
 
-          job.teardown
+            job.after_call
 
-          true
+            true
+          end
         else
           false
         end

data/lib/karafka/routing/consumer_group.rb

@@ -38,7 +38,7 @@ module Karafka
       # @return [Array<Routing::SubscriptionGroup>] all the subscription groups build based on
       #   the consumer group topics
       def subscription_groups
-        App.config.internal.subscription_groups_builder.call(topics)
+        App.config.internal.routing.subscription_groups_builder.call(topics)
       end
 
       # Hashed version of consumer group that can be used for validation purposes

data/lib/karafka/routing/subscription_group.rb

@@ -44,7 +44,7 @@ module Karafka
         kafka[:'auto.offset.reset'] ||= @topics.first.initial_offset
         # Karafka manages the offsets based on the processing state, thus we do not rely on the
         # rdkafka offset auto-storing
-        kafka[:'enable.auto.offset.store'] = 'false'
+        kafka[:'enable.auto.offset.store'] = false
         kafka.freeze
         kafka
       end

data/lib/karafka/routing/topic.rb

@@ -66,6 +66,20 @@ module Karafka
         end
       end
 
+      # @return [Class] consumer class that we should use
+      # @note This is just an alias to the `#consumer` method. We however want to use it internally
+      #   instead of referencing the `#consumer`. We use this to indicate that this method returns
+      #   class and not an instance. In the routing we want to keep the `#consumer Consumer`
+      #   routing syntax, but for references outside, we should use this one.
+      def consumer_class
+        consumer
+      end
+
+      # @return [Boolean] true if this topic offset is handled by the end user
+      def manual_offset_management?
+        manual_offset_management
+      end
+
       # @return [Hash] hash with all the topic attributes
       # @note This is being used when we validate the consumer_group and its topics
       def to_h

data/lib/karafka/setup/config.rb

@@ -83,21 +83,31 @@ module Karafka
       # @see https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
       setting :kafka, default: {}
 
-      # Namespace for internal settings that should not be modified
+      # Namespace for internal settings that should not be modified directly
       setting :internal do
-        # option routing_builder [Karafka::Routing::Builder] builder instance
-        setting :routing_builder, default: Routing::Builder.new
         # option status [Karafka::Status] app status
         setting :status, default: Status.new
         # option process [Karafka::Process] process status
         # @note In the future, we need to have a single process representation for all the karafka
         #   instances
         setting :process, default: Process.new
-        # option subscription_groups_builder [Routing::SubscriptionGroupsBuilder] subscription
-        #   group builder
-        setting :subscription_groups_builder, default: Routing::SubscriptionGroupsBuilder.new
-        # option scheduler [Class] scheduler we will be using
-        setting :scheduler, default: Scheduler.new
+
+        setting :routing do
+          # option builder [Karafka::Routing::Builder] builder instance
+          setting :builder, default: Routing::Builder.new
+          # option subscription_groups_builder [Routing::SubscriptionGroupsBuilder] subscription
+          #   group builder
+          setting :subscription_groups_builder, default: Routing::SubscriptionGroupsBuilder.new
+        end
+
+        setting :processing do
+          # option scheduler [Object] scheduler we will be using
+          setting :scheduler, default: Processing::Scheduler.new
+          # option jobs_builder [Object] jobs builder we want to use
+          setting :jobs_builder, default: Processing::JobsBuilder.new
+          # option coordinator [Class] work coordinator we want to user for processing coordination
+          setting :coordinator_class, default: Processing::Coordinator
+        end
 
         # Karafka components for ActiveJob
         setting :active_job do
@@ -107,7 +117,7 @@ module Karafka
           #   ensuring, that extra job options defined are valid
           setting :job_options_contract, default: ActiveJob::JobOptionsContract.new
           # option consumer [Class] consumer class that should be used to consume ActiveJob data
-          setting :consumer, default: ActiveJob::Consumer
+          setting :consumer_class, default: ActiveJob::Consumer
         end
       end
 

data/lib/karafka/status.rb

@@ -31,9 +31,7 @@ module Karafka
         # We skip as during this state we do not have yet a monitor
         return if initializing?
 
-        # Trap context disallows to run certain things that we instrument
-        # so the state changes are executed from a separate thread
-        Thread.new { Karafka.monitor.instrument("app.#{state}") }.join
+        Karafka.monitor.instrument("app.#{state}")
       end
     end
   end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.0.beta2'
+  VERSION = '2.0.0.beta5'
 end