karafka 2.0.0.alpha6 → 2.0.0.beta3

data/lib/karafka/processing/executor.rb

@@ -4,10 +4,10 @@ module Karafka
   # Namespace that encapsulates all the logic related to processing data.
   module Processing
     # Executors:
-    # - run consumers code with provided messages batch (for `#call`) or run given teardown
-    #   operations when needed from separate threads.
-    # - they re-create consumer instances in case of partitions that were revoked
-    #   and assigned back.
+    # - run consumers code (for `#call`) or run given preparation / teardown operations when needed
+    #   from separate threads.
+    # - they re-create consumer instances in case of partitions that were revoked and assigned
+    #   back.
     #
     # @note Executors are not removed after partition is revoked. They are not that big and will
     #   be re-used in case of a re-claim
@@ -18,24 +18,34 @@ module Karafka
       # @return [String] subscription group id to which a given executor belongs
       attr_reader :group_id
 
+      # @return [Karafka::Messages::Messages] messages batch
+      attr_reader :messages
+
+      # Topic accessibility may be needed for the jobs builder to be able to build a proper job
+      # based on the topic settings defined by the end user
+      #
+      # @return [Karafka::Routing::Topic] topic of this executor
+      attr_reader :topic
+
       # @param group_id [String] id of the subscription group to which the executor belongs
       # @param client [Karafka::Connection::Client] kafka client
       # @param topic [Karafka::Routing::Topic] topic for which this executor will run
-      # @param pause [Karafka::TimeTrackers::Pause] fetch pause object for crash pausing
-      def initialize(group_id, client, topic, pause)
+      # @param pause_tracker [Karafka::TimeTrackers::Pause] fetch pause tracker for pausing
+      def initialize(group_id, client, topic, pause_tracker)
         @id = SecureRandom.uuid
         @group_id = group_id
         @client = client
         @topic = topic
-        @pause = pause
+        @pause_tracker = pause_tracker
       end
 
-      # Runs consumer data processing against given batch and handles failures and errors.
+      # Builds the consumer instance, builds messages batch and sets all that is needed to run the
+      # user consumption logic
       #
-      # @param messages [Array<Rdkafka::Consumer::Message>] raw rdkafka messages
+      # @param messages [Array<Karafka::Messages::Message>]
       # @param received_at [Time] the moment we've received the batch (actually the moment we've)
       #   enqueued it, but good enough
-      def consume(messages, received_at)
+      def prepare(messages, received_at)
         # Recreate consumer with each batch if persistence is not enabled
         # We reload the consumers with each batch instead of relying on some external signals
         # when needed for consistency. That way devs may have it on or off and not in this
@@ -49,6 +59,11 @@ module Karafka
           received_at
         )
 
+        consumer.on_prepare
+      end
+
+      # Runs consumer data processing against given batch and handles failures and errors.
+      def consume
         # We run the consumer client logic...
         consumer.on_consume
       end
@@ -86,7 +101,7 @@ module Karafka
           consumer = @topic.consumer.new
           consumer.topic = @topic
           consumer.client = @client
-          consumer.pause = @pause
+          consumer.pause_tracker = @pause_tracker
           consumer.producer = ::Karafka::App.producer
           consumer
         end

data/lib/karafka/processing/executors_buffer.rb

@@ -23,21 +23,29 @@ module Karafka
         partition,
         pause
       )
-        topic = @subscription_group.topics.find { |ktopic| ktopic.name == topic }
+        ktopic = @subscription_group.topics.find(topic)
 
-        topic || raise(Errors::TopicNotFoundError, topic)
+        ktopic || raise(Errors::TopicNotFoundError, topic)
 
-        @buffer[topic][partition] ||= Executor.new(
+        @buffer[ktopic][partition] ||= Executor.new(
           @subscription_group.id,
           @client,
-          topic,
+          ktopic,
           pause
         )
       end
 
-      # Runs the shutdown on all active executors.
-      def shutdown
-        @buffer.values.map(&:values).flatten.each(&:shutdown)
+      # Iterates over all available executors and yields them together with topic and partition
+      # info
+      # @yieldparam [Routing::Topic] karafka routing topic object
+      # @yieldparam [Integer] partition number
+      # @yieldparam [Executor] given executor
+      def each
+        @buffer.each do |ktopic, partitions|
+          partitions.each do |partition, executor|
+            yield(ktopic, partition, executor)
+          end
+        end
       end
 
       # Clears the executors buffer. Useful for critical errors recovery.

data/lib/karafka/processing/jobs/base.rb

@@ -5,6 +5,8 @@ module Karafka
     # Namespace for all the jobs that are suppose to run in workers.
     module Jobs
       # Base class for all the jobs types that are suppose to run in workers threads.
+      # Each job can have 3 main entry-points: `#prepare`, `#call` and `#teardown`
+      # Only `#call` is required.
       class Base
         extend Forwardable
 
@@ -12,6 +14,32 @@ module Karafka
         def_delegators :executor, :id, :group_id
 
         attr_reader :executor
+
+        # Creates a new job instance
+        def initialize
+          # All jobs are blocking by default and they can release the lock when blocking operations
+          # are done (if needed)
+          @non_blocking = false
+        end
+
+        # When redefined can run any code that should run before executing the proper code
+        def prepare; end
+
+        # When redefined can run any code that should run after executing the proper code
+        def teardown; end
+
+        # @return [Boolean] is this a non-blocking job
+        #
+        # @note Blocking job is a job, that will cause the job queue to wait until it is finished
+        #   before removing the lock on new jobs being added
+        #
+        # @note All the jobs are blocking by default
+        #
+        # @note Job **needs** to mark itself as non-blocking only **after** it is done with all
+        #   the blocking things (pausing partition, etc).
+        def non_blocking?
+          @non_blocking
+        end
       end
     end
   end

data/lib/karafka/processing/jobs/consume.rb

@@ -6,10 +6,12 @@ module Karafka
       # The main job type. It runs the executor that triggers given topic partition messages
       # processing in an underlying consumer instance.
       class Consume < Base
+        # @return [Array<Rdkafka::Consumer::Message>] array with messages
+        attr_reader :messages
+
         # @param executor [Karafka::Processing::Executor] executor that is suppose to run a given
         #   job
-        # @param messages [Array<dkafka::Consumer::Message>] array with raw rdkafka messages with
-        #   which we are suppose to work
+        # @param messages [Karafka::Messages::Messages] karafka messages batch
         # @return [Consume]
         def initialize(executor, messages)
           @executor = executor
@@ -18,9 +20,14 @@ module Karafka
           super()
         end
 
-        # Runs the given executor.
+        # Runs the preparations on the executor
+        def prepare
+          executor.prepare(@messages, @created_at)
+        end
+
+        # Runs the given executor
         def call
-          executor.consume(@messages, @created_at)
+          executor.consume
         end
       end
     end

data/lib/karafka/processing/jobs_builder.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Class responsible for deciding what type of job should we build to run a given command and
+    # for building a proper job for it.
+    class JobsBuilder
+      # @param executor [Karafka::Processing::Executor]
+      # @param messages [Karafka::Messages::Messages] messages batch to be consumed
+      # @return [Karafka::Processing::Jobs::Consume] consumption job
+      def consume(executor, messages)
+        Jobs::Consume.new(executor, messages)
+      end
+
+      # @param executor [Karafka::Processing::Executor]
+      # @return [Karafka::Processing::Jobs::Revoked] revocation job
+      def revoked(executor)
+        Jobs::Revoked.new(executor)
+      end
+
+      # @param executor [Karafka::Processing::Executor]
+      # @return [Karafka::Processing::Jobs::Shutdown] shutdown job
+      def shutdown(executor)
+        Jobs::Shutdown.new(executor)
+      end
+    end
+  end
+end

data/lib/karafka/processing/jobs_queue.rb

@@ -12,7 +12,7 @@ module Karafka
     class JobsQueue
       # @return [Karafka::Processing::JobsQueue]
       def initialize
-        @queue = ::Queue.new
+        @queue = Queue.new
         # Those queues will act as a semaphores internally. Since we need an indicator for waiting
         # we could use Thread.pass but this is expensive. Instead we can just lock until any
         # of the workers finishes their work and we can re-check. This means that in the worse
@@ -21,7 +21,7 @@ module Karafka
         # We cannot use a single semaphore as it could potentially block in listeners that should
         # process with their data and also could unlock when a given group needs to remain locked
         @semaphores = Hash.new { |h, k| h[k] = Queue.new }
-        @in_processing = Hash.new { |h, k| h[k] = {} }
+        @in_processing = Hash.new { |h, k| h[k] = [] }
         @mutex = Mutex.new
       end
 
@@ -44,9 +44,9 @@ module Karafka
         @mutex.synchronize do
           group = @in_processing[job.group_id]
 
-          raise(Errors::JobsQueueSynchronizationError, job.group_id) if group.key?(job.id)
+          raise(Errors::JobsQueueSynchronizationError, job.group_id) if group.include?(job)
 
-          group[job.id] = true
+          group << job
         end
 
         @queue << job
@@ -60,14 +60,21 @@ module Karafka
         @queue.pop
       end
 
+      # Causes the wait lock to re-check the lock conditions and potential unlock.
+      # @param group_id [String] id of the group we want to unlock for one tick
+      # @note This does not release the wait lock. It just causes a conditions recheck
+      def tick(group_id)
+        @semaphores[group_id] << true
+      end
+
       # Marks a given job from a given group as completed. When there are no more jobs from a given
       # group to be executed, we won't wait.
       #
       # @param [Jobs::Base] job that was completed
       def complete(job)
         @mutex.synchronize do
-          @in_processing[job.group_id].delete(job.id)
-          @semaphores[job.group_id] << true
+          @in_processing[job.group_id].delete(job)
+          tick(job.group_id)
         end
       end
 
@@ -79,7 +86,7 @@ module Karafka
         @mutex.synchronize do
           @in_processing[group_id].clear
           # We unlock it just in case it was blocked when clearing started
-          @semaphores[group_id] << true
+          tick(group_id)
         end
       end
 
@@ -93,8 +100,17 @@ module Karafka
         end
       end
 
-      # Blocks when there are things in the queue in a given group and waits until all the jobs
-      #   from a given group are completed
+      # @param group_id [String]
+      #
+      # @return [Boolean] tell us if we have anything in the processing (or for processing) from
+      # a given group.
+      def empty?(group_id)
+        @in_processing[group_id].empty?
+      end
+
+      # Blocks when there are things in the queue in a given group and waits until all the blocking
+      #   jobs from a given group are completed
+      #
       # @param group_id [String] id of the group in which jobs we're interested.
       # @note This method is blocking.
       def wait(group_id)
@@ -107,14 +123,10 @@ module Karafka
 
       # @param group_id [String] id of the group in which jobs we're interested.
       # @return [Boolean] should we keep waiting or not
+      # @note We do not wait for non-blocking jobs. Their flow should allow for `poll` running
+      #   as they may exceed `max.poll.interval`
       def wait?(group_id)
-        # If it is stopping, all the previous messages that are processed at the moment need to
-        # finish. Otherwise we may risk closing the client and committing offsets afterwards
-        return false if Karafka::App.stopping? && @in_processing[group_id].empty?
-        return false if @queue.closed?
-        return false if @in_processing[group_id].empty?
-
-        true
+        !@in_processing[group_id].all?(&:non_blocking?)
       end
     end
   end

data/lib/karafka/processing/worker.rb

@@ -4,25 +4,39 @@ module Karafka
   module Processing
     # Workers are used to run jobs in separate threads.
     # Workers are the main processing units of the Karafka framework.
+    #
+    # Each job runs in three stages:
+    #   - prepare - here we can run any code that we would need to run blocking before we allow
+    #               the job to run fully async (non blocking). This will always run in a blocking
+    #               way and can be used to make sure all the resources and external dependencies
+    #               are satisfied before going async.
+    #
+    #   - call - actual processing logic that can run sync or async
+    #
+    #   - teardown - it should include any code that we want to run after we executed the user
+    #                code. This can be used to unlock certain resources or do other things that are
+    #                not user code but need to run after user code base is executed.
     class Worker
-      extend Forwardable
+      include Helpers::Async
 
-      def_delegators :@thread, :join, :terminate, :alive?
+      # @return [String] id of this worker
+      attr_reader :id
 
       # @param jobs_queue [JobsQueue]
       # @return [Worker]
       def initialize(jobs_queue)
+        @id = SecureRandom.uuid
         @jobs_queue = jobs_queue
-        @thread = Thread.new do
-          # If anything goes wrong in this worker thread, it means something went really wrong and
-          # we should terminate.
-          Thread.current.abort_on_exception = true
-          loop { break unless process }
-        end
       end
 
       private
 
+      # Runs processing of jobs in a loop
+      # Stops when queue is closed.
+      def call
+        loop { break unless process }
+      end
+
       # Fetches a single job, processes it and marks as completed.
       #
       # @note We do not have error handling here, as no errors should propagate this far. If they
@@ -33,8 +47,23 @@ module Karafka
         job = @jobs_queue.pop
 
         if job
-          job.call
-          true
+          Karafka.monitor.instrument('worker.process', caller: self, job: job)
+
+          Karafka.monitor.instrument('worker.processed', caller: self, job: job) do
+            job.prepare
+
+            # If a job is marked as non blocking, we can run a tick in the job queue and if there
+            # are no other blocking factors, the job queue will be unlocked.
+            # If this does not run, all the things will be blocking and job queue won't allow to
+            # pass it until done.
+            @jobs_queue.tick(job.group_id) if job.non_blocking?
+
+            job.call
+
+            job.teardown
+
+            true
+          end
         else
           false
         end

data/lib/karafka/processing/workers_batch.rb

@@ -17,6 +17,11 @@ module Karafka
       def each(&block)
         @batch.each(&block)
       end
+
+      # @return [Integer] number of workers in the batch
+      def size
+        @batch.size
+      end
     end
   end
 end

data/lib/karafka/routing/consumer_group.rb

@@ -17,7 +17,7 @@ module Karafka
       def initialize(name)
         @name = name
         @id = Karafka::App.config.consumer_mapper.call(name)
-        @topics = []
+        @topics = Topics.new([])
       end
 
       # @return [Boolean] true if this consumer group should be active in our current process

data/lib/karafka/routing/subscription_group.rb

@@ -10,7 +10,7 @@ module Karafka
     class SubscriptionGroup
       attr_reader :id, :topics
 
-      # @param topics [Array<Topic>] all the topics that share the same key settings
+      # @param topics [Karafka::Routing::Topics] all the topics that share the same key settings
       # @return [SubscriptionGroup] built subscription group
       def initialize(topics)
         @id = SecureRandom.uuid
@@ -44,7 +44,7 @@ module Karafka
         kafka[:'auto.offset.reset'] ||= @topics.first.initial_offset
         # Karafka manages the offsets based on the processing state, thus we do not rely on the
         # rdkafka offset auto-storing
-        kafka[:'enable.auto.offset.store'] = 'false'
+        kafka[:'enable.auto.offset.store'] = false
         kafka.freeze
         kafka
       end

data/lib/karafka/routing/subscription_groups_builder.rb

@@ -23,8 +23,8 @@ module Karafka
 
       private_constant :DISTRIBUTION_KEYS
 
-      # @param topics [Array<Topic>] array with topics based on which we want to build subscription
-      #   groups
+      # @param topics [Karafka::Routing::Topics] all the topics based on which we want to build
+      #   subscription groups
       # @return [Array<SubscriptionGroup>] all subscription groups we need in separate threads
       def call(topics)
         topics
@@ -32,6 +32,7 @@ module Karafka
           .group_by(&:first)
           .values
           .map { |value| value.map(&:last) }
+          .map { |topics_array| Routing::Topics.new(topics_array) }
           .map { |grouped_topics| SubscriptionGroup.new(grouped_topics) }
       end
 

data/lib/karafka/routing/topics.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # Abstraction layer on top of groups of topics
+    class Topics
+      include Enumerable
+      extend Forwardable
+
+      def_delegators :@accumulator, :[], :size, :empty?, :last, :<<
+
+      # @param topics_array [Array<Karafka::Routing::Topic>] array with topics
+      def initialize(topics_array)
+        @accumulator = topics_array.dup
+      end
+
+      # Yields each topic
+      #
+      # @param [Proc] block we want to yield with on each topic
+      def each(&block)
+        @accumulator.each(&block)
+      end
+
+      # Finds topic by its name
+      #
+      # @param topic_name [String] topic name
+      # @return [Karafka::Routing::Topic]
+      # @raise [Karafka::Errors::TopicNotFoundError] this should never happen. If you see it,
+      #   please create an issue.
+      def find(topic_name)
+        @accumulator.find { |topic| topic.name == topic_name } ||
+          raise(Karafka::Errors::TopicNotFoundError, topic_name)
+      end
+    end
+  end
+end

data/lib/karafka/runner.rb

@@ -3,32 +3,37 @@
 module Karafka
   # Class used to run the Karafka listeners in separate threads
   class Runner
-    # Starts listening on all the listeners asynchronously
-    # Fetch loop should never end. If they do, it is a critical error
+    # Starts listening on all the listeners asynchronously and handles the jobs queue closing
+    # after listeners are done with their work.
     def call
       # Despite possibility of having several independent listeners, we aim to have one queue for
       # jobs across and one workers poll for that
       jobs_queue = Processing::JobsQueue.new
 
       workers = Processing::WorkersBatch.new(jobs_queue)
-      Karafka::Server.workers = workers
+      listeners = Connection::ListenersBatch.new(jobs_queue)
 
-      threads = listeners(jobs_queue).map do |listener|
-        # We abort on exception because there should be an exception handling developed for
-        # each listener running in separate threads, so the exceptions should never leak
-        # and if that happens, it means that something really bad happened and we should stop
-        # the whole process
-        Thread
-          .new { listener.call }
-          .tap { |thread| thread.abort_on_exception = true }
-      end
+      workers.each(&:async_call)
+      listeners.each(&:async_call)
 
       # We aggregate threads here for a supervised shutdown process
-      Karafka::Server.consumer_threads = threads
+      Karafka::Server.workers = workers
+      Karafka::Server.listeners = listeners
 
       # All the listener threads need to finish
-      threads.each(&:join)
+      listeners.each(&:join)
+
+      # We close the jobs queue only when no listener threads are working.
+      # This ensures, that everything was closed prior to us not accepting anymore jobs and that
+      # no more jobs will be enqueued. Since each listener waits for jobs to finish, once those
+      # are done, we can close.
+      jobs_queue.close
+
       # All the workers need to stop processing anything before we can stop the runner completely
+      # This ensures that even async long-running jobs have time to finish before we are done
+      # with everything. One thing worth keeping in mind though: It is the end user responsibility
+      # to handle the shutdown detection in their long-running processes. Otherwise if timeout
+      # is exceeded, there will be a forced shutdown.
       workers.each(&:join)
     # If anything crashes here, we need to raise the error and crush the runner because it means
     # that something terrible happened
@@ -42,18 +47,5 @@ module Karafka
       Karafka::App.stop!
       raise e
     end
-
-    private
-
-    # @param jobs_queue [Processing::JobsQueue] the main processing queue
-    # @return [Array<Karafka::Connection::Listener>] listeners that will consume messages for each
-    #   of the subscription groups
-    def listeners(jobs_queue)
-      App
-        .subscription_groups
-        .map do |subscription_group|
-          Karafka::Connection::Listener.new(subscription_group, jobs_queue)
-        end
-    end
   end
 end

data/lib/karafka/scheduler.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Karafka
+  # FIFO scheduler for messages coming from various topics and partitions
+  class Scheduler
+    # Schedules jobs in the fifo order
+    #
+    # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
+    # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
+    def schedule_consumption(queue, jobs_array)
+      jobs_array.each do |job|
+        queue << job
+      end
+    end
+
+    # Both revocation and shutdown jobs can also run in fifo by default
+    alias schedule_revocation schedule_consumption
+    alias schedule_shutdown schedule_consumption
+  end
+end

data/lib/karafka/server.rb

@@ -15,7 +15,7 @@ module Karafka
 
     class << self
       # Set of consuming threads. Each consumer thread contains a single consumer
-      attr_accessor :consumer_threads
+      attr_accessor :listeners
 
       # Set of workers
       attr_accessor :workers
@@ -25,9 +25,12 @@ module Karafka
 
       # Method which runs app
       def run
-        process.on_sigint { stop }
-        process.on_sigquit { stop }
-        process.on_sigterm { stop }
+        # Since we do a lot of threading and queuing, we don't want to stop from the trap context
+        # as some things may not work there as expected, that is why we spawn a separate thread to
+        # handle the stopping process
+        process.on_sigint { Thread.new { stop } }
+        process.on_sigquit { Thread.new { stop } }
+        process.on_sigterm { Thread.new { stop } }
 
         # Start is blocking until stop is called and when we stop, it will wait until
         # all of the things are ready to stop
@@ -35,6 +38,8 @@ module Karafka
 
         # We always need to wait for Karafka to stop here since we should wait for the stop running
         # in a separate thread (or trap context) to indicate everything is closed
+        # Since `#start` is blocking, we were get here only after the runner is done. This will
+        # not add any performance degradation because of that.
         Thread.pass until Karafka::App.stopped?
       # Try its best to shutdown underlying components before re-raising
       # rubocop:disable Lint/RescueException
@@ -70,16 +75,16 @@ module Karafka
       def stop
         Karafka::App.stop!
 
-        timeout = Thread.new { Karafka::App.config.shutdown_timeout }.join.value
+        timeout = Karafka::App.config.shutdown_timeout
 
         # We check from time to time (for the timeout period) if all the threads finished
         # their work and if so, we can just return and normal shutdown process will take place
         # We divide it by 1000 because we use time in ms.
         ((timeout / 1_000) * SUPERVISION_CHECK_FACTOR).to_i.times do
-          if consumer_threads.count(&:alive?).zero? &&
+          if listeners.count(&:alive?).zero? &&
              workers.count(&:alive?).zero?
 
-            Thread.new { Karafka::App.producer.close }.join
+            Karafka::App.producer.close
 
             return
           end
@@ -89,22 +94,18 @@ module Karafka
 
         raise Errors::ForcefulShutdownError
       rescue Errors::ForcefulShutdownError => e
-        thread = Thread.new do
-          Karafka.monitor.instrument(
-            'error.occurred',
-            caller: self,
-            error: e,
-            type: 'app.stopping.error'
-          )
-
-          # We're done waiting, lets kill them!
-          workers.each(&:terminate)
-          consumer_threads.each(&:terminate)
-
-          Karafka::App.producer.close
-        end
-
-        thread.join
+        Karafka.monitor.instrument(
+          'error.occurred',
+          caller: self,
+          error: e,
+          type: 'app.stopping.error'
+        )
+
+        # We're done waiting, lets kill them!
+        workers.each(&:terminate)
+        listeners.each(&:terminate)
+
+        Karafka::App.producer.close
 
         # exit! is not within the instrumentation as it would not trigger due to exit
         Kernel.exit! FORCEFUL_EXIT_CODE