karafka 2.0.0.beta1 → 2.0.0.beta2

data/lib/karafka/messages/builders/batch_metadata.rb

@@ -8,42 +8,30 @@ module Karafka
         class << self
           # Creates metadata based on the kafka batch data.
           #
-          # @param kafka_batch [Array<Rdkafka::Consumer::Message>] raw fetched messages
+          # @param messages [Array<Karafka::Messages::Message>] messages array
           # @param topic [Karafka::Routing::Topic] topic for which we've fetched the batch
           # @param scheduled_at [Time] moment when the batch was scheduled for processing
           # @return [Karafka::Messages::BatchMetadata] batch metadata object
-          # @note Regarding the time lags: we can use the current time here, as batch metadata is
-          #   created in the worker. So whenever this is being built, it means that the processing
-          #   of this batch has already started.
-          def call(kafka_batch, topic, scheduled_at)
-            now = Time.now
-
+          #
+          # @note We do not set `processed_at` as this needs to be assigned when the batch is
+          #   picked up for processing.
+          def call(messages, topic, scheduled_at)
             Karafka::Messages::BatchMetadata.new(
-              size: kafka_batch.count,
-              first_offset: kafka_batch.first.offset,
-              last_offset: kafka_batch.last.offset,
+              size: messages.count,
+              first_offset: messages.first.offset,
+              last_offset: messages.last.offset,
               deserializer: topic.deserializer,
-              partition: kafka_batch[0].partition,
+              partition: messages.first.partition,
               topic: topic.name,
+              # We go with the assumption that the creation of the whole batch is the last message
+              # creation time
+              created_at: messages.last.timestamp,
+              # When this batch was built and scheduled for execution
               scheduled_at: scheduled_at,
-              # This lag describes how long did it take for a message to be consumed from the
-              # moment it was created
-              consumption_lag: time_distance_in_ms(now, kafka_batch.last.timestamp),
-              # This lag describes how long did a batch have to wait before it was picked up by
-              # one of the workers
-              processing_lag: time_distance_in_ms(now, scheduled_at)
-            ).freeze
-          end
-
-          private
-
-          # Computes time distance in between two times in ms
-          #
-          # @param time1 [Time]
-          # @param time2 [Time]
-          # @return [Integer] distance in between two times in ms
-          def time_distance_in_ms(time1, time2)
-            ((time1 - time2) * 1_000).round
+              # We build the batch metadata when we pick up the job in the worker, thus we can use
+              # current time here
+              processed_at: Time.now
+            )
           end
         end
       end

data/lib/karafka/messages/builders/message.rb

@@ -26,6 +26,7 @@ module Karafka
               received_at: received_at
             ).freeze
 
+            # Karafka messages cannot be frozen because of the lazy deserialization feature
             Karafka::Messages::Message.new(
               kafka_message.payload,
               metadata

data/lib/karafka/messages/builders/messages.rb

@@ -9,27 +9,19 @@ module Karafka
           # Creates messages batch with messages inside based on the incoming messages and the
           # topic from which it comes.
           #
-          # @param kafka_messages [Array<Rdkafka::Consumer::Message>] raw fetched messages
+          # @param messages [Array<Karafka::Messages::Message>] karafka messages array
           # @param topic [Karafka::Routing::Topic] topic for which we're received messages
           # @param received_at [Time] moment in time when the messages were received
           # @return [Karafka::Messages::Messages] messages batch object
-          def call(kafka_messages, topic, received_at)
-            messages_array = kafka_messages.map do |message|
-              Karafka::Messages::Builders::Message.call(
-                message,
-                topic,
-                received_at
-              )
-            end
-
+          def call(messages, topic, received_at)
             metadata = BatchMetadata.call(
-              kafka_messages,
+              messages,
               topic,
               received_at
             ).freeze
 
             Karafka::Messages::Messages.new(
-              messages_array,
+              messages,
               metadata
             ).freeze
           end

data/lib/karafka/pro/active_job/consumer.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Pro
+    module ActiveJob
+      # This Karafka component is a Pro component.
+      # All of the commercial components are present in the lib/karafka/pro directory of this
+      # repository and their usage requires commercial license agreement.
+      #
+      # Karafka has also commercial-friendly license, commercial support and commercial components.
+      #
+      # By sending a pull request to the pro components, you are agreeing to transfer the copyright
+      # of your code to Maciej Mensfeld.
+
+      # Pro ActiveJob consumer that is suppose to handle long-running jobs as well as short
+      # running jobs
+      class Consumer < Karafka::ActiveJob::Consumer
+      end
+    end
+  end
+end

data/lib/karafka/pro/active_job/dispatcher.rb

@@ -18,7 +18,7 @@ module Karafka
       # much better and more granular control over the dispatch and consumption process.
       class Dispatcher < ::Karafka::ActiveJob::Dispatcher
         # Defaults for dispatching
-        # The can be updated by using `#karafka_options` on the job
+        # They can be updated by using `#karafka_options` on the job
         DEFAULTS = {
           dispatch_method: :produce_async,
           # We don't create a dummy proc based partitioner as we would have to evaluate it with

data/lib/karafka/pro/loader.rb

@@ -19,13 +19,17 @@ module Karafka
         #   components
         def setup(config)
           require_relative 'performance_tracker'
+          require_relative 'scheduler'
+          require_relative 'processing/jobs/consume_non_blocking'
+          require_relative 'active_job/consumer'
           require_relative 'active_job/dispatcher'
           require_relative 'active_job/job_options_contract'
 
+          config.internal.scheduler = Scheduler.new
+          config.internal.active_job.consumer = ActiveJob::Consumer
           config.internal.active_job.dispatcher = ActiveJob::Dispatcher.new
           config.internal.active_job.job_options_contract = ActiveJob::JobOptionsContract.new
 
-          # Monitor time needed to process each message from a single partition
           config.monitor.subscribe(PerformanceTracker.instance)
         end
       end

data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Pro
+    # Pro components related to processing part of Karafka
+    module Processing
+      # Pro jobs
+      module Jobs
+        # This Karafka component is a Pro component.
+        # All of the commercial components are present in the lib/karafka/pro directory of this
+        # repository and their usage requires commercial license agreement.
+        #
+        # Karafka has also commercial-friendly license, commercial support and commercial
+        # components.
+        #
+        # By sending a pull request to the pro components, you are agreeing to transfer the
+        # copyright of your code to Maciej Mensfeld.
+
+        # The main job type in a non-blocking variant.
+        # This variant works "like" the regular consumption but pauses the partition for as long
+        # as it is needed until a job is done.
+        #
+        # It can be useful when having long lasting jobs that would exceed `max.poll.interval`
+        # if would block.
+        #
+        # @note It needs to be working with a proper consumer that will handle the partition
+        #   management. This layer of the framework knows nothing about Kafka messages consumption.
+        class ConsumeNonBlocking < ::Karafka::Processing::Jobs::Consume
+          # Releases the blocking lock after it is done with the preparation phase for this job
+          def prepare
+            super
+            @non_blocking = true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduler.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Pro
+    # This Karafka component is a Pro component.
+    # All of the commercial components are present in the lib/karafka/pro directory of this
+    # repository and their usage requires commercial license agreement.
+    #
+    # Karafka has also commercial-friendly license, commercial support and commercial components.
+    #
+    # By sending a pull request to the pro components, you are agreeing to transfer the copyright
+    # of your code to Maciej Mensfeld.
+
+    # Optimizes scheduler that takes into consideration of execution time needed to process
+    # messages from given topics partitions. It uses the non-preemptive LJF algorithm
+    #
+    # This scheduler is designed to optimize execution times on jobs that perform IO operations as
+    # when taking IO into consideration, the can achieve optimized parallel processing.
+    #
+    # This scheduler can also work with virtual partitions.
+    #
+    # Aside from consumption jobs, other jobs do not run often, thus we can leave them with
+    # default FIFO scheduler from the default Karafka scheduler
+    class Scheduler < ::Karafka::Scheduler
+      # Schedules jobs in the LJF order for consumption
+      #
+      # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
+      # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
+      #
+      def schedule_consumption(queue, jobs_array)
+        pt = PerformanceTracker.instance
+
+        ordered = []
+
+        jobs_array.each do |job|
+          messages = job.messages
+          message = messages.first
+
+          cost = pt.processing_time_p95(message.topic, message.partition) * messages.size
+
+          ordered << [job, cost]
+        end
+
+        ordered.sort_by!(&:last)
+        ordered.reverse!
+        ordered.map!(&:first)
+
+        ordered.each do |job|
+          queue << job
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/executor.rb

@@ -18,6 +18,8 @@ module Karafka
       # @return [String] subscription group id to which a given executor belongs
       attr_reader :group_id
 
+      attr_reader :messages
+
       # @param group_id [String] id of the subscription group to which the executor belongs
       # @param client [Karafka::Connection::Client] kafka client
       # @param topic [Karafka::Routing::Topic] topic for which this executor will run
@@ -30,9 +32,10 @@ module Karafka
         @pause_tracker = pause_tracker
       end
 
-      # Builds the consumer instance and sets all that is needed to run the user consumption logic
+      # Builds the consumer instance, builds messages batch and sets all that is needed to run the
+      # user consumption logic
       #
-      # @param messages [Array<Rdkafka::Consumer::Message>] raw rdkafka messages
+      # @param messages [Array<Karafka::Messages::Message>]
       # @param received_at [Time] the moment we've received the batch (actually the moment we've)
       #   enqueued it, but good enough
       def prepare(messages, received_at)

data/lib/karafka/processing/executors_buffer.rb

@@ -23,21 +23,29 @@ module Karafka
         partition,
         pause
       )
-        topic = @subscription_group.topics.find { |ktopic| ktopic.name == topic }
+        ktopic = @subscription_group.topics.find(topic)
 
-        topic || raise(Errors::TopicNotFoundError, topic)
+        ktopic || raise(Errors::TopicNotFoundError, topic)
 
-        @buffer[topic][partition] ||= Executor.new(
+        @buffer[ktopic][partition] ||= Executor.new(
           @subscription_group.id,
           @client,
-          topic,
+          ktopic,
           pause
         )
       end
 
-      # Runs the shutdown on all active executors.
-      def shutdown
-        @buffer.values.map(&:values).flatten.each(&:shutdown)
+      # Iterates over all available executors and yields them together with topic and partition
+      # info
+      # @yieldparam [Routing::Topic] karafka routing topic object
+      # @yieldparam [Integer] partition number
+      # @yieldparam [Executor] given executor
+      def each
+        @buffer.each do |ktopic, partitions|
+          partitions.each do |partition, executor|
+            yield(ktopic, partition, executor)
+          end
+        end
       end
 
       # Clears the executors buffer. Useful for critical errors recovery.

data/lib/karafka/processing/jobs/base.rb

@@ -15,6 +15,13 @@ module Karafka
 
         attr_reader :executor
 
+        # Creates a new job instance
+        def initialize
+          # All jobs are blocking by default and they can release the lock when blocking operations
+          # are done (if needed)
+          @non_blocking = false
+        end
+
         # When redefined can run any code that should run before executing the proper code
         def prepare; end
 
@@ -22,11 +29,16 @@ module Karafka
         def teardown; end
 
         # @return [Boolean] is this a non-blocking job
+        #
         # @note Blocking job is a job, that will cause the job queue to wait until it is finished
         #   before removing the lock on new jobs being added
+        #
         # @note All the jobs are blocking by default
+        #
+        # @note Job **needs** to mark itself as non-blocking only **after** it is done with all
+        #   the blocking things (pausing partition, etc).
         def non_blocking?
-          false
+          @non_blocking
         end
       end
     end

data/lib/karafka/processing/jobs/consume.rb

@@ -6,10 +6,12 @@ module Karafka
       # The main job type. It runs the executor that triggers given topic partition messages
       # processing in an underlying consumer instance.
       class Consume < Base
+        # @return [Array<Rdkafka::Consumer::Message>] array with messages
+        attr_reader :messages
+
         # @param executor [Karafka::Processing::Executor] executor that is suppose to run a given
         #   job
-        # @param messages [Array<dkafka::Consumer::Message>] array with raw rdkafka messages with
-        #   which we are suppose to work
+        # @param messages [Karafka::Messages::Messages] karafka messages batch
         # @return [Consume]
         def initialize(executor, messages)
           @executor = executor

data/lib/karafka/processing/jobs_queue.rb

@@ -12,7 +12,7 @@ module Karafka
     class JobsQueue
       # @return [Karafka::Processing::JobsQueue]
       def initialize
-        @queue = ::Queue.new
+        @queue = Queue.new
         # Those queues will act as a semaphores internally. Since we need an indicator for waiting
         # we could use Thread.pass but this is expensive. Instead we can just lock until any
         # of the workers finishes their work and we can re-check. This means that in the worse
@@ -100,8 +100,17 @@ module Karafka
         end
       end
 
-      # Blocks when there are things in the queue in a given group and waits until all the jobs
-      #   from a given group are completed
+      # @param group_id [String]
+      #
+      # @return [Boolean] tell us if we have anything in the processing (or for processing) from
+      # a given group.
+      def empty?(group_id)
+        @in_processing[group_id].empty?
+      end
+
+      # Blocks when there are things in the queue in a given group and waits until all the blocking
+      #   jobs from a given group are completed
+      #
       # @param group_id [String] id of the group in which jobs we're interested.
       # @note This method is blocking.
       def wait(group_id)
@@ -114,16 +123,10 @@ module Karafka
 
       # @param group_id [String] id of the group in which jobs we're interested.
       # @return [Boolean] should we keep waiting or not
+      # @note We do not wait for non-blocking jobs. Their flow should allow for `poll` running
+      #   as they may exceed `max.poll.interval`
       def wait?(group_id)
-        group = @in_processing[group_id]
-
-        # If it is stopping, all the previous messages that are processed at the moment need to
-        # finish. Otherwise we may risk closing the client and committing offsets afterwards
-        return false if Karafka::App.stopping? && group.empty?
-        return false if @queue.closed?
-        return false if group.empty?
-
-        !group.all?(&:non_blocking?)
+        !@in_processing[group_id].all?(&:non_blocking?)
       end
     end
   end

data/lib/karafka/processing/worker.rb

@@ -17,24 +17,22 @@ module Karafka
     #                code. This can be used to unlock certain resources or do other things that are
     #                not user code but need to run after user code base is executed.
     class Worker
-      extend Forwardable
-
-      def_delegators :@thread, :join, :terminate, :alive?
+      include Helpers::Async
 
       # @param jobs_queue [JobsQueue]
       # @return [Worker]
       def initialize(jobs_queue)
         @jobs_queue = jobs_queue
-        @thread = Thread.new do
-          # If anything goes wrong in this worker thread, it means something went really wrong and
-          # we should terminate.
-          Thread.current.abort_on_exception = true
-          loop { break unless process }
-        end
       end
 
       private
 
+      # Runs processing of jobs in a loop
+      # Stops when queue is closed.
+      def call
+        loop { break unless process }
+      end
+
       # Fetches a single job, processes it and marks as completed.
       #
       # @note We do not have error handling here, as no errors should propagate this far. If they

data/lib/karafka/processing/workers_batch.rb

@@ -17,6 +17,11 @@ module Karafka
       def each(&block)
         @batch.each(&block)
       end
+
+      # @return [Integer] number of workers in the batch
+      def size
+        @batch.size
+      end
     end
   end
 end

data/lib/karafka/routing/consumer_group.rb

@@ -17,7 +17,7 @@ module Karafka
       def initialize(name)
         @name = name
         @id = Karafka::App.config.consumer_mapper.call(name)
-        @topics = []
+        @topics = Topics.new([])
       end
 
       # @return [Boolean] true if this consumer group should be active in our current process

data/lib/karafka/routing/subscription_group.rb

@@ -10,7 +10,7 @@ module Karafka
     class SubscriptionGroup
       attr_reader :id, :topics
 
-      # @param topics [Array<Topic>] all the topics that share the same key settings
+      # @param topics [Karafka::Routing::Topics] all the topics that share the same key settings
       # @return [SubscriptionGroup] built subscription group
       def initialize(topics)
         @id = SecureRandom.uuid

data/lib/karafka/routing/subscription_groups_builder.rb

@@ -23,8 +23,8 @@ module Karafka
 
       private_constant :DISTRIBUTION_KEYS
 
-      # @param topics [Array<Topic>] array with topics based on which we want to build subscription
-      #   groups
+      # @param topics [Karafka::Routing::Topics] all the topics based on which we want to build
+      #   subscription groups
       # @return [Array<SubscriptionGroup>] all subscription groups we need in separate threads
       def call(topics)
         topics
@@ -32,6 +32,7 @@ module Karafka
           .group_by(&:first)
           .values
           .map { |value| value.map(&:last) }
+          .map { |topics_array| Routing::Topics.new(topics_array) }
           .map { |grouped_topics| SubscriptionGroup.new(grouped_topics) }
       end
 

data/lib/karafka/routing/topics.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # Abstraction layer on top of groups of topics
+    class Topics
+      include Enumerable
+      extend Forwardable
+
+      def_delegators :@accumulator, :[], :size, :empty?, :last, :<<
+
+      # @param topics_array [Array<Karafka::Routing::Topic>] array with topics
+      def initialize(topics_array)
+        @accumulator = topics_array.dup
+      end
+
+      # Yields each topic
+      #
+      # @param [Proc] block we want to yield with on each topic
+      def each(&block)
+        @accumulator.each(&block)
+      end
+
+      # Finds topic by its name
+      #
+      # @param topic_name [String] topic name
+      # @return [Karafka::Routing::Topic]
+      # @raise [Karafka::Errors::TopicNotFoundError] this should never happen. If you see it,
+      #   please create an issue.
+      def find(topic_name)
+        @accumulator.find { |topic| topic.name == topic_name } ||
+          raise(Karafka::Errors::TopicNotFoundError, topic_name)
+      end
+    end
+  end
+end

data/lib/karafka/runner.rb

@@ -3,32 +3,37 @@
 module Karafka
   # Class used to run the Karafka listeners in separate threads
   class Runner
-    # Starts listening on all the listeners asynchronously
-    # Fetch loop should never end. If they do, it is a critical error
+    # Starts listening on all the listeners asynchronously and handles the jobs queue closing
+    # after listeners are done with their work.
     def call
       # Despite possibility of having several independent listeners, we aim to have one queue for
       # jobs across and one workers poll for that
       jobs_queue = Processing::JobsQueue.new
 
       workers = Processing::WorkersBatch.new(jobs_queue)
-      Karafka::Server.workers = workers
+      listeners = Connection::ListenersBatch.new(jobs_queue)
 
-      threads = listeners(jobs_queue).map do |listener|
-        # We abort on exception because there should be an exception handling developed for
-        # each listener running in separate threads, so the exceptions should never leak
-        # and if that happens, it means that something really bad happened and we should stop
-        # the whole process
-        Thread
-          .new { listener.call }
-          .tap { |thread| thread.abort_on_exception = true }
-      end
+      workers.each(&:async_call)
+      listeners.each(&:async_call)
 
       # We aggregate threads here for a supervised shutdown process
-      Karafka::Server.consumer_threads = threads
+      Karafka::Server.workers = workers
+      Karafka::Server.listeners = listeners
 
       # All the listener threads need to finish
-      threads.each(&:join)
+      listeners.each(&:join)
+
+      # We close the jobs queue only when no listener threads are working.
+      # This ensures, that everything was closed prior to us not accepting anymore jobs and that
+      # no more jobs will be enqueued. Since each listener waits for jobs to finish, once those
+      # are done, we can close.
+      jobs_queue.close
+
       # All the workers need to stop processing anything before we can stop the runner completely
+      # This ensures that even async long-running jobs have time to finish before we are done
+      # with everything. One thing worth keeping in mind though: It is the end user responsibility
+      # to handle the shutdown detection in their long-running processes. Otherwise if timeout
+      # is exceeded, there will be a forced shutdown.
       workers.each(&:join)
     # If anything crashes here, we need to raise the error and crush the runner because it means
     # that something terrible happened
@@ -42,18 +47,5 @@ module Karafka
       Karafka::App.stop!
       raise e
     end
-
-    private
-
-    # @param jobs_queue [Processing::JobsQueue] the main processing queue
-    # @return [Array<Karafka::Connection::Listener>] listeners that will consume messages for each
-    #   of the subscription groups
-    def listeners(jobs_queue)
-      App
-        .subscription_groups
-        .map do |subscription_group|
-          Karafka::Connection::Listener.new(subscription_group, jobs_queue)
-        end
-    end
   end
 end

data/lib/karafka/scheduler.rb

@@ -3,19 +3,18 @@
 module Karafka
   # FIFO scheduler for messages coming from various topics and partitions
   class Scheduler
-    # Yields messages from partitions in the fifo order
+    # Schedules jobs in the fifo order
     #
-    # @param messages_buffer [Karafka::Connection::MessagesBuffer] messages buffer with data from
-    #   multiple topics and partitions
-    # @yieldparam [String] topic name
-    # @yieldparam [Integer] partition number
-    # @yieldparam [Array<Rdkafka::Consumer::Message>] topic partition aggregated results
-    def call(messages_buffer)
-      messages_buffer.each do |topic, partitions|
-        partitions.each do |partition, messages|
-          yield(topic, partition, messages)
-        end
+    # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
+    # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
+    def schedule_consumption(queue, jobs_array)
+      jobs_array.each do |job|
+        queue << job
       end
     end
+
+    # Both revocation and shutdown jobs can also run in fifo by default
+    alias schedule_revocation schedule_consumption
+    alias schedule_shutdown schedule_consumption
   end
 end