karafka 2.0.0.beta1 → 2.0.0.beta4

data/lib/karafka/pro/processing/jobs_builder.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Pro jobs builder that supports lrj
+      class JobsBuilder < ::Karafka::Processing::JobsBuilder
+        # @param executor [Karafka::Processing::Executor]
+        # @param messages [Karafka::Messages::Messages] messages batch to be consumed
+        # @return [Karafka::Processing::Jobs::Consume] blocking job
+        # @return [Karafka::Pro::Processing::Jobs::ConsumeNonBlocking] non blocking for lrj
+        def consume(executor, messages)
+          if executor.topic.long_running_job?
+            Jobs::ConsumeNonBlocking.new(executor, messages)
+          else
+            super
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/extensions.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Pro routing components
+    module Routing
+      # Routing extensions that allow to configure some extra PRO routing options
+      module Extensions
+        class << self
+          # @param base [Class] class we extend
+          def included(base)
+            base.attr_accessor :long_running_job
+          end
+        end
+
+        # @return [Boolean] is a given job on a topic a long running one
+        def long_running_job?
+          @long_running_job || false
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduler.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Optimizes scheduler that takes into consideration of execution time needed to process
+    # messages from given topics partitions. It uses the non-preemptive LJF algorithm
+    #
+    # This scheduler is designed to optimize execution times on jobs that perform IO operations as
+    # when taking IO into consideration, the can achieve optimized parallel processing.
+    #
+    # This scheduler can also work with virtual partitions.
+    #
+    # Aside from consumption jobs, other jobs do not run often, thus we can leave them with
+    # default FIFO scheduler from the default Karafka scheduler
+    class Scheduler < ::Karafka::Scheduler
+      # Schedules jobs in the LJF order for consumption
+      #
+      # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
+      # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
+      #
+      def schedule_consumption(queue, jobs_array)
+        pt = PerformanceTracker.instance
+
+        ordered = []
+
+        jobs_array.each do |job|
+          messages = job.messages
+          message = messages.first
+
+          cost = pt.processing_time_p95(message.topic, message.partition) * messages.size
+
+          ordered << [job, cost]
+        end
+
+        ordered.sort_by!(&:last)
+        ordered.reverse!
+        ordered.map!(&:first)
+
+        ordered.each do |job|
+          queue << job
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/executor.rb

@@ -18,6 +18,15 @@ module Karafka
       # @return [String] subscription group id to which a given executor belongs
       attr_reader :group_id
 
+      # @return [Karafka::Messages::Messages] messages batch
+      attr_reader :messages
+
+      # Topic accessibility may be needed for the jobs builder to be able to build a proper job
+      # based on the topic settings defined by the end user
+      #
+      # @return [Karafka::Routing::Topic] topic of this executor
+      attr_reader :topic
+
       # @param group_id [String] id of the subscription group to which the executor belongs
       # @param client [Karafka::Connection::Client] kafka client
       # @param topic [Karafka::Routing::Topic] topic for which this executor will run
@@ -30,17 +39,26 @@ module Karafka
         @pause_tracker = pause_tracker
       end
 
-      # Builds the consumer instance and sets all that is needed to run the user consumption logic
+      # Builds the consumer instance, builds messages batch and sets all that is needed to run the
+      # user consumption logic
       #
-      # @param messages [Array<Rdkafka::Consumer::Message>] raw rdkafka messages
+      # @param messages [Array<Karafka::Messages::Message>]
       # @param received_at [Time] the moment we've received the batch (actually the moment we've)
       #   enqueued it, but good enough
-      def prepare(messages, received_at)
+      def before_consume(messages, received_at)
         # Recreate consumer with each batch if persistence is not enabled
         # We reload the consumers with each batch instead of relying on some external signals
         # when needed for consistency. That way devs may have it on or off and not in this
         # middle state, where re-creation of a consumer instance would occur only sometimes
-        @consumer = nil unless ::Karafka::App.config.consumer_persistence
+        @recreate = true unless ::Karafka::App.config.consumer_persistence
+
+        # If @recreate was set to true (aside from non persistent), it means, that revocation or
+        # a shutdown happened and we need to have a new instance for running another consume for
+        # this topic partition
+        if @recreate
+          @consumer = nil
+          @recreate = false
+        end
 
         # First we build messages batch...
         consumer.messages = Messages::Builders::Messages.call(
@@ -49,7 +67,7 @@ module Karafka
           received_at
         )
 
-        consumer.on_prepared
+        consumer.on_before_consume
       end
 
       # Runs consumer data processing against given batch and handles failures and errors.
@@ -58,6 +76,11 @@ module Karafka
         consumer.on_consume
       end
 
+      # Runs consumer after consumption code
+      def after_consume
+        consumer.on_after_consume if @consumer
+      end
+
       # Runs the controller `#revoked` method that should be triggered when a given consumer is
       # no longer needed due to partitions reassignment.
       #
@@ -66,9 +89,13 @@ module Karafka
       #
       # @note We run it only when consumer was present, because presence indicates, that at least
       #   a single message has been consumed.
+      #
+      # @note We do not reset the consumer but we indicate need for recreation instead, because
+      #   after the revocation, there still may be `#after_consume` running that needs a given
+      #   consumer instance.
       def revoked
         consumer.on_revoked if @consumer
-        @consumer = nil
+        @recreate = true
       end
 
       # Runs the controller `#shutdown` method that should be triggered when a given consumer is
@@ -80,7 +107,7 @@ module Karafka
         # There is a case, where the consumer no longer exists because it was revoked, in case like
         # that we do not build a new instance and shutdown should not be triggered.
         consumer.on_shutdown if @consumer
-        @consumer = nil
+        @recreate = true
       end
 
       private

data/lib/karafka/processing/executors_buffer.rb

@@ -23,21 +23,29 @@ module Karafka
         partition,
         pause
       )
-        topic = @subscription_group.topics.find { |ktopic| ktopic.name == topic }
+        ktopic = @subscription_group.topics.find(topic)
 
-        topic || raise(Errors::TopicNotFoundError, topic)
+        ktopic || raise(Errors::TopicNotFoundError, topic)
 
-        @buffer[topic][partition] ||= Executor.new(
+        @buffer[ktopic][partition] ||= Executor.new(
           @subscription_group.id,
           @client,
-          topic,
+          ktopic,
           pause
         )
       end
 
-      # Runs the shutdown on all active executors.
-      def shutdown
-        @buffer.values.map(&:values).flatten.each(&:shutdown)
+      # Iterates over all available executors and yields them together with topic and partition
+      # info
+      # @yieldparam [Routing::Topic] karafka routing topic object
+      # @yieldparam [Integer] partition number
+      # @yieldparam [Executor] given executor
+      def each
+        @buffer.each do |ktopic, partitions|
+          partitions.each do |partition, executor|
+            yield(ktopic, partition, executor)
+          end
+        end
       end
 
       # Clears the executors buffer. Useful for critical errors recovery.

data/lib/karafka/processing/jobs/base.rb

@@ -5,7 +5,7 @@ module Karafka
     # Namespace for all the jobs that are suppose to run in workers.
     module Jobs
       # Base class for all the jobs types that are suppose to run in workers threads.
-      # Each job can have 3 main entry-points: `#prepare`, `#call` and `#teardown`
+      # Each job can have 3 main entry-points: `#before_call`, `#call` and `#after_call`
       # Only `#call` is required.
       class Base
         extend Forwardable
@@ -15,18 +15,35 @@ module Karafka
 
         attr_reader :executor
 
+        # Creates a new job instance
+        def initialize
+          # All jobs are blocking by default and they can release the lock when blocking operations
+          # are done (if needed)
+          @non_blocking = false
+        end
+
         # When redefined can run any code that should run before executing the proper code
-        def prepare; end
+        def before_call; end
+
+        # The main entry-point of a job
+        def call
+          raise NotImplementedError, 'Please implement in a subclass'
+        end
 
         # When redefined can run any code that should run after executing the proper code
-        def teardown; end
+        def after_call; end
 
         # @return [Boolean] is this a non-blocking job
+        #
         # @note Blocking job is a job, that will cause the job queue to wait until it is finished
         #   before removing the lock on new jobs being added
+        #
         # @note All the jobs are blocking by default
+        #
+        # @note Job **needs** to mark itself as non-blocking only **after** it is done with all
+        #   the blocking things (pausing partition, etc).
         def non_blocking?
-          false
+          @non_blocking
         end
       end
     end

data/lib/karafka/processing/jobs/consume.rb

@@ -6,10 +6,12 @@ module Karafka
       # The main job type. It runs the executor that triggers given topic partition messages
       # processing in an underlying consumer instance.
       class Consume < Base
+        # @return [Array<Rdkafka::Consumer::Message>] array with messages
+        attr_reader :messages
+
         # @param executor [Karafka::Processing::Executor] executor that is suppose to run a given
         #   job
-        # @param messages [Array<dkafka::Consumer::Message>] array with raw rdkafka messages with
-        #   which we are suppose to work
+        # @param messages [Karafka::Messages::Messages] karafka messages batch
         # @return [Consume]
         def initialize(executor, messages)
           @executor = executor
@@ -18,15 +20,20 @@ module Karafka
           super()
         end
 
-        # Runs the preparations on the executor
-        def prepare
-          executor.prepare(@messages, @created_at)
+        # Runs the before consumption preparations on the executor
+        def before_call
+          executor.before_consume(@messages, @created_at)
         end
 
         # Runs the given executor
         def call
           executor.consume
         end
+
+        # Runs any error handling and other post-consumption stuff on the executor
+        def after_call
+          executor.after_consume
+        end
       end
     end
   end

data/lib/karafka/processing/jobs_builder.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Class responsible for deciding what type of job should we build to run a given command and
+    # for building a proper job for it.
+    class JobsBuilder
+      # @param executor [Karafka::Processing::Executor]
+      # @param messages [Karafka::Messages::Messages] messages batch to be consumed
+      # @return [Karafka::Processing::Jobs::Consume] consumption job
+      def consume(executor, messages)
+        Jobs::Consume.new(executor, messages)
+      end
+
+      # @param executor [Karafka::Processing::Executor]
+      # @return [Karafka::Processing::Jobs::Revoked] revocation job
+      def revoked(executor)
+        Jobs::Revoked.new(executor)
+      end
+
+      # @param executor [Karafka::Processing::Executor]
+      # @return [Karafka::Processing::Jobs::Shutdown] shutdown job
+      def shutdown(executor)
+        Jobs::Shutdown.new(executor)
+      end
+    end
+  end
+end

data/lib/karafka/processing/jobs_queue.rb

@@ -12,7 +12,7 @@ module Karafka
     class JobsQueue
       # @return [Karafka::Processing::JobsQueue]
       def initialize
-        @queue = ::Queue.new
+        @queue = Queue.new
         # Those queues will act as a semaphores internally. Since we need an indicator for waiting
         # we could use Thread.pass but this is expensive. Instead we can just lock until any
         # of the workers finishes their work and we can re-check. This means that in the worse
@@ -100,8 +100,17 @@ module Karafka
         end
       end
 
-      # Blocks when there are things in the queue in a given group and waits until all the jobs
-      #   from a given group are completed
+      # @param group_id [String]
+      #
+      # @return [Boolean] tell us if we have anything in the processing (or for processing) from
+      # a given group.
+      def empty?(group_id)
+        @in_processing[group_id].empty?
+      end
+
+      # Blocks when there are things in the queue in a given group and waits until all the blocking
+      #   jobs from a given group are completed
+      #
       # @param group_id [String] id of the group in which jobs we're interested.
       # @note This method is blocking.
       def wait(group_id)
@@ -114,16 +123,10 @@ module Karafka
 
       # @param group_id [String] id of the group in which jobs we're interested.
       # @return [Boolean] should we keep waiting or not
+      # @note We do not wait for non-blocking jobs. Their flow should allow for `poll` running
+      #   as they may exceed `max.poll.interval`
       def wait?(group_id)
-        group = @in_processing[group_id]
-
-        # If it is stopping, all the previous messages that are processed at the moment need to
-        # finish. Otherwise we may risk closing the client and committing offsets afterwards
-        return false if Karafka::App.stopping? && group.empty?
-        return false if @queue.closed?
-        return false if group.empty?
-
-        !group.all?(&:non_blocking?)
+        !@in_processing[group_id].all?(&:non_blocking?)
       end
     end
   end

data/lib/karafka/processing/result.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # A simple object that allows us to keep track of processing state.
+    # It allows to indicate if given thing moved from success to a failure or the other way around
+    # Useful for tracking consumption state
+    class Result
+      def initialize
+        @success = true
+      end
+
+      # @return [Boolean]
+      def failure?
+        !success?
+      end
+
+      # @return [Boolean]
+      def success?
+        @success
+      end
+
+      # Marks state as successful
+      def success!
+        @success = true
+      end
+
+      # Marks state as failure
+      def failure!
+        @success = false
+      end
+    end
+  end
+end

data/lib/karafka/processing/worker.rb

@@ -17,24 +17,26 @@ module Karafka
     #                code. This can be used to unlock certain resources or do other things that are
     #                not user code but need to run after user code base is executed.
     class Worker
-      extend Forwardable
+      include Helpers::Async
 
-      def_delegators :@thread, :join, :terminate, :alive?
+      # @return [String] id of this worker
+      attr_reader :id
 
       # @param jobs_queue [JobsQueue]
       # @return [Worker]
       def initialize(jobs_queue)
+        @id = SecureRandom.uuid
         @jobs_queue = jobs_queue
-        @thread = Thread.new do
-          # If anything goes wrong in this worker thread, it means something went really wrong and
-          # we should terminate.
-          Thread.current.abort_on_exception = true
-          loop { break unless process }
-        end
       end
 
       private
 
+      # Runs processing of jobs in a loop
+      # Stops when queue is closed.
+      def call
+        loop { break unless process }
+      end
+
       # Fetches a single job, processes it and marks as completed.
       #
       # @note We do not have error handling here, as no errors should propagate this far. If they
@@ -45,19 +47,23 @@ module Karafka
         job = @jobs_queue.pop
 
         if job
-          job.prepare
+          Karafka.monitor.instrument('worker.process', caller: self, job: job)
+
+          Karafka.monitor.instrument('worker.processed', caller: self, job: job) do
+            job.before_call
 
-          # If a job is marked as non blocking, we can run a tick in the job queue and if there
-          # are no other blocking factors, the job queue will be unlocked.
-          # If this does not run, all the things will be blocking and job queue won't allow to
-          # pass it until done.
-          @jobs_queue.tick(job.group_id) if job.non_blocking?
+            # If a job is marked as non blocking, we can run a tick in the job queue and if there
+            # are no other blocking factors, the job queue will be unlocked.
+            # If this does not run, all the things will be blocking and job queue won't allow to
+            # pass it until done.
+            @jobs_queue.tick(job.group_id) if job.non_blocking?
 
-          job.call
+            job.call
 
-          job.teardown
+            job.after_call
 
-          true
+            true
+          end
         else
           false
         end

data/lib/karafka/processing/workers_batch.rb

@@ -17,6 +17,11 @@ module Karafka
       def each(&block)
         @batch.each(&block)
       end
+
+      # @return [Integer] number of workers in the batch
+      def size
+        @batch.size
+      end
     end
   end
 end

data/lib/karafka/routing/consumer_group.rb

@@ -17,7 +17,7 @@ module Karafka
       def initialize(name)
         @name = name
         @id = Karafka::App.config.consumer_mapper.call(name)
-        @topics = []
+        @topics = Topics.new([])
       end
 
       # @return [Boolean] true if this consumer group should be active in our current process

data/lib/karafka/routing/subscription_group.rb

@@ -10,7 +10,7 @@ module Karafka
     class SubscriptionGroup
       attr_reader :id, :topics
 
-      # @param topics [Array<Topic>] all the topics that share the same key settings
+      # @param topics [Karafka::Routing::Topics] all the topics that share the same key settings
       # @return [SubscriptionGroup] built subscription group
       def initialize(topics)
         @id = SecureRandom.uuid
@@ -44,7 +44,7 @@ module Karafka
         kafka[:'auto.offset.reset'] ||= @topics.first.initial_offset
         # Karafka manages the offsets based on the processing state, thus we do not rely on the
         # rdkafka offset auto-storing
-        kafka[:'enable.auto.offset.store'] = 'false'
+        kafka[:'enable.auto.offset.store'] = false
         kafka.freeze
         kafka
       end

data/lib/karafka/routing/subscription_groups_builder.rb

@@ -23,8 +23,8 @@ module Karafka
 
       private_constant :DISTRIBUTION_KEYS
 
-      # @param topics [Array<Topic>] array with topics based on which we want to build subscription
-      #   groups
+      # @param topics [Karafka::Routing::Topics] all the topics based on which we want to build
+      #   subscription groups
       # @return [Array<SubscriptionGroup>] all subscription groups we need in separate threads
       def call(topics)
         topics
@@ -32,6 +32,7 @@ module Karafka
           .group_by(&:first)
           .values
           .map { |value| value.map(&:last) }
+          .map { |topics_array| Routing::Topics.new(topics_array) }
           .map { |grouped_topics| SubscriptionGroup.new(grouped_topics) }
       end
 

data/lib/karafka/routing/topic.rb

@@ -66,6 +66,11 @@ module Karafka
         end
       end
 
+      # @return [Boolean] true if this topic offset is handled by the end user
+      def manual_offset_management?
+        manual_offset_management
+      end
+
       # @return [Hash] hash with all the topic attributes
       # @note This is being used when we validate the consumer_group and its topics
       def to_h

data/lib/karafka/routing/topics.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # Abstraction layer on top of groups of topics
+    class Topics
+      include Enumerable
+      extend Forwardable
+
+      def_delegators :@accumulator, :[], :size, :empty?, :last, :<<
+
+      # @param topics_array [Array<Karafka::Routing::Topic>] array with topics
+      def initialize(topics_array)
+        @accumulator = topics_array.dup
+      end
+
+      # Yields each topic
+      #
+      # @param [Proc] block we want to yield with on each topic
+      def each(&block)
+        @accumulator.each(&block)
+      end
+
+      # Finds topic by its name
+      #
+      # @param topic_name [String] topic name
+      # @return [Karafka::Routing::Topic]
+      # @raise [Karafka::Errors::TopicNotFoundError] this should never happen. If you see it,
+      #   please create an issue.
+      def find(topic_name)
+        @accumulator.find { |topic| topic.name == topic_name } ||
+          raise(Karafka::Errors::TopicNotFoundError, topic_name)
+      end
+    end
+  end
+end