karafka 2.0.0.alpha6 → 2.0.0.beta3

data/lib/karafka/licenser.rb

@@ -33,6 +33,8 @@ module Karafka
 
       return if license_config.expires_on > Date.today
 
+      raise_expired_license_token_in_dev(license_config.expires_on)
+
       notify_if_license_expired(license_config.expires_on)
     end
 
@@ -53,24 +55,41 @@ module Karafka
       )
     end
 
+    # Raises an error for test and dev environments if running pro with expired license
+    # We never want to cause any non-dev problems and we should never crash anything else than
+    # tests and development envs.
+    #
+    # @param expires_on [Date] when the license expires
+    def raise_expired_license_token_in_dev(expires_on)
+      env = Karafka::App.env
+
+      return unless env.development? || env.test?
+
+      raise Errors::ExpiredLicenseTokenError.new, expired_message(expires_on)
+    end
+
     # We do not raise an error here as we don't want to cause any problems to someone that runs
     # Karafka on production. Error message is enough.
     #
     # @param expires_on [Date] when the license expires
     def notify_if_license_expired(expires_on)
-      message = <<~MSG.tr("\n", ' ')
-        Your license expired on #{expires_on}.
-        Please reach us at contact@karafka.io or visit https://karafka.io to obtain a valid one.
-      MSG
-
-      Karafka.logger.error(message)
+      Karafka.logger.error(expired_message(expires_on))
 
       Karafka.monitor.instrument(
         'error.occurred',
         caller: self,
-        error: Errors::ExpiredLicenseTokenError.new(message),
+        error: Errors::ExpiredLicenseTokenError.new(expired_message(expires_on)),
         type: 'licenser.expired'
       )
     end
+
+    # @param expires_on [Date] when the license expires
+    # @return [String] expired message
+    def expired_message(expires_on)
+      <<~MSG.tr("\n", ' ')
+        Your license expired on #{expires_on}.
+        Please reach us at contact@karafka.io or visit https://karafka.io to obtain a valid one.
+      MSG
+    end
   end
 end

data/lib/karafka/messages/batch_metadata.rb

@@ -13,10 +13,33 @@ module Karafka
       :deserializer,
       :partition,
       :topic,
+      :created_at,
       :scheduled_at,
-      :consumption_lag,
-      :processing_lag,
+      :processed_at,
       keyword_init: true
-    )
+    ) do
+      # This lag describes how long did it take for a message to be consumed from the moment it was
+      # created
+      def consumption_lag
+        time_distance_in_ms(processed_at, created_at)
+      end
+
+      # This lag describes how long did a batch have to wait before it was picked up by one of the
+      # workers
+      def processing_lag
+        time_distance_in_ms(processed_at, scheduled_at)
+      end
+
+      private
+
+      # Computes time distance in between two times in ms
+      #
+      # @param time1 [Time]
+      # @param time2 [Time]
+      # @return [Integer] distance in between two times in ms
+      def time_distance_in_ms(time1, time2)
+        ((time1 - time2) * 1_000).round
+      end
+    end
   end
 end

data/lib/karafka/messages/builders/batch_metadata.rb

@@ -8,42 +8,30 @@ module Karafka
         class << self
           # Creates metadata based on the kafka batch data.
           #
-          # @param kafka_batch [Array<Rdkafka::Consumer::Message>] raw fetched messages
+          # @param messages [Array<Karafka::Messages::Message>] messages array
           # @param topic [Karafka::Routing::Topic] topic for which we've fetched the batch
           # @param scheduled_at [Time] moment when the batch was scheduled for processing
           # @return [Karafka::Messages::BatchMetadata] batch metadata object
-          # @note Regarding the time lags: we can use the current time here, as batch metadata is
-          #   created in the worker. So whenever this is being built, it means that the processing
-          #   of this batch has already started.
-          def call(kafka_batch, topic, scheduled_at)
-            now = Time.now
-
+          #
+          # @note We do not set `processed_at` as this needs to be assigned when the batch is
+          #   picked up for processing.
+          def call(messages, topic, scheduled_at)
             Karafka::Messages::BatchMetadata.new(
-              size: kafka_batch.count,
-              first_offset: kafka_batch.first.offset,
-              last_offset: kafka_batch.last.offset,
+              size: messages.count,
+              first_offset: messages.first.offset,
+              last_offset: messages.last.offset,
               deserializer: topic.deserializer,
-              partition: kafka_batch[0].partition,
+              partition: messages.first.partition,
               topic: topic.name,
+              # We go with the assumption that the creation of the whole batch is the last message
+              # creation time
+              created_at: messages.last.timestamp,
+              # When this batch was built and scheduled for execution
               scheduled_at: scheduled_at,
-              # This lag describes how long did it take for a message to be consumed from the
-              # moment it was created
-              consumption_lag: time_distance_in_ms(now, kafka_batch.last.timestamp),
-              # This lag describes how long did a batch have to wait before it was picked up by
-              # one of the workers
-              processing_lag: time_distance_in_ms(now, scheduled_at)
-            ).freeze
-          end
-
-          private
-
-          # Computes time distance in between two times in ms
-          #
-          # @param time1 [Time]
-          # @param time2 [Time]
-          # @return [Integer] distance in between two times in ms
-          def time_distance_in_ms(time1, time2)
-            ((time1 - time2) * 1_000).round
+              # We build the batch metadata when we pick up the job in the worker, thus we can use
+              # current time here
+              processed_at: Time.now
+            )
           end
         end
       end

data/lib/karafka/messages/builders/message.rb

@@ -26,6 +26,7 @@ module Karafka
               received_at: received_at
             ).freeze
 
+            # Karafka messages cannot be frozen because of the lazy deserialization feature
             Karafka::Messages::Message.new(
               kafka_message.payload,
               metadata

data/lib/karafka/messages/builders/messages.rb

@@ -9,27 +9,19 @@ module Karafka
           # Creates messages batch with messages inside based on the incoming messages and the
           # topic from which it comes.
           #
-          # @param kafka_messages [Array<Rdkafka::Consumer::Message>] raw fetched messages
+          # @param messages [Array<Karafka::Messages::Message>] karafka messages array
           # @param topic [Karafka::Routing::Topic] topic for which we're received messages
           # @param received_at [Time] moment in time when the messages were received
           # @return [Karafka::Messages::Messages] messages batch object
-          def call(kafka_messages, topic, received_at)
-            messages_array = kafka_messages.map do |message|
-              Karafka::Messages::Builders::Message.call(
-                message,
-                topic,
-                received_at
-              )
-            end
-
+          def call(messages, topic, received_at)
             metadata = BatchMetadata.call(
-              kafka_messages,
+              messages,
               topic,
               received_at
             ).freeze
 
             Karafka::Messages::Messages.new(
-              messages_array,
+              messages,
               metadata
             ).freeze
           end

data/lib/karafka/pro/active_job/consumer.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module ActiveJob
+      # Pro ActiveJob consumer that is suppose to handle long-running jobs as well as short
+      # running jobs
+      #
+      # When in LRJ, it will pause a given partition forever and will resume its processing only
+      # when all the jobs are done processing.
+      #
+      # It contains slightly better revocation warranties than the regular blocking consumer as
+      # it can stop processing batch of jobs in the middle after the revocation.
+      class Consumer < Karafka::ActiveJob::Consumer
+        # Runs ActiveJob jobs processing and handles lrj if needed
+        def consume
+          messages.each do |message|
+            # If for any reason we've lost this partition, not worth iterating over new messages
+            # as they are no longer ours
+            return if revoked?
+            break if Karafka::App.stopping?
+
+            ::ActiveJob::Base.execute(
+              ::ActiveSupport::JSON.decode(message.raw_payload)
+            )
+
+            # We check it twice as the job may be long running
+            return if revoked?
+
+            mark_as_consumed(message)
+
+            # Do not process more if we are shutting down
+            break if Karafka::App.stopping?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/active_job/dispatcher.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 # This Karafka component is a Pro component.
-# All of the commercial components are present in the lib/karafka/pro directory of this repository
-# and their usage requires commercial license agreement.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
 #
 # Karafka has also commercial-friendly license, commercial support and commercial components.
 #
@@ -18,7 +18,7 @@ module Karafka
       # much better and more granular control over the dispatch and consumption process.
       class Dispatcher < ::Karafka::ActiveJob::Dispatcher
         # Defaults for dispatching
-        # The can be updated by using `#karafka_options` on the job
+        # They can be updated by using `#karafka_options` on the job
         DEFAULTS = {
           dispatch_method: :produce_async,
           # We don't create a dummy proc based partitioner as we would have to evaluate it with

data/lib/karafka/pro/active_job/job_options_contract.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 # This Karafka component is a Pro component.
-# All of the commercial components are present in the lib/karafka/pro directory of this repository
-# and their usage requires commercial license agreement.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
 #
 # Karafka has also commercial-friendly license, commercial support and commercial components.
 #

data/lib/karafka/pro/base_consumer_extensions.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Extensions to the base consumer that make it more pro and fancy
+    #
+    # @note In case of using lrj, manual pausing may not be the best idea as resume needs to happen
+    #   after each batch is processed.
+    #
+    # They need to be added to the consumer via `#prepend`
+    module BaseConsumerExtensions
+      # Pause for tops 31 years
+      MAX_PAUSE_TIME = 1_000_000_000_000
+
+      private_constant :MAX_PAUSE_TIME
+
+      # Pauses processing of a given partition until we're done with the processing
+      # This ensures, that we can easily poll not reaching the `max.poll.interval`
+      def on_prepare
+        # Pause at the first message in a batch. That way in case of a crash, we will not loose
+        # any messages
+        pause(messages.first.offset, MAX_PAUSE_TIME) if topic.long_running_job?
+
+        super
+      end
+
+      # After user code, we seek and un-pause our partition
+      def on_consume
+        # If anything went wrong here, we should not run any partition management as it's Karafka
+        # core that will handle the backoff
+        return unless super
+
+        return unless topic.long_running_job?
+
+        # Nothing to resume if it was revoked
+        return if revoked?
+
+        # Once processing is done, we move to the new offset based on commits
+        seek(@seek_offset || messages.first.offset)
+        resume
+      end
+
+      # Marks this consumer revoked state as true
+      # This allows us for things like lrj to finish early as this state may change during lrj
+      # execution
+      def on_revoked
+        @revoked = true
+        super
+      end
+
+      # @return [Boolean] true if partition was revoked from the current consumer
+      def revoked?
+        @revoked || false
+      end
+    end
+  end
+end

data/lib/karafka/pro/loader.rb

@@ -1,27 +1,50 @@
 # frozen_string_literal: true
 
 # This Karafka component is a Pro component.
-# All of the commercial components are present in the lib/karafka/pro directory of this repository
-# and their usage requires commercial license agreement.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
 #
 # Karafka has also commercial-friendly license, commercial support and commercial components.
 #
 # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
 # your code to Maciej Mensfeld.
+
 module Karafka
   module Pro
     # Loader requires and loads all the pro components only when they are needed
     class Loader
+      # All the pro components that need to be loaded
+      COMPONENTS = %w[
+        performance_tracker
+        scheduler
+        base_consumer_extensions
+        processing/jobs/consume_non_blocking
+        processing/jobs_builder
+        routing/extensions
+        active_job/consumer
+        active_job/dispatcher
+        active_job/job_options_contract
+      ].freeze
+
+      private_constant :COMPONENTS
+
       class << self
         # Loads all the pro components and configures them wherever it is expected
         # @param config [Dry::Configurable::Config] whole app config that we can alter with pro
         #   components
         def setup(config)
-          require_relative 'active_job/dispatcher'
-          require_relative 'active_job/job_options_contract'
+          COMPONENTS.each { |component| require_relative(component) }
 
+          config.internal.scheduler = Scheduler.new
+          config.internal.jobs_builder = Processing::JobsBuilder.new
+          config.internal.active_job.consumer = ActiveJob::Consumer
           config.internal.active_job.dispatcher = ActiveJob::Dispatcher.new
           config.internal.active_job.job_options_contract = ActiveJob::JobOptionsContract.new
+
+          ::Karafka::Routing::Topic.include(Routing::Extensions)
+          ::Karafka::BaseConsumer.prepend(BaseConsumerExtensions)
+
+          config.monitor.subscribe(PerformanceTracker.instance)
         end
       end
     end

data/lib/karafka/pro/performance_tracker.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Tracker used to keep track of performance metrics
+    # It provides insights that can be used to optimize processing flow
+    class PerformanceTracker
+      include Singleton
+
+      # How many samples do we collect per topic partition
+      SAMPLES_COUNT = 200
+
+      private_constant :SAMPLES_COUNT
+
+      # Builds up nested concurrent hash for data tracking
+      def initialize
+        @processing_times = Concurrent::Hash.new do |topics_hash, topic|
+          topics_hash[topic] = Concurrent::Hash.new do |partitions_hash, partition|
+            # This array does not have to be concurrent because we always access single partition
+            # data via instrumentation that operates in a single thread via consumer
+            partitions_hash[partition] = []
+          end
+        end
+      end
+
+      # @param topic [String]
+      # @param partition [Integer]
+      # @return [Float] p95 processing time of a single message from a single topic partition
+      def processing_time_p95(topic, partition)
+        values = @processing_times[topic][partition]
+
+        return 0 if values.empty?
+        return values.first if values.size == 1
+
+        percentile(0.95, values)
+      end
+
+      # @private
+      # @param event [Dry::Events::Event] event details
+      # Tracks time taken to process a single message of a given topic partition
+      def on_consumer_consumed(event)
+        consumer = event[:caller]
+        messages = consumer.messages
+        topic = messages.metadata.topic
+        partition = messages.metadata.partition
+
+        samples = @processing_times[topic][partition]
+        samples << event[:time] / messages.count
+
+        return unless samples.size > SAMPLES_COUNT
+
+        samples.shift
+      end
+
+      private
+
+      # Computers the requested percentile out of provided values
+      # @param percentile [Float]
+      # @param values [Array<String>] all the values based on which we should
+      # @return [Float] computed percentile
+      def percentile(percentile, values)
+        values_sorted = values.sort
+
+        floor = (percentile * (values_sorted.length - 1) + 1).floor - 1
+        mod = (percentile * (values_sorted.length - 1) + 1).modulo(1)
+
+        values_sorted[floor] + (mod * (values_sorted[floor + 1] - values_sorted[floor]))
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Pro components related to processing part of Karafka
+    module Processing
+      # Pro jobs
+      module Jobs
+        # The main job type in a non-blocking variant.
+        # This variant works "like" the regular consumption but pauses the partition for as long
+        # as it is needed until a job is done.
+        #
+        # It can be useful when having long lasting jobs that would exceed `max.poll.interval`
+        # if would block.
+        #
+        # @note It needs to be working with a proper consumer that will handle the partition
+        #   management. This layer of the framework knows nothing about Kafka messages consumption.
+        class ConsumeNonBlocking < ::Karafka::Processing::Jobs::Consume
+          # Releases the blocking lock after it is done with the preparation phase for this job
+          def prepare
+            super
+            @non_blocking = true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/jobs_builder.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Pro jobs builder that supports lrj
+      class JobsBuilder < ::Karafka::Processing::JobsBuilder
+        # @param executor [Karafka::Processing::Executor]
+        # @param messages [Karafka::Messages::Messages] messages batch to be consumed
+        # @return [Karafka::Processing::Jobs::Consume] blocking job
+        # @return [Karafka::Pro::Processing::Jobs::ConsumeNonBlocking] non blocking for lrj
+        def consume(executor, messages)
+          if executor.topic.long_running_job?
+            Jobs::ConsumeNonBlocking.new(executor, messages)
+          else
+            super
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/extensions.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Pro routing components
+    module Routing
+      # Routing extensions that allow to configure some extra PRO routing options
+      module Extensions
+        class << self
+          # @param base [Class] class we extend
+          def included(base)
+            base.attr_accessor :long_running_job
+          end
+        end
+
+        # @return [Boolean] is a given job on a topic a long running one
+        def long_running_job?
+          @long_running_job || false
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduler.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Optimizes scheduler that takes into consideration of execution time needed to process
+    # messages from given topics partitions. It uses the non-preemptive LJF algorithm
+    #
+    # This scheduler is designed to optimize execution times on jobs that perform IO operations as
+    # when taking IO into consideration, the can achieve optimized parallel processing.
+    #
+    # This scheduler can also work with virtual partitions.
+    #
+    # Aside from consumption jobs, other jobs do not run often, thus we can leave them with
+    # default FIFO scheduler from the default Karafka scheduler
+    class Scheduler < ::Karafka::Scheduler
+      # Schedules jobs in the LJF order for consumption
+      #
+      # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
+      # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
+      #
+      def schedule_consumption(queue, jobs_array)
+        pt = PerformanceTracker.instance
+
+        ordered = []
+
+        jobs_array.each do |job|
+          messages = job.messages
+          message = messages.first
+
+          cost = pt.processing_time_p95(message.topic, message.partition) * messages.size
+
+          ordered << [job, cost]
+        end
+
+        ordered.sort_by!(&:last)
+        ordered.reverse!
+        ordered.map!(&:first)
+
+        ordered.each do |job|
+          queue << job
+        end
+      end
+    end
+  end
+end