karafka 2.0.0.beta2 → 2.0.0.beta5

data/lib/karafka/contracts/config.rb

@@ -30,11 +30,25 @@ module Karafka
 
         # We validate internals just to be sure, that they are present and working
         required(:internal).schema do
-          required(:routing_builder)
           required(:status)
           required(:process)
-          required(:scheduler)
-          required(:subscription_groups_builder)
+
+          required(:routing).schema do
+            required(:builder)
+            required(:subscription_groups_builder)
+          end
+
+          required(:processing).schema do
+            required(:jobs_builder)
+            required(:scheduler)
+            required(:coordinator_class)
+          end
+
+          required(:active_job).schema do
+            required(:dispatcher)
+            required(:job_options_contract)
+            required(:consumer_class)
+          end
         end
       end
 

data/lib/karafka/contracts/server_cli_options.rb

@@ -12,7 +12,7 @@ module Karafka
         # If there were no consumer_groups declared in the server cli, it means that we will
         # run all of them and no need to validate them here at all
         if !value.nil? &&
-           !(value - Karafka::App.config.internal.routing_builder.map(&:name)).empty?
+           !(value - Karafka::App.config.internal.routing.builder.map(&:name)).empty?
           key(:consumer_groups).failure(:consumer_groups_inclusion)
         end
       end

data/lib/karafka/errors.rb

@@ -47,5 +47,8 @@ module Karafka
     # Used to instrument this error into the error notifications
     # We do not raise it so we won't crash deployed systems
     ExpiredLicenseTokenError = Class.new(BaseError)
+
+    # This should never happen. Please open an issue if it does.
+    InvalidCoordinatorState = Class.new(BaseError)
   end
 end

data/lib/karafka/instrumentation/logger_listener.rb

@@ -15,16 +15,43 @@ module Karafka
 
       # Logs each messages fetching attempt
       #
-      # @param _event [Dry::Events::Event] event details including payload
-      def on_connection_listener_fetch_loop(_event)
-        info 'Receiving new messages from Kafka...'
+      # @param event [Dry::Events::Event] event details including payload
+      def on_connection_listener_fetch_loop(event)
+        listener = event[:caller]
+        info "[#{listener.id}] Polling messages..."
       end
 
       # Logs about messages that we've received from Kafka
       #
       # @param event [Dry::Events::Event] event details including payload
       def on_connection_listener_fetch_loop_received(event)
-        info "Received #{event[:messages_buffer].size} new messages from Kafka"
+        listener = event[:caller]
+        time = event[:time]
+        messages_count = event[:messages_buffer].size
+        info "[#{listener.id}] Polled #{messages_count} messages in #{time}ms"
+      end
+
+      # Prints info about the fact that a given job has started
+      #
+      # @param event [Dry::Events::Event] event details including payload
+      def on_worker_process(event)
+        job = event[:job]
+        job_type = job.class.to_s.split('::').last
+        consumer = job.executor.topic.consumer
+        topic = job.executor.topic.name
+        info "[#{job.id}] #{job_type} job for #{consumer} on #{topic} started"
+      end
+
+      # Prints info about the fact that a given job has finished
+      #
+      # @param event [Dry::Events::Event] event details including payload
+      def on_worker_processed(event)
+        job = event[:job]
+        time = event[:time]
+        job_type = job.class.to_s.split('::').last
+        consumer = job.executor.topic.consumer
+        topic = job.executor.topic.name
+        info "[#{job.id}] #{job_type} job for #{consumer} on #{topic} finished in #{time}ms"
       end
 
       # Logs info about system signals that Karafka received.
@@ -52,16 +79,14 @@ module Karafka
       #
       # @param _event [Dry::Events::Event] event details including payload
       def on_app_stopping(_event)
-        # We use a separate thread as logging can't be called from trap context
-        Thread.new { info 'Stopping Karafka server' }
+        info 'Stopping Karafka server'
       end
 
       # Logs info that we stopped the Karafka server.
       #
       # @param _event [Dry::Events::Event] event details including payload
       def on_app_stopped(_event)
-        # We use a separate thread as logging can't be called from trap context
-        Thread.new { info 'Stopped Karafka server' }
+        info 'Stopped Karafka server'
       end
 
       # There are many types of errors that can occur in many places, but we provide a single
@@ -95,8 +120,7 @@ module Karafka
           fatal "Runner crashed due to an error: #{error}"
           fatal details
         when 'app.stopping.error'
-          # We use a separate thread as logging can't be called from trap context
-          Thread.new { error 'Forceful Karafka server stop' }
+          error 'Forceful Karafka server stop'
         when 'librdkafka.error'
           error "librdkafka internal error occurred: #{error}"
           error details

data/lib/karafka/instrumentation/monitor.rb

@@ -22,7 +22,6 @@ module Karafka
         app.stopping
         app.stopped
 
-        consumer.prepared
         consumer.consumed
         consumer.revoked
         consumer.shutdown
@@ -33,6 +32,9 @@ module Karafka
         connection.listener.fetch_loop
         connection.listener.fetch_loop.received
 
+        worker.process
+        worker.processed
+
         statistics.emitted
 
         error.occurred

data/lib/karafka/licenser.rb

@@ -33,6 +33,8 @@ module Karafka
 
       return if license_config.expires_on > Date.today
 
+      raise_expired_license_token_in_dev(license_config.expires_on)
+
       notify_if_license_expired(license_config.expires_on)
     end
 
@@ -53,24 +55,41 @@ module Karafka
       )
     end
 
+    # Raises an error for test and dev environments if running pro with expired license
+    # We never want to cause any non-dev problems and we should never crash anything else than
+    # tests and development envs.
+    #
+    # @param expires_on [Date] when the license expires
+    def raise_expired_license_token_in_dev(expires_on)
+      env = Karafka::App.env
+
+      return unless env.development? || env.test?
+
+      raise Errors::ExpiredLicenseTokenError.new, expired_message(expires_on)
+    end
+
     # We do not raise an error here as we don't want to cause any problems to someone that runs
     # Karafka on production. Error message is enough.
     #
     # @param expires_on [Date] when the license expires
     def notify_if_license_expired(expires_on)
-      message = <<~MSG.tr("\n", ' ')
-        Your license expired on #{expires_on}.
-        Please reach us at contact@karafka.io or visit https://karafka.io to obtain a valid one.
-      MSG
-
-      Karafka.logger.error(message)
+      Karafka.logger.error(expired_message(expires_on))
 
       Karafka.monitor.instrument(
         'error.occurred',
         caller: self,
-        error: Errors::ExpiredLicenseTokenError.new(message),
+        error: Errors::ExpiredLicenseTokenError.new(expired_message(expires_on)),
         type: 'licenser.expired'
       )
     end
+
+    # @param expires_on [Date] when the license expires
+    # @return [String] expired message
+    def expired_message(expires_on)
+      <<~MSG.tr("\n", ' ')
+        Your license expired on #{expires_on}.
+        Please reach us at contact@karafka.io or visit https://karafka.io to obtain a valid one.
+      MSG
+    end
   end
 end

data/lib/karafka/pro/active_job/consumer.rb

@@ -1,20 +1,41 @@
 # frozen_string_literal: true
 
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
 module Karafka
   module Pro
     module ActiveJob
-      # This Karafka component is a Pro component.
-      # All of the commercial components are present in the lib/karafka/pro directory of this
-      # repository and their usage requires commercial license agreement.
+      # Pro ActiveJob consumer that is suppose to handle long-running jobs as well as short
+      # running jobs
       #
-      # Karafka has also commercial-friendly license, commercial support and commercial components.
+      # When in LRJ, it will pause a given partition forever and will resume its processing only
+      # when all the jobs are done processing.
       #
-      # By sending a pull request to the pro components, you are agreeing to transfer the copyright
-      # of your code to Maciej Mensfeld.
+      # It contains slightly better revocation warranties than the regular blocking consumer as
+      # it can stop processing batch of jobs in the middle after the revocation.
+      class Consumer < Karafka::Pro::BaseConsumer
+        # Runs ActiveJob jobs processing and handles lrj if needed
+        def consume
+          messages.each do |message|
+            # If for any reason we've lost this partition, not worth iterating over new messages
+            # as they are no longer ours
+            break if revoked?
+            break if Karafka::App.stopping?
 
-      # Pro ActiveJob consumer that is suppose to handle long-running jobs as well as short
-      # running jobs
-      class Consumer < Karafka::ActiveJob::Consumer
+            ::ActiveJob::Base.execute(
+              ::ActiveSupport::JSON.decode(message.raw_payload)
+            )
+
+            mark_as_consumed(message)
+          end
+        end
       end
     end
   end

data/lib/karafka/pro/active_job/dispatcher.rb

@@ -1,18 +1,18 @@
 # frozen_string_literal: true
 
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
 module Karafka
   module Pro
     # Karafka Pro ActiveJob components
     module ActiveJob
-      # This Karafka component is a Pro component.
-      # All of the commercial components are present in the lib/karafka/pro directory of this
-      # repository and their usage requires commercial license agreement.
-      #
-      # Karafka has also commercial-friendly license, commercial support and commercial components.
-      #
-      # By sending a pull request to the pro components, you are agreeing to transfer the copyright
-      # of your code to Maciej Mensfeld.
-
       # Pro dispatcher that sends the ActiveJob job to a proper topic based on the queue name
       # and that allows to inject additional options into the producer, effectively allowing for a
       # much better and more granular control over the dispatch and consumption process.

data/lib/karafka/pro/active_job/job_options_contract.rb

@@ -1,17 +1,17 @@
 # frozen_string_literal: true
 
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
 module Karafka
   module Pro
     module ActiveJob
-      # This Karafka component is a Pro component.
-      # All of the commercial components are present in the lib/karafka/pro directory of this
-      # repository and their usage requires commercial license agreement.
-      #
-      # Karafka has also commercial-friendly license, commercial support and commercial components.
-      #
-      # By sending a pull request to the pro components, you are agreeing to transfer the copyright
-      # of your code to Maciej Mensfeld.
-
       # Contract for validating the options that can be altered with `#karafka_options` per job
       # class that works with Pro features.
       class JobOptionsContract < ::Karafka::ActiveJob::JobOptionsContract

data/lib/karafka/pro/base_consumer.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Karafka PRO consumer.
+    #
+    # If you use PRO, all your consumers should inherit (indirectly) from it.
+    #
+    # @note In case of using lrj, manual pausing may not be the best idea as resume needs to happen
+    #   after each batch is processed.
+    class BaseConsumer < Karafka::BaseConsumer
+      # Pause for tops 31 years
+      MAX_PAUSE_TIME = 1_000_000_000_000
+
+      private_constant :MAX_PAUSE_TIME
+
+      # Pauses processing of a given partition until we're done with the processing
+      # This ensures, that we can easily poll not reaching the `max.poll.interval`
+      def on_before_consume
+        # Pause at the first message in a batch. That way in case of a crash, we will not loose
+        # any messages
+        return unless topic.long_running_job?
+
+        pause(messages.first.offset, MAX_PAUSE_TIME)
+      end
+
+      # Runs extra logic after consumption that is related to handling long running jobs
+      # @note This overwrites the '#on_after_consume' from the base consumer
+      def on_after_consume
+        # Nothing to do if we lost the partition
+        return if revoked?
+
+        if @coordinator.success?
+          coordinator.pause_tracker.reset
+
+          # We use the non-blocking one here. If someone needs the blocking one, can implement it
+          # with manual offset management
+          # Mark as consumed only if manual offset management is not on
+          mark_as_consumed(messages.last) unless topic.manual_offset_management?
+
+          # We check it twice as marking could change this state
+          return if revoked?
+
+          # If this is not a long running job there is nothing for us to do here
+          return unless topic.long_running_job?
+
+          # Once processing is done, we move to the new offset based on commits
+          # Here, in case manual offset management is off, we have the new proper offset of a
+          # first message from another batch from `@seek_offset`. If manual offset management
+          # is on, we move to place where the user indicated it was finished. This can create an
+          # interesting (yet valid) corner case, where with manual offset management on and no
+          # marking as consumed, we end up with an infinite loop processing same messages over and
+          # over again
+          seek(@seek_offset || messages.first.offset)
+
+          resume
+        else
+          # If processing failed, we need to pause
+          pause(@seek_offset || messages.first.offset)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/loader.rb

@@ -1,34 +1,52 @@
 # frozen_string_literal: true
 
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
 module Karafka
   module Pro
-    # This Karafka component is a Pro component.
-    # All of the commercial components are present in the lib/karafka/pro directory of this
-    # repository and their usage requires commercial license agreement.
-    #
-    # Karafka has also commercial-friendly license, commercial support and commercial components.
-    #
-    # By sending a pull request to the pro components, you are agreeing to transfer the copyright
-    # of your code to Maciej Mensfeld.
-
     # Loader requires and loads all the pro components only when they are needed
     class Loader
+      # All the pro components that need to be loaded
+      COMPONENTS = %w[
+        base_consumer
+        performance_tracker
+        processing/scheduler
+        processing/jobs/consume_non_blocking
+        processing/jobs_builder
+        processing/coordinator
+        routing/extensions
+        active_job/consumer
+        active_job/dispatcher
+        active_job/job_options_contract
+      ].freeze
+
+      private_constant :COMPONENTS
+
       class << self
         # Loads all the pro components and configures them wherever it is expected
         # @param config [Dry::Configurable::Config] whole app config that we can alter with pro
         #   components
         def setup(config)
-          require_relative 'performance_tracker'
-          require_relative 'scheduler'
-          require_relative 'processing/jobs/consume_non_blocking'
-          require_relative 'active_job/consumer'
-          require_relative 'active_job/dispatcher'
-          require_relative 'active_job/job_options_contract'
-
-          config.internal.scheduler = Scheduler.new
-          config.internal.active_job.consumer = ActiveJob::Consumer
-          config.internal.active_job.dispatcher = ActiveJob::Dispatcher.new
-          config.internal.active_job.job_options_contract = ActiveJob::JobOptionsContract.new
+          COMPONENTS.each { |component| require_relative(component) }
+
+          icfg = config.internal
+
+          icfg.processing.coordinator_class = Processing::Coordinator
+          icfg.processing.scheduler = Processing::Scheduler.new
+          icfg.processing.jobs_builder = Processing::JobsBuilder.new
+
+          icfg.active_job.consumer_class = ActiveJob::Consumer
+          icfg.active_job.dispatcher = ActiveJob::Dispatcher.new
+          icfg.active_job.job_options_contract = ActiveJob::JobOptionsContract.new
+
+          ::Karafka::Routing::Topic.include(Routing::Extensions)
 
           config.monitor.subscribe(PerformanceTracker.instance)
         end

data/lib/karafka/pro/performance_tracker.rb

@@ -1,16 +1,16 @@
 # frozen_string_literal: true
 
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
 module Karafka
   module Pro
-    # This Karafka component is a Pro component.
-    # All of the commercial components are present in the lib/karafka/pro directory of this
-    # repository and their usage requires commercial license agreement.
-    #
-    # Karafka has also commercial-friendly license, commercial support and commercial components.
-    #
-    # By sending a pull request to the pro components, you are agreeing to transfer the copyright
-    # of your code to Maciej Mensfeld.
-
     # Tracker used to keep track of performance metrics
     # It provides insights that can be used to optimize processing flow
     class PerformanceTracker

data/lib/karafka/pro/processing/coordinator.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Pro
+    module Processing
+      # Pro coordinator that provides extra orchestration methods useful for parallel processing
+      # within the same partition
+      class Coordinator < ::Karafka::Processing::Coordinator
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb

@@ -1,21 +1,20 @@
 # frozen_string_literal: true
 
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
 module Karafka
   module Pro
     # Pro components related to processing part of Karafka
     module Processing
       # Pro jobs
       module Jobs
-        # This Karafka component is a Pro component.
-        # All of the commercial components are present in the lib/karafka/pro directory of this
-        # repository and their usage requires commercial license agreement.
-        #
-        # Karafka has also commercial-friendly license, commercial support and commercial
-        # components.
-        #
-        # By sending a pull request to the pro components, you are agreeing to transfer the
-        # copyright of your code to Maciej Mensfeld.
-
         # The main job type in a non-blocking variant.
         # This variant works "like" the regular consumption but pauses the partition for as long
         # as it is needed until a job is done.
@@ -27,7 +26,7 @@ module Karafka
         #   management. This layer of the framework knows nothing about Kafka messages consumption.
         class ConsumeNonBlocking < ::Karafka::Processing::Jobs::Consume
           # Releases the blocking lock after it is done with the preparation phase for this job
-          def prepare
+          def before_call
             super
             @non_blocking = true
           end

data/lib/karafka/pro/processing/jobs_builder.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Pro jobs builder that supports lrj
+      class JobsBuilder < ::Karafka::Processing::JobsBuilder
+        # @param executor [Karafka::Processing::Executor]
+        # @param messages [Karafka::Messages::Messages] messages batch to be consumed
+        # @param coordinator [Karafka::Processing::Coordinator]
+        # @return [Karafka::Processing::Jobs::Consume] blocking job
+        # @return [Karafka::Pro::Processing::Jobs::ConsumeNonBlocking] non blocking for lrj
+        def consume(executor, messages, coordinator)
+          if executor.topic.long_running_job?
+            Jobs::ConsumeNonBlocking.new(executor, messages, coordinator)
+          else
+            super
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/scheduler.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Optimizes scheduler that takes into consideration of execution time needed to process
+      # messages from given topics partitions. It uses the non-preemptive LJF algorithm
+      #
+      # This scheduler is designed to optimize execution times on jobs that perform IO operations
+      # as when taking IO into consideration, the can achieve optimized parallel processing.
+      #
+      # This scheduler can also work with virtual partitions.
+      #
+      # Aside from consumption jobs, other jobs do not run often, thus we can leave them with
+      # default FIFO scheduler from the default Karafka scheduler
+      class Scheduler < ::Karafka::Processing::Scheduler
+        # Schedules jobs in the LJF order for consumption
+        #
+        # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
+        # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
+        #
+        def schedule_consumption(queue, jobs_array)
+          pt = PerformanceTracker.instance
+
+          ordered = []
+
+          jobs_array.each do |job|
+            messages = job.messages
+            message = messages.first
+
+            cost = pt.processing_time_p95(message.topic, message.partition) * messages.size
+
+            ordered << [job, cost]
+          end
+
+          ordered.sort_by!(&:last)
+          ordered.reverse!
+          ordered.map!(&:first)
+
+          ordered.each do |job|
+            queue << job
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/extensions.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Pro routing components
+    module Routing
+      # Routing extensions that allow to configure some extra PRO routing options
+      module Extensions
+        class << self
+          # @param base [Class] class we extend
+          def included(base)
+            base.attr_accessor :long_running_job
+          end
+        end
+
+        # @return [Boolean] is a given job on a topic a long running one
+        def long_running_job?
+          @long_running_job || false
+        end
+      end
+    end
+  end
+end