karafka 1.4.0 → 2.0.10

data/lib/karafka/processing/jobs/consume.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Jobs
+      # The main job type. It runs the executor that triggers given topic partition messages
+      # processing in an underlying consumer instance.
+      class Consume < Base
+        # @return [Array<Rdkafka::Consumer::Message>] array with messages
+        attr_reader :messages
+
+        # @param executor [Karafka::Processing::Executor] executor that is suppose to run a given
+        #   job
+        # @param messages [Karafka::Messages::Messages] karafka messages batch
+        # @param coordinator [Karafka::Processing::Coordinator] processing coordinator
+        # @return [Consume]
+        def initialize(executor, messages, coordinator)
+          @executor = executor
+          @messages = messages
+          @coordinator = coordinator
+          super()
+        end
+
+        # Runs all the preparation code on the executor that needs to happen before the job is
+        # enqueued.
+        def before_enqueue
+          executor.before_enqueue(@messages, @coordinator)
+        end
+
+        # Runs the before consumption preparations on the executor
+        def before_call
+          executor.before_consume
+        end
+
+        # Runs the given executor
+        def call
+          executor.consume
+        end
+
+        # Runs any error handling and other post-consumption stuff on the executor
+        def after_call
+          executor.after_consume
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/jobs/revoked.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Jobs
+      # Job that runs the revoked operation when we loose a partition on a consumer that lost it.
+      class Revoked < Base
+        # @param executor [Karafka::Processing::Executor] executor that is suppose to run the job
+        # @return [Revoked]
+        def initialize(executor)
+          @executor = executor
+          super()
+        end
+
+        # Runs the revoking job via an executor.
+        def call
+          executor.revoked
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/jobs/shutdown.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Jobs
+      # Job that runs on each active consumer upon process shutdown (one job per consumer).
+      class Shutdown < Base
+        # @param executor [Karafka::Processing::Executor] executor that is suppose to run a given
+        #   job on an active consumer
+        # @return [Shutdown]
+        def initialize(executor)
+          @executor = executor
+          super()
+        end
+
+        # Runs the shutdown job via an executor.
+        def call
+          executor.shutdown
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/jobs_builder.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Class responsible for deciding what type of job should we build to run a given command and
+    # for building a proper job for it.
+    class JobsBuilder
+      # @param executor [Karafka::Processing::Executor]
+      # @param messages [Karafka::Messages::Messages] messages batch to be consumed
+      # @param coordinator [Karafka::Processing::Coordinator]
+      # @return [Karafka::Processing::Jobs::Consume] consumption job
+      def consume(executor, messages, coordinator)
+        Jobs::Consume.new(executor, messages, coordinator)
+      end
+
+      # @param executor [Karafka::Processing::Executor]
+      # @return [Karafka::Processing::Jobs::Revoked] revocation job
+      def revoked(executor)
+        Jobs::Revoked.new(executor)
+      end
+
+      # @param executor [Karafka::Processing::Executor]
+      # @return [Karafka::Processing::Jobs::Shutdown] shutdown job
+      def shutdown(executor)
+        Jobs::Shutdown.new(executor)
+      end
+    end
+  end
+end

data/lib/karafka/processing/jobs_queue.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # This is the key work component for Karafka jobs distribution. It provides API for running
+    # jobs in parallel while operating within more than one subscription group.
+    #
+    # We need to take into consideration fact, that more than one subscription group can operate
+    # on this queue, that's why internally we keep track of processing per group.
+    #
+    # We work with the assumption, that partitions data is evenly distributed.
+    class JobsQueue
+      # @return [Karafka::Processing::JobsQueue]
+      def initialize
+        @queue = Queue.new
+        # Those queues will act as semaphores internally. Since we need an indicator for waiting
+        # we could use Thread.pass but this is expensive. Instead we can just lock until any
+        # of the workers finishes their work and we can re-check. This means that in the worse
+        # scenario, we will context switch 10 times per poll instead of getting this thread
+        # scheduled by Ruby hundreds of thousands of times per group.
+        # We cannot use a single semaphore as it could potentially block in listeners that should
+        # process with their data and also could unlock when a given group needs to remain locked
+        @semaphores = Hash.new { |h, k| h[k] = Queue.new }
+        @in_processing = Hash.new { |h, k| h[k] = [] }
+        @mutex = Mutex.new
+      end
+
+      # Returns number of jobs that are either enqueued or in processing (but not finished)
+      # @return [Integer] number of elements in the queue
+      # @note Using `#pop` won't decrease this number as only marking job as completed does this
+      def size
+        @in_processing.values.map(&:size).sum
+      end
+
+      # Adds the job to the internal main queue, scheduling it for execution in a worker and marks
+      # this job as in processing pipeline.
+      #
+      # @param job [Jobs::Base] job that we want to run
+      def <<(job)
+        # We do not push the job if the queue is closed as it means that it would anyhow not be
+        # executed
+        return if @queue.closed?
+
+        @mutex.synchronize do
+          group = @in_processing[job.group_id]
+
+          raise(Errors::JobsQueueSynchronizationError, job.group_id) if group.include?(job)
+
+          group << job
+        end
+
+        @queue << job
+      end
+
+      # @return [Jobs::Base, nil] waits for a job from the main queue and returns it once available
+      #   or returns nil if the queue has been stopped and there won't be anything more to process
+      #   ever.
+      # @note This command is blocking and will wait until any job is available on the main queue
+      def pop
+        @queue.pop
+      end
+
+      # Causes the wait lock to re-check the lock conditions and potential unlock.
+      # @param group_id [String] id of the group we want to unlock for one tick
+      # @note This does not release the wait lock. It just causes a conditions recheck
+      def tick(group_id)
+        @semaphores[group_id] << true
+      end
+
+      # Marks a given job from a given group as completed. When there are no more jobs from a given
+      # group to be executed, we won't wait.
+      #
+      # @param [Jobs::Base] job that was completed
+      def complete(job)
+        @mutex.synchronize do
+          @in_processing[job.group_id].delete(job)
+          tick(job.group_id)
+        end
+      end
+
+      # Clears the processing states for a provided group. Useful when a recovery happens and we
+      # need to clean up state but only for a given subscription group.
+      #
+      # @param group_id [String]
+      def clear(group_id)
+        @mutex.synchronize do
+          @in_processing[group_id].clear
+          # We unlock it just in case it was blocked when clearing started
+          tick(group_id)
+        end
+      end
+
+      # Stops the whole processing queue.
+      def close
+        @mutex.synchronize do
+          return if @queue.closed?
+
+          @queue.close
+          @semaphores.values.each(&:close)
+        end
+      end
+
+      # @param group_id [String]
+      #
+      # @return [Boolean] tell us if we have anything in the processing (or for processing) from
+      # a given group.
+      def empty?(group_id)
+        @in_processing[group_id].empty?
+      end
+
+      # Blocks when there are things in the queue in a given group and waits until all the blocking
+      #   jobs from a given group are completed
+      #
+      # @param group_id [String] id of the group in which jobs we're interested.
+      # @note This method is blocking.
+      def wait(group_id)
+        # Go doing other things while we cannot process and wait for anyone to finish their work
+        # and re-check the wait status
+        @semaphores[group_id].pop while wait?(group_id)
+      end
+
+      # - `processing` - number of jobs that are currently being processed (active work)
+      # - `enqueued` - number of jobs in the queue that are waiting to be picked up by a worker
+      #
+      # @return [Hash] hash with basic usage statistics of this queue.
+      def statistics
+        {
+          processing: size - @queue.size,
+          enqueued: @queue.size
+        }.freeze
+      end
+
+      private
+
+      # @param group_id [String] id of the group in which jobs we're interested.
+      # @return [Boolean] should we keep waiting or not
+      # @note We do not wait for non-blocking jobs. Their flow should allow for `poll` running
+      #   as they may exceed `max.poll.interval`
+      def wait?(group_id)
+        !@in_processing[group_id].all?(&:non_blocking?)
+      end
+    end
+  end
+end

data/lib/karafka/processing/partitioner.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Basic partitioner for work division
+    # It does not divide any work.
+    class Partitioner
+      # @param subscription_group [Karafka::Routing::SubscriptionGroup] subscription group
+      def initialize(subscription_group)
+        @subscription_group = subscription_group
+      end
+
+      # @param _topic [String] topic name
+      # @param messages [Array<Karafka::Messages::Message>] karafka messages
+      # @yieldparam [Integer] group id
+      # @yieldparam [Array<Karafka::Messages::Message>] karafka messages
+      def call(_topic, messages)
+        yield(0, messages)
+      end
+    end
+  end
+end

data/lib/karafka/processing/result.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # A simple object that allows us to keep track of processing state.
+    # It allows to indicate if given thing moved from success to a failure or the other way around
+    # Useful for tracking consumption state
+    class Result
+      attr_reader :cause
+
+      def initialize
+        @success = true
+        @cause = false
+      end
+
+      # @return [Boolean]
+      def success?
+        @success
+      end
+
+      # Marks state as successful
+      def success!
+        @success = true
+        # We set cause to false so the previous error that occurred does not leak when error is
+        # no longer present
+        @cause = false
+      end
+
+      # Marks state as failure
+      # @param cause [StandardError] error that occurred and caused failure
+      def failure!(cause)
+        @success = false
+        @cause = cause
+      end
+    end
+  end
+end

data/lib/karafka/processing/scheduler.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # FIFO scheduler for messages coming from various topics and partitions
+    class Scheduler
+      # Schedules jobs in the fifo order
+      #
+      # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
+      # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
+      def schedule_consumption(queue, jobs_array)
+        jobs_array.each do |job|
+          queue << job
+        end
+      end
+
+      # Both revocation and shutdown jobs can also run in fifo by default
+      alias schedule_revocation schedule_consumption
+      alias schedule_shutdown schedule_consumption
+    end
+  end
+end

data/lib/karafka/processing/worker.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Workers are used to run jobs in separate threads.
+    # Workers are the main processing units of the Karafka framework.
+    #
+    # Each job runs in three stages:
+    #   - prepare - here we can run any code that we would need to run blocking before we allow
+    #               the job to run fully async (non blocking). This will always run in a blocking
+    #               way and can be used to make sure all the resources and external dependencies
+    #               are satisfied before going async.
+    #
+    #   - call - actual processing logic that can run sync or async
+    #
+    #   - teardown - it should include any code that we want to run after we executed the user
+    #                code. This can be used to unlock certain resources or do other things that are
+    #                not user code but need to run after user code base is executed.
+    class Worker
+      include Helpers::Async
+
+      # @return [String] id of this worker
+      attr_reader :id
+
+      # @param jobs_queue [JobsQueue]
+      # @return [Worker]
+      def initialize(jobs_queue)
+        @id = SecureRandom.uuid
+        @jobs_queue = jobs_queue
+      end
+
+      private
+
+      # Runs processing of jobs in a loop
+      # Stops when queue is closed.
+      def call
+        loop { break unless process }
+      end
+
+      # Fetches a single job, processes it and marks as completed.
+      #
+      # @note We do not have error handling here, as no errors should propagate this far. If they
+      #   do, it is a critical error and should bubble up.
+      #
+      # @note Upon closing the jobs queue, worker will close it's thread
+      def process
+        job = @jobs_queue.pop
+
+        instrument_details = { caller: self, job: job, jobs_queue: @jobs_queue }
+
+        if job
+          Karafka.monitor.instrument('worker.process', instrument_details)
+
+          Karafka.monitor.instrument('worker.processed', instrument_details) do
+            job.before_call
+
+            # If a job is marked as non blocking, we can run a tick in the job queue and if there
+            # are no other blocking factors, the job queue will be unlocked.
+            # If this does not run, all the things will be blocking and job queue won't allow to
+            # pass it until done.
+            @jobs_queue.tick(job.group_id) if job.non_blocking?
+
+            job.call
+
+            job.after_call
+
+            true
+          end
+        else
+          false
+        end
+      # We signal critical exceptions, notify and do not allow worker to fail
+      # rubocop:disable Lint/RescueException
+      rescue Exception => e
+        # rubocop:enable Lint/RescueException
+        Karafka.monitor.instrument(
+          'error.occurred',
+          caller: self,
+          error: e,
+          type: 'worker.process.error'
+        )
+      ensure
+        # job can be nil when the queue is being closed
+        @jobs_queue.complete(job) if job
+
+        # Always publish info, that we completed all the work despite its result
+        Karafka.monitor.instrument('worker.completed', instrument_details)
+      end
+    end
+  end
+end

data/lib/karafka/processing/workers_batch.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Abstraction layer around workers batch.
+    class WorkersBatch
+      include Enumerable
+
+      # @param jobs_queue [JobsQueue]
+      # @return [WorkersBatch]
+      def initialize(jobs_queue)
+        @batch = Array.new(App.config.concurrency) { Processing::Worker.new(jobs_queue) }
+      end
+
+      # Iterates over available workers and yields each worker
+      # @param block [Proc] block we want to run
+      def each(&block)
+        @batch.each(&block)
+      end
+
+      # @return [Integer] number of workers in the batch
+      def size
+        @batch.size
+      end
+    end
+  end
+end

data/lib/karafka/railtie.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+# This file contains Railtie for auto-configuration
+
+rails = false
+
+begin
+  require 'rails'
+
+  rails = true
+rescue LoadError
+  # Without defining this in any way, Zeitwerk ain't happy so we do it that way
+  module Karafka
+    class Railtie
+    end
+  end
+end
+
+if rails
+  # Load Karafka
+  require 'karafka'
+
+  # Load ActiveJob adapter
+  require 'active_job/karafka'
+
+  # Setup env if configured (may be configured later by .net, etc)
+  ENV['KARAFKA_ENV'] ||= ENV['RAILS_ENV'] if ENV.key?('RAILS_ENV')
+
+  module Karafka
+    # Railtie for setting up Rails integration
+    class Railtie < Rails::Railtie
+      railtie_name :karafka
+
+      initializer 'karafka.active_job_integration' do
+        ActiveSupport.on_load(:active_job) do
+          # Extend ActiveJob with some Karafka specific ActiveJob magic
+          extend ::Karafka::ActiveJob::JobExtensions
+        end
+      end
+
+      # This lines will make Karafka print to stdout like puma or unicorn when we run karafka
+      # server + will support code reloading with each fetched loop. We do it only for karafka
+      # based commands as Rails processes and console will have it enabled already
+      initializer 'karafka.configure_rails_logger' do
+        # Make Karafka use Rails logger
+        ::Karafka::App.config.logger = Rails.logger
+
+        next unless Rails.env.development?
+        next unless ENV.key?('KARAFKA_CLI')
+
+        logger = ActiveSupport::Logger.new($stdout)
+        # Inherit the logger level from Rails, otherwise would always run with the debug level
+        logger.level = Rails.logger.level
+
+        Rails.logger.extend(
+          ActiveSupport::Logger.broadcast(
+            logger
+          )
+        )
+      end
+
+      initializer 'karafka.configure_rails_auto_load_paths' do |app|
+        # Consumers should autoload by default in the Rails app so they are visible
+        app.config.autoload_paths += %w[app/consumers]
+      end
+
+      initializer 'karafka.configure_rails_code_reloader' do
+        # There are components that won't work with older Rails version, so we check it and
+        # provide a failover
+        rails6plus = Rails.gem_version >= Gem::Version.new('6.0.0')
+
+        next unless Rails.env.development?
+        next unless ENV.key?('KARAFKA_CLI')
+        next unless rails6plus
+
+        # We can have many listeners, but it does not matter in which we will reload the code
+        # as long as all the consumers will be re-created as Rails reload is thread-safe
+        ::Karafka::App.monitor.subscribe('connection.listener.fetch_loop') do
+          # Reload code each time there is a change in the code
+          next unless Rails.application.reloaders.any?(&:updated?)
+
+          Rails.application.reloader.reload!
+        end
+
+        ::Karafka::App.monitor.subscribe('worker.completed') do
+          # Skip in case someone is using Rails without ActiveRecord
+          next unless Object.const_defined?('ActiveRecord::Base')
+
+          # Always release the connection after processing is done. Otherwise thread may hang
+          # blocking the reload and further processing
+          # @see https://github.com/rails/rails/issues/44183
+          ActiveRecord::Base.connection_pool.release_connection
+        end
+      end
+
+      initializer 'karafka.require_karafka_boot_file' do |app|
+        rails6plus = Rails.gem_version >= Gem::Version.new('6.0.0')
+
+        # If the boot file location is set to "false", we should not raise an exception and we
+        # should just not load karafka stuff. Setting this explicitly to false indicates, that
+        # karafka is part of the supply chain but it is not a first class citizen of a given
+        # system (may be just a dependency of a dependency), thus railtie should not kick in to
+        # load the non-existing boot file
+        next if Karafka.boot_file.to_s == 'false'
+
+        karafka_boot_file = Rails.root.join(Karafka.boot_file.to_s).to_s
+
+        # Provide more comprehensive error for when no boot file
+        unless File.exist?(karafka_boot_file)
+          raise(Karafka::Errors::MissingBootFileError, karafka_boot_file)
+        end
+
+        if rails6plus
+          app.reloader.to_prepare do
+            # Load Karafka boot file, so it can be used in Rails server context
+            require karafka_boot_file
+          end
+        else
+          # Load Karafka main setup for older Rails versions
+          app.config.after_initialize do
+            require karafka_boot_file
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/builder.rb

@@ -10,13 +10,9 @@ module Karafka
     #     end
     #   end
     class Builder < Concurrent::Array
-      # Consumer group consistency checking contract
-      CONTRACT = Karafka::Contracts::ConsumerGroup.new.freeze
-
-      private_constant :CONTRACT
-
       def initialize
         @draws = Concurrent::Array.new
+        super
       end
 
       # Used to draw routes for Karafka
@@ -37,11 +33,7 @@ module Karafka
         instance_eval(&block)
 
         each do |consumer_group|
-          hashed_group = consumer_group.to_h
-          validation_result = CONTRACT.call(hashed_group)
-          next if validation_result.success?
-
-          raise Errors::InvalidConfigurationError, validation_result.errors.to_h
+          Contracts::ConsumerGroup.new.validate!(consumer_group.to_h)
         end
       end
 
@@ -58,30 +50,41 @@ module Karafka
         super
       end
 
-      # Redraws all the routes for the in-process code reloading.
-      # @note This won't allow registration of new topics without process restart but will trigger
-      #   cache invalidation so all the classes, etc are re-fetched after code reload
-      def reload
-        draws = @draws.dup
-        clear
-        draws.each { |block| draw(&block) }
-      end
-
       private
 
       # Builds and saves given consumer group
       # @param group_id [String, Symbol] name for consumer group
       # @param block [Proc] proc that should be executed in the proxy context
       def consumer_group(group_id, &block)
-        consumer_group = ConsumerGroup.new(group_id.to_s)
-        self << Proxy.new(consumer_group, &block).target
+        consumer_group = find { |cg| cg.name == group_id.to_s }
+
+        if consumer_group
+          Proxy.new(consumer_group, &block).target
+        else
+          consumer_group = ConsumerGroup.new(group_id.to_s)
+          self << Proxy.new(consumer_group, &block).target
+        end
+      end
+
+      # Handles the simple routing case where we create one consumer group and allow for further
+      # subscription group customization
+      # @param subscription_group_name [String, Symbol] subscription group id. When not provided,
+      #   a random uuid will be used
+      # @param block [Proc] further topics definitions
+      def subscription_group(subscription_group_name = SecureRandom.uuid, &block)
+        consumer_group('app') do
+          target.public_send(:subscription_group=, subscription_group_name, &block)
+        end
       end
 
+      # In case we use simple style of routing, all topics will be assigned to the same consumer
+      # group that will be based on the client_id
+      #
       # @param topic_name [String, Symbol] name of a topic from which we want to consumer
       # @param block [Proc] proc we want to evaluate in the topic context
       def topic(topic_name, &block)
-        consumer_group(topic_name) do
-          topic(topic_name, &block).tap(&:build)
+        consumer_group('app') do
+          topic(topic_name, &block)
         end
       end
     end