karafka 1.4.15 → 2.0.0.alpha1

data/lib/karafka/processing/executor.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace that encapsulates all the logic related to processing data.
+  module Processing
+    # Executors:
+    # - run consumers code with provided messages batch (for `#call`) or run given teardown
+    #   operations when needed from separate threads.
+    # - they re-create consumer instances in case of partitions that were revoked
+    #   and assigned back.
+    #
+    # @note Executors are not removed after partition is revoked. They are not that big and will
+    #   be re-used in case of a re-claim
+    class Executor
+      # @return [String] unique id that we use to ensure, that we use for state tracking
+      attr_reader :id
+
+      # @return [String] subscription group id to which a given executor belongs
+      attr_reader :group_id
+
+      # @param group_id [String] id of the subscription group to which the executor belongs
+      # @param client [Karafka::Connection::Client] kafka client
+      # @param topic [Karafka::Routing::Topic] topic for which this executor will run
+      # @param pause [Karafka::TimeTrackers::Pause] fetch pause object for crash pausing
+      def initialize(group_id, client, topic, pause)
+        @id = SecureRandom.uuid
+        @group_id = group_id
+        @client = client
+        @topic = topic
+        @pause = pause
+      end
+
+      # Runs consumer data processing against given batch and handles failures and errors.
+      #
+      # @param messages [Array<Rdkafka::Consumer::Message>] raw rdkafka messages
+      # @param received_at [Time] the moment we've received the batch (actually the moment we've)
+      #   enqueued it, but good enough
+      def consume(messages, received_at)
+        # Recreate consumer with each batch if persistence is not enabled
+        # We reload the consumers with each batch instead of relying on some external signals
+        # when needed for consistency. That way devs may have it on or off and not in this
+        # middle state, where re-creation of a consumer instance would occur only sometimes
+        @consumer = nil unless ::Karafka::App.config.consumer_persistence
+
+        # First we build messages batch...
+        consumer.messages = Messages::Builders::Messages.call(
+          messages,
+          @topic,
+          received_at
+        )
+
+        # We run the consumer client logic...
+        consumer.on_consume
+      end
+
+      # Runs the controller `#revoked` method that should be triggered when a given consumer is
+      # no longer needed due to partitions reassignment.
+      #
+      # @note Clearing the consumer will ensure, that if we get the partition back, it will be
+      #   handled with a consumer with a clean state.
+      #
+      # @note We run it only when consumer was present, because presence indicates, that at least
+      #   a single message has been consumed.
+      def revoked
+        consumer.on_revoked if @consumer
+        @consumer = nil
+      end
+
+      # Runs the controller `#shutdown` method that should be triggered when a given consumer is
+      # no longer needed as we're closing the process.
+      #
+      # @note While we do not need to clear the consumer here, it's a good habit to clean after
+      #   work is done.
+      def shutdown
+        # There is a case, where the consumer no longer exists because it was revoked, in case like
+        # that we do not build a new instance and shutdown should not be triggered.
+        consumer.on_shutdown if @consumer
+        @consumer = nil
+      end
+
+      private
+
+      # @return [Object] cached consumer instance
+      def consumer
+        @consumer ||= begin
+          consumer = @topic.consumer.new
+          consumer.topic = @topic
+          consumer.client = @client
+          consumer.pause = @pause
+          consumer.producer = ::Karafka::App.producer
+          consumer
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/executors_buffer.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Buffer for executors of a given subscription group. It wraps around the concept of building
+    # and caching them, so we can re-use them instead of creating new each time.
+    class ExecutorsBuffer
+      # @param client [Connection::Client]
+      # @param subscription_group [Routing::SubscriptionGroup]
+      # @return [ExecutorsBuffer]
+      def initialize(client, subscription_group)
+        @subscription_group = subscription_group
+        @client = client
+        @buffer = Hash.new { |h, k| h[k] = {} }
+      end
+
+      # @param topic [String] topic name
+      # @param partition [Integer] partition number
+      # @param pause [TimeTrackers::Pause] pause corresponding with provided topic and partition
+      # @return [Executor] consumer executor
+      def fetch(
+        topic,
+        partition,
+        pause
+      )
+        topic = @subscription_group.topics.find { |ktopic| ktopic.name == topic }
+
+        topic || raise(Errors::TopicNotFoundError, topic)
+
+        @buffer[topic][partition] ||= Executor.new(
+          @subscription_group.id,
+          @client,
+          topic,
+          pause
+        )
+      end
+
+      # Runs the shutdown on all active executors.
+      def shutdown
+        @buffer.values.map(&:values).flatten.each(&:shutdown)
+      end
+
+      # Clears the executors buffer. Useful for critical errors recovery.
+      def clear
+        @buffer.clear
+      end
+    end
+  end
+end

data/lib/karafka/processing/jobs/base.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Namespace for all the jobs that are suppose to run in workers.
+    module Jobs
+      # Base class for all the jobs types that are suppose to run in workers threads.
+      class Base
+        extend Forwardable
+
+        # @note Since one job has always one executer, we use the jobs id and group id as reference
+        def_delegators :executor, :id, :group_id
+
+        attr_reader :executor
+      end
+    end
+  end
+end

data/lib/karafka/processing/jobs/consume.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Jobs
+      # The main job type. It runs the executor that triggers given topic partition messages
+      # processing in an underlying consumer instance.
+      class Consume < Base
+        # @param executor [Karafka::Processing::Executor] executor that is suppose to run a given
+        #   job
+        # @param messages [Array<dkafka::Consumer::Message>] array with raw rdkafka messages with
+        #   which we are suppose to work
+        # @return [Consume]
+        def initialize(executor, messages)
+          @executor = executor
+          @messages = messages
+          @created_at = Time.now
+          super()
+        end
+
+        # Runs the given executor.
+        def call
+          executor.consume(@messages, @created_at)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/jobs/revoked.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Jobs
+      # Job that runs the revoked operation when we loose a partition on a consumer that lost it.
+      class Revoked < Base
+        # @param executor [Karafka::Processing::Executor] executor that is suppose to run the job
+        # @return [Revoked]
+        def initialize(executor)
+          @executor = executor
+          super()
+        end
+
+        # Runs the revoking job via an executor.
+        def call
+          executor.revoked
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/jobs/shutdown.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Jobs
+      # Job that runs on each active consumer upon process shutdown (one job per consumer).
+      class Shutdown < Base
+        # @param executor [Karafka::Processing::Executor] executor that is suppose to run a given
+        #   job on an active consumer
+        # @return [Shutdown]
+        def initialize(executor)
+          @executor = executor
+          super()
+        end
+
+        # Runs the shutdown job via an executor.
+        def call
+          executor.shutdown
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/jobs_queue.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # This is the key work component for Karafka jobs distribution. It provides API for running
+    # jobs in parallel while operating within more than one subscription group.
+    #
+    # We need to take into consideration fact, that more than one subscription group can operate
+    # on this queue, that's why internally we keep track of processing per group.
+    #
+    # We work with the assumption, that partitions data is evenly distributed.
+    class JobsQueue
+      # @return [Karafka::Processing::JobsQueue]
+      def initialize
+        @queue = ::Queue.new
+        # Those queues will act as a semaphores internally. Since we need an indicator for waiting
+        # we could use Thread.pass but this is expensive. Instead we can just lock until any
+        # of the workers finishes their work and we can re-check. This means that in the worse
+        # scenario, we will context switch 10 times per poll instead of getting this thread
+        # scheduled by Ruby hundreds of thousands of times per group.
+        # We cannot use a single semaphore as it could potentially block in listeners that should
+        # process with their data and also could unlock when a given group needs to remain locked
+        @semaphores = Hash.new { |h, k| h[k] = Queue.new }
+        @in_processing = Hash.new { |h, k| h[k] = {} }
+        @mutex = Mutex.new
+      end
+
+      # Returns number of jobs that are either enqueued or in processing (but not finished)
+      # @return [Integer] number of elements in the queue
+      # @note Using `#pop` won't decrease this number as only marking job as completed does this
+      def size
+        @in_processing.values.map(&:size).sum
+      end
+
+      # Adds the job to the internal main queue, scheduling it for execution in a worker and marks
+      # this job as in processing pipeline.
+      #
+      # @param job [Jobs::Base] job that we want to run
+      def <<(job)
+        # We do not push the job if the queue is closed as it means that it would anyhow not be
+        # executed
+        return if @queue.closed?
+
+        @mutex.synchronize do
+          group = @in_processing[job.group_id]
+
+          raise(Errors::JobsQueueSynchronizationError, job.group_id) if group.key?(job.id)
+
+          group[job.id] = true
+        end
+
+        @queue << job
+      end
+
+      # @return [Jobs::Base, nil] waits for a job from the main queue and returns it once available
+      #   or returns nil if the queue has been stopped and there won't be anything more to process
+      #   ever.
+      # @note This command is blocking and will wait until any job is available on the main queue
+      def pop
+        @queue.pop
+      end
+
+      # Marks a given job from a given group as completed. When there are no more jobs from a given
+      # group to be executed, we won't wait.
+      #
+      # @param [Jobs::Base] job that was completed
+      def complete(job)
+        @mutex.synchronize do
+          @in_processing[job.group_id].delete(job.id)
+          @semaphores[job.group_id] << true
+        end
+      end
+
+      # Clears the processing states for a provided group. Useful when a recovery happens and we
+      # need to clean up state but only for a given subscription group.
+      #
+      # @param group_id [String]
+      def clear(group_id)
+        @mutex.synchronize do
+          @in_processing[group_id].clear
+          # We unlock it just in case it was blocked when clearing started
+          @semaphores[group_id] << true
+        end
+      end
+
+      # Stops the whole processing queue.
+      def close
+        @mutex.synchronize do
+          return if @queue.closed?
+
+          @queue.close
+          @semaphores.values.each(&:close)
+        end
+      end
+
+      # Blocks when there are things in the queue in a given group and waits until all the jobs
+      #   from a given group are completed
+      # @param group_id [String] id of the group in which jobs we're interested.
+      # @note This method is blocking.
+      def wait(group_id)
+        # Go doing other things while we cannot process and wait for anyone to finish their work
+        # and re-check the wait status
+        @semaphores[group_id].pop while wait?(group_id)
+      end
+
+      private
+
+      # @param group_id [String] id of the group in which jobs we're interested.
+      # @return [Boolean] should we keep waiting or not
+      def wait?(group_id)
+        # If it is stopping, all the previous messages that are processed at the moment need to
+        # finish. Otherwise we may risk closing the client and committing offsets afterwards
+        return false if Karafka::App.stopping? && @in_processing[group_id].empty?
+        return false if @queue.closed?
+        return false if @in_processing[group_id].empty?
+
+        true
+      end
+    end
+  end
+end

data/lib/karafka/processing/worker.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Workers are used to run jobs in separate threads.
+    # Workers are the main processing units of the Karafka framework.
+    class Worker
+      extend Forwardable
+
+      def_delegators :@thread, :join, :terminate, :alive?
+
+      # @param jobs_queue [JobsQueue]
+      # @return [Worker]
+      def initialize(jobs_queue)
+        @jobs_queue = jobs_queue
+        @thread = Thread.new do
+          # If anything goes wrong in this worker thread, it means something went really wrong and
+          # we should terminate.
+          Thread.current.abort_on_exception = true
+          loop { break unless process }
+        end
+      end
+
+      private
+
+      # Fetches a single job, processes it and marks as completed.
+      #
+      # @note We do not have error handling here, as no errors should propagate this far. If they
+      #   do, it is a critical error and should bubble up.
+      #
+      # @note Upon closing the jobs queue, worker will close it's thread
+      def process
+        job = @jobs_queue.pop
+
+        if job
+          job.call
+          true
+        else
+          false
+        end
+      # We signal critical exceptions, notify and do not allow worker to fail
+      # rubocop:disable Lint/RescueException
+      rescue Exception => e
+      # rubocop:enable Lint/RescueException
+        Karafka.monitor.instrument(
+          'error.occurred',
+          caller: self,
+          error: e,
+          type: 'worker.process.error'
+        )
+      ensure
+        # job can be nil when the queue is being closed
+        @jobs_queue.complete(job) if job
+      end
+    end
+  end
+end

data/lib/karafka/processing/workers_batch.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Abstraction layer around workers batch.
+    class WorkersBatch
+      include Enumerable
+
+      # @param jobs_queue [JobsQueue]
+      # @return [WorkersBatch]
+      def initialize(jobs_queue)
+        @batch = Array.new(App.config.concurrency) { Processing::Worker.new(jobs_queue) }
+      end
+
+      # Iterates over available workers and yields each worker
+      # @param block [Proc] block we want to run
+      def each(&block)
+        @batch.each(&block)
+      end
+    end
+  end
+end

data/lib/karafka/railtie.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+# This file contains Railtie for auto-configuration
+
+rails = false
+
+begin
+  require 'rails'
+
+  rails = true
+rescue LoadError
+  # Without defining this in any way, Zeitwerk ain't happy so we do it that way
+  module Karafka
+    class Railtie
+    end
+  end
+end
+
+if rails
+  # Load Karafka
+  require 'karafka'
+  # Load ActiveJob adapter
+  require 'active_job/karafka'
+
+  # Setup env if configured (may be configured later by .net, etc)
+  ENV['KARAFKA_ENV'] ||= ENV['RAILS_ENV'] if ENV.key?('RAILS_ENV')
+
+  module Karafka
+    # Railtie for setting up Rails integration
+    class Railtie < Rails::Railtie
+      railtie_name :karafka
+
+      initializer 'karafka.configure_rails_initialization' do |app|
+        # Consumers should autoload by default in the Rails app so they are visible
+        app.config.autoload_paths += %w[app/consumers]
+
+        # Make Karafka use Rails logger
+        ::Karafka::App.config.logger = Rails.logger
+
+        # This lines will make Karafka print to stdout like puma or unicorn
+        if Rails.env.development?
+          Rails.logger.extend(
+            ActiveSupport::Logger.broadcast(
+              ActiveSupport::Logger.new($stdout)
+            )
+          )
+
+          # We can have many listeners, but it does not matter in which we will reload the code as
+          # long as all the consumers will be re-created as Rails reload is thread-safe
+          ::Karafka::App.monitor.subscribe('connection.listener.fetch_loop') do
+            # Reload code each time there is a change in the code
+            next unless Rails.application.reloaders.any?(&:updated?)
+
+            Rails.application.reloader.reload!
+          end
+        end
+
+        app.reloader.to_prepare do
+          # Load Karafka bot file, so it can be used in Rails server context
+          require Rails.root.join(Karafka.boot_file.to_s).to_s
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/builder.rb

@@ -16,8 +16,8 @@ module Karafka
       private_constant :CONTRACT
 
       def initialize
-        super
         @draws = Concurrent::Array.new
+        super
       end
 
       # Used to draw routes for Karafka
@@ -40,6 +40,7 @@ module Karafka
         each do |consumer_group|
           hashed_group = consumer_group.to_h
           validation_result = CONTRACT.call(hashed_group)
+
           next if validation_result.success?
 
           raise Errors::InvalidConfigurationError, validation_result.errors.to_h
@@ -59,30 +60,30 @@ module Karafka
         super
       end
 
-      # Redraws all the routes for the in-process code reloading.
-      # @note This won't allow registration of new topics without process restart but will trigger
-      #   cache invalidation so all the classes, etc are re-fetched after code reload
-      def reload
-        draws = @draws.dup
-        clear
-        draws.each { |block| draw(&block) }
-      end
-
       private
 
       # Builds and saves given consumer group
       # @param group_id [String, Symbol] name for consumer group
       # @param block [Proc] proc that should be executed in the proxy context
       def consumer_group(group_id, &block)
-        consumer_group = ConsumerGroup.new(group_id.to_s)
-        self << Proxy.new(consumer_group, &block).target
+        consumer_group = find { |cg| cg.name == group_id.to_s }
+
+        if consumer_group
+          Proxy.new(consumer_group, &block).target
+        else
+          consumer_group = ConsumerGroup.new(group_id.to_s)
+          self << Proxy.new(consumer_group, &block).target
+        end
       end
 
+      # In case we use simple style of routing, all topics will be assigned to the same consumer
+      # group that will be based on the client_id
+      #
       # @param topic_name [String, Symbol] name of a topic from which we want to consumer
       # @param block [Proc] proc we want to evaluate in the topic context
       def topic(topic_name, &block)
-        consumer_group(topic_name) do
-          topic(topic_name, &block).tap(&:build)
+        consumer_group('app') do
+          topic(topic_name, &block)
         end
       end
     end

data/lib/karafka/routing/consumer_group.rb

@@ -5,14 +5,10 @@ module Karafka
     # Object used to describe a single consumer group that is going to subscribe to
     # given topics
     # It is a part of Karafka's DSL
+    # @note A single consumer group represents Kafka consumer group, but it may not match 1:1 with
+    #   subscription groups. There can be more subscription groups than consumer groups
     class ConsumerGroup
-      extend Helpers::ConfigRetriever
-
-      attr_reader(
-        :topics,
-        :id,
-        :name
-      )
+      attr_reader :id, :topics, :name
 
       # @param name [String, Symbol] raw name of this consumer group. Raw means, that it does not
       #   yet have an application client_id namespace, this will be added here by default.
@@ -35,28 +31,24 @@ module Karafka
       # @return [Karafka::Routing::Topic] newly built topic instance
       def topic=(name, &block)
         topic = Topic.new(name, self)
-        @topics << Proxy.new(topic, &block).target.tap(&:build)
+        @topics << Proxy.new(topic, &block).target
         @topics.last
       end
 
-      Karafka::AttributesMap.consumer_group.each do |attribute|
-        config_retriever_for(attribute)
+      # @return [Array<Routing::SubscriptionGroup>] all the subscription groups build based on
+      #   the consumer group topics
+      def subscription_groups
+        App.config.internal.subscription_groups_builder.call(topics)
       end
 
       # Hashed version of consumer group that can be used for validation purposes
       # @return [Hash] hash with consumer group attributes including serialized to hash
       # topics inside of it.
       def to_h
-        result = {
+        {
           topics: topics.map(&:to_h),
           id: id
-        }
-
-        Karafka::AttributesMap.consumer_group.each do |attribute|
-          result[attribute] = public_send(attribute)
-        end
-
-        result
+        }.freeze
       end
     end
   end

data/lib/karafka/routing/consumer_mapper.rb

@@ -26,8 +26,7 @@ module Karafka
       # @param raw_consumer_group_name [String, Symbol] string or symbolized consumer group name
       # @return [String] remapped final consumer group name
       def call(raw_consumer_group_name)
-        client_name = Karafka::Helpers::Inflector.map(Karafka::App.config.client_id.to_s)
-        "#{client_name}_#{raw_consumer_group_name}"
+        "#{Karafka::App.config.client_id}_#{raw_consumer_group_name}"
       end
     end
   end

data/lib/karafka/routing/router.rb

@@ -10,7 +10,7 @@ module Karafka
       # Find a proper topic based on full topic id
       # @param topic_id [String] proper topic id (already mapped, etc) for which we want to find
       #   routing topic
-      # @return [Karafka::Routing::Route] proper route details
+      # @return [Karafka::Routing::Topic] proper route details
       # @raise [Karafka::Topic::NonMatchingTopicError] raised if topic name does not match
       #   any route defined by user using routes.draw
       def find(topic_id)