karafka 1.4.13 → 2.0.0

data/lib/karafka/processing/worker.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Workers are used to run jobs in separate threads.
+    # Workers are the main processing units of the Karafka framework.
+    #
+    # Each job runs in three stages:
+    #   - prepare - here we can run any code that we would need to run blocking before we allow
+    #               the job to run fully async (non blocking). This will always run in a blocking
+    #               way and can be used to make sure all the resources and external dependencies
+    #               are satisfied before going async.
+    #
+    #   - call - actual processing logic that can run sync or async
+    #
+    #   - teardown - it should include any code that we want to run after we executed the user
+    #                code. This can be used to unlock certain resources or do other things that are
+    #                not user code but need to run after user code base is executed.
+    class Worker
+      include Helpers::Async
+
+      # @return [String] id of this worker
+      attr_reader :id
+
+      # @param jobs_queue [JobsQueue]
+      # @return [Worker]
+      def initialize(jobs_queue)
+        @id = SecureRandom.uuid
+        @jobs_queue = jobs_queue
+      end
+
+      private
+
+      # Runs processing of jobs in a loop
+      # Stops when queue is closed.
+      def call
+        loop { break unless process }
+      end
+
+      # Fetches a single job, processes it and marks as completed.
+      #
+      # @note We do not have error handling here, as no errors should propagate this far. If they
+      #   do, it is a critical error and should bubble up.
+      #
+      # @note Upon closing the jobs queue, worker will close it's thread
+      def process
+        job = @jobs_queue.pop
+
+        if job
+          instrument_details = { caller: self, job: job, jobs_queue: @jobs_queue }
+
+          Karafka.monitor.instrument('worker.process', instrument_details)
+
+          Karafka.monitor.instrument('worker.processed', instrument_details) do
+            job.before_call
+
+            # If a job is marked as non blocking, we can run a tick in the job queue and if there
+            # are no other blocking factors, the job queue will be unlocked.
+            # If this does not run, all the things will be blocking and job queue won't allow to
+            # pass it until done.
+            @jobs_queue.tick(job.group_id) if job.non_blocking?
+
+            job.call
+
+            job.after_call
+
+            true
+          end
+        else
+          false
+        end
+      # We signal critical exceptions, notify and do not allow worker to fail
+      # rubocop:disable Lint/RescueException
+      rescue Exception => e
+        # rubocop:enable Lint/RescueException
+        Karafka.monitor.instrument(
+          'error.occurred',
+          caller: self,
+          error: e,
+          type: 'worker.process.error'
+        )
+      ensure
+        # job can be nil when the queue is being closed
+        @jobs_queue.complete(job) if job
+      end
+    end
+  end
+end

data/lib/karafka/processing/workers_batch.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Abstraction layer around workers batch.
+    class WorkersBatch
+      include Enumerable
+
+      # @param jobs_queue [JobsQueue]
+      # @return [WorkersBatch]
+      def initialize(jobs_queue)
+        @batch = Array.new(App.config.concurrency) { Processing::Worker.new(jobs_queue) }
+      end
+
+      # Iterates over available workers and yields each worker
+      # @param block [Proc] block we want to run
+      def each(&block)
+        @batch.each(&block)
+      end
+
+      # @return [Integer] number of workers in the batch
+      def size
+        @batch.size
+      end
+    end
+  end
+end

data/lib/karafka/railtie.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+# This file contains Railtie for auto-configuration
+
+rails = false
+
+begin
+  require 'rails'
+
+  rails = true
+rescue LoadError
+  # Without defining this in any way, Zeitwerk ain't happy so we do it that way
+  module Karafka
+    class Railtie
+    end
+  end
+end
+
+if rails
+  # Load Karafka
+  require 'karafka'
+
+  # Load ActiveJob adapter
+  require 'active_job/karafka'
+
+  # Setup env if configured (may be configured later by .net, etc)
+  ENV['KARAFKA_ENV'] ||= ENV['RAILS_ENV'] if ENV.key?('RAILS_ENV')
+
+  module Karafka
+    # Railtie for setting up Rails integration
+    class Railtie < Rails::Railtie
+      railtie_name :karafka
+
+      initializer 'karafka.active_job_integration' do
+        ActiveSupport.on_load(:active_job) do
+          # Extend ActiveJob with some Karafka specific ActiveJob magic
+          extend ::Karafka::ActiveJob::JobExtensions
+        end
+      end
+
+      # This lines will make Karafka print to stdout like puma or unicorn when we run karafka
+      # server + will support code reloading with each fetched loop. We do it only for karafka
+      # based commands as Rails processes and console will have it enabled already
+      initializer 'karafka.configure_rails_logger' do
+        # Make Karafka use Rails logger
+        ::Karafka::App.config.logger = Rails.logger
+
+        next unless Rails.env.development?
+        next unless ENV.key?('KARAFKA_CLI')
+
+        Rails.logger.extend(
+          ActiveSupport::Logger.broadcast(
+            ActiveSupport::Logger.new($stdout)
+          )
+        )
+      end
+
+      initializer 'karafka.configure_rails_auto_load_paths' do |app|
+        # Consumers should autoload by default in the Rails app so they are visible
+        app.config.autoload_paths += %w[app/consumers]
+      end
+
+      initializer 'karafka.configure_rails_code_reloader' do
+        # There are components that won't work with older Rails version, so we check it and
+        # provide a failover
+        rails6plus = Rails.gem_version >= Gem::Version.new('6.0.0')
+
+        next unless Rails.env.development?
+        next unless ENV.key?('KARAFKA_CLI')
+        next unless rails6plus
+
+        # We can have many listeners, but it does not matter in which we will reload the code
+        # as long as all the consumers will be re-created as Rails reload is thread-safe
+        ::Karafka::App.monitor.subscribe('connection.listener.fetch_loop') do
+          # Reload code each time there is a change in the code
+          next unless Rails.application.reloaders.any?(&:updated?)
+
+          Rails.application.reloader.reload!
+        end
+      end
+
+      initializer 'karafka.require_karafka_boot_file' do |app|
+        rails6plus = Rails.gem_version >= Gem::Version.new('6.0.0')
+
+        # If the boot file location is set to "false", we should not raise an exception and we
+        # should just not load karafka stuff. Setting this explicitly to false indicates, that
+        # karafka is part of the supply chain but it is not a first class citizen of a given
+        # system (may be just a dependency of a dependency), thus railtie should not kick in to
+        # load the non-existing boot file
+        next if Karafka.boot_file.to_s == 'false'
+
+        karafka_boot_file = Rails.root.join(Karafka.boot_file.to_s).to_s
+
+        # Provide more comprehensive error for when no boot file
+        unless File.exist?(karafka_boot_file)
+          raise(Karafka::Errors::MissingBootFileError, karafka_boot_file)
+        end
+
+        if rails6plus
+          app.reloader.to_prepare do
+            # Load Karafka boot file, so it can be used in Rails server context
+            require karafka_boot_file
+          end
+        else
+          # Load Karafka main setup for older Rails versions
+          app.config.after_initialize do
+            require karafka_boot_file
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/builder.rb

@@ -10,14 +10,9 @@ module Karafka
     #     end
     #   end
     class Builder < Concurrent::Array
-      # Consumer group consistency checking contract
-      CONTRACT = Karafka::Contracts::ConsumerGroup.new.freeze
-
-      private_constant :CONTRACT
-
       def initialize
-        super
         @draws = Concurrent::Array.new
+        super
       end
 
       # Used to draw routes for Karafka
@@ -38,11 +33,7 @@ module Karafka
         instance_eval(&block)
 
         each do |consumer_group|
-          hashed_group = consumer_group.to_h
-          validation_result = CONTRACT.call(hashed_group)
-          next if validation_result.success?
-
-          raise Errors::InvalidConfigurationError, validation_result.errors.to_h
+          Contracts::ConsumerGroup.new.validate!(consumer_group.to_h)
         end
       end
 
@@ -59,30 +50,30 @@ module Karafka
         super
       end
 
-      # Redraws all the routes for the in-process code reloading.
-      # @note This won't allow registration of new topics without process restart but will trigger
-      #   cache invalidation so all the classes, etc are re-fetched after code reload
-      def reload
-        draws = @draws.dup
-        clear
-        draws.each { |block| draw(&block) }
-      end
-
       private
 
       # Builds and saves given consumer group
       # @param group_id [String, Symbol] name for consumer group
       # @param block [Proc] proc that should be executed in the proxy context
       def consumer_group(group_id, &block)
-        consumer_group = ConsumerGroup.new(group_id.to_s)
-        self << Proxy.new(consumer_group, &block).target
+        consumer_group = find { |cg| cg.name == group_id.to_s }
+
+        if consumer_group
+          Proxy.new(consumer_group, &block).target
+        else
+          consumer_group = ConsumerGroup.new(group_id.to_s)
+          self << Proxy.new(consumer_group, &block).target
+        end
       end
 
+      # In case we use simple style of routing, all topics will be assigned to the same consumer
+      # group that will be based on the client_id
+      #
       # @param topic_name [String, Symbol] name of a topic from which we want to consumer
       # @param block [Proc] proc we want to evaluate in the topic context
       def topic(topic_name, &block)
-        consumer_group(topic_name) do
-          topic(topic_name, &block).tap(&:build)
+        consumer_group('app') do
+          topic(topic_name, &block)
         end
       end
     end

data/lib/karafka/routing/consumer_group.rb

@@ -5,14 +5,10 @@ module Karafka
     # Object used to describe a single consumer group that is going to subscribe to
     # given topics
     # It is a part of Karafka's DSL
+    # @note A single consumer group represents Kafka consumer group, but it may not match 1:1 with
+    #   subscription groups. There can be more subscription groups than consumer groups
     class ConsumerGroup
-      extend Helpers::ConfigRetriever
-
-      attr_reader(
-        :topics,
-        :id,
-        :name
-      )
+      attr_reader :id, :topics, :name
 
       # @param name [String, Symbol] raw name of this consumer group. Raw means, that it does not
       #   yet have an application client_id namespace, this will be added here by default.
@@ -21,7 +17,7 @@ module Karafka
       def initialize(name)
         @name = name
         @id = Karafka::App.config.consumer_mapper.call(name)
-        @topics = []
+        @topics = Topics.new([])
       end
 
       # @return [Boolean] true if this consumer group should be active in our current process
@@ -35,28 +31,24 @@ module Karafka
       # @return [Karafka::Routing::Topic] newly built topic instance
       def topic=(name, &block)
         topic = Topic.new(name, self)
-        @topics << Proxy.new(topic, &block).target.tap(&:build)
+        @topics << Proxy.new(topic, &block).target
         @topics.last
       end
 
-      Karafka::AttributesMap.consumer_group.each do |attribute|
-        config_retriever_for(attribute)
+      # @return [Array<Routing::SubscriptionGroup>] all the subscription groups build based on
+      #   the consumer group topics
+      def subscription_groups
+        App.config.internal.routing.subscription_groups_builder.call(topics)
       end
 
       # Hashed version of consumer group that can be used for validation purposes
       # @return [Hash] hash with consumer group attributes including serialized to hash
       # topics inside of it.
       def to_h
-        result = {
+        {
           topics: topics.map(&:to_h),
           id: id
-        }
-
-        Karafka::AttributesMap.consumer_group.each do |attribute|
-          result[attribute] = public_send(attribute)
-        end
-
-        result
+        }.freeze
       end
     end
   end

data/lib/karafka/routing/consumer_mapper.rb

@@ -26,8 +26,7 @@ module Karafka
       # @param raw_consumer_group_name [String, Symbol] string or symbolized consumer group name
       # @return [String] remapped final consumer group name
       def call(raw_consumer_group_name)
-        client_name = Karafka::Helpers::Inflector.map(Karafka::App.config.client_id.to_s)
-        "#{client_name}_#{raw_consumer_group_name}"
+        "#{Karafka::App.config.client_id}_#{raw_consumer_group_name}"
       end
     end
   end

data/lib/karafka/routing/router.rb

@@ -10,7 +10,7 @@ module Karafka
       # Find a proper topic based on full topic id
       # @param topic_id [String] proper topic id (already mapped, etc) for which we want to find
       #   routing topic
-      # @return [Karafka::Routing::Route] proper route details
+      # @return [Karafka::Routing::Topic] proper route details
       # @raise [Karafka::Topic::NonMatchingTopicError] raised if topic name does not match
       #   any route defined by user using routes.draw
       def find(topic_id)

data/lib/karafka/routing/subscription_group.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # Object representing a set of single consumer group topics that can be subscribed together
+    # with one connection.
+    #
+    # @note One subscription group will always belong to one consumer group, but one consumer
+    #   group can have multiple subscription groups.
+    class SubscriptionGroup
+      attr_reader :id, :topics
+
+      # @param topics [Karafka::Routing::Topics] all the topics that share the same key settings
+      # @return [SubscriptionGroup] built subscription group
+      def initialize(topics)
+        @id = SecureRandom.uuid
+        @topics = topics
+        freeze
+      end
+
+      # @return [String] consumer group id
+      def consumer_group_id
+        kafka[:'group.id']
+      end
+
+      # @return [Integer] max messages fetched in a single go
+      def max_messages
+        @topics.first.max_messages
+      end
+
+      # @return [Integer] max milliseconds we can wait for incoming messages
+      def max_wait_time
+        @topics.first.max_wait_time
+      end
+
+      # @return [Hash] kafka settings are a bit special. They are exactly the same for all of the
+      #   topics but they lack the group.id (unless explicitly) provided. To make it compatible
+      #   with our routing engine, we inject it before it will go to the consumer
+      def kafka
+        kafka = @topics.first.kafka.dup
+
+        kafka[:'client.id'] ||= Karafka::App.config.client_id
+        kafka[:'group.id'] ||= @topics.first.consumer_group.id
+        kafka[:'auto.offset.reset'] ||= @topics.first.initial_offset
+        # Karafka manages the offsets based on the processing state, thus we do not rely on the
+        # rdkafka offset auto-storing
+        kafka[:'enable.auto.offset.store'] = false
+        kafka.freeze
+        kafka
+      end
+    end
+  end
+end

data/lib/karafka/routing/subscription_groups_builder.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # rdkafka allows us to group topics subscriptions when they have same settings.
+    # This builder groups topics from a single consumer group into subscription groups that can be
+    # subscribed with one rdkafka connection.
+    # This way we save resources as having several rdkafka consumers under the hood is not the
+    # cheapest thing in a bigger system.
+    #
+    # In general, if we can, we try to subscribe to as many topics with one rdkafka connection as
+    # possible, but if not possible, we divide.
+    class SubscriptionGroupsBuilder
+      # Keys used to build up a hash for subscription groups distribution.
+      # In order to be able to use the same rdkafka connection for several topics, those keys need
+      # to have same values.
+      DISTRIBUTION_KEYS = %i[
+        kafka
+        max_messages
+        max_wait_time
+        initial_offset
+      ].freeze
+
+      private_constant :DISTRIBUTION_KEYS
+
+      # @param topics [Karafka::Routing::Topics] all the topics based on which we want to build
+      #   subscription groups
+      # @return [Array<SubscriptionGroup>] all subscription groups we need in separate threads
+      def call(topics)
+        topics
+          .map { |topic| [checksum(topic), topic] }
+          .group_by(&:first)
+          .values
+          .map { |value| value.map(&:last) }
+          .map { |topics_array| Routing::Topics.new(topics_array) }
+          .map { |grouped_topics| SubscriptionGroup.new(grouped_topics) }
+      end
+
+      private
+
+      # @param topic [Karafka::Routing::Topic] topic for which we compute the grouping checksum
+      # @return [Integer] checksum that we can use to check if topics have the same set of
+      #   settings based on which we group
+      def checksum(topic)
+        accu = {}
+
+        DISTRIBUTION_KEYS.each { |key| accu[key] = topic.public_send(key) }
+
+        accu.hash
+      end
+    end
+  end
+end

data/lib/karafka/routing/topic.rb

@@ -2,17 +2,24 @@
 
 module Karafka
   module Routing
-    # Topic stores all the details on how we should interact with Kafka given topic
+    # Topic stores all the details on how we should interact with Kafka given topic.
     # It belongs to a consumer group as from 0.6 all the topics can work in the same consumer group
-    # It is a part of Karafka's DSL
+    # It is a part of Karafka's DSL.
     class Topic
-      extend Helpers::ConfigRetriever
-      extend Forwardable
+      attr_reader :id, :name, :consumer_group
+      attr_writer :consumer
 
-      attr_reader :id, :consumer_group
-      attr_accessor :consumer
+      # Attributes we can inherit from the root unless they were defined on this level
+      INHERITABLE_ATTRIBUTES = %i[
+        kafka
+        deserializer
+        manual_offset_management
+        max_messages
+        max_wait_time
+        initial_offset
+      ].freeze
 
-      def_delegator :@consumer_group, :batch_fetching
+      private_constant :INHERITABLE_ATTRIBUTES
 
       # @param [String, Symbol] name of a topic on which we want to listen
       # @param consumer_group [Karafka::Routing::ConsumerGroup] owning consumer group of this topic
@@ -22,40 +29,70 @@ module Karafka
         @attributes = {}
         # @note We use identifier related to the consumer group that owns a topic, because from
         #   Karafka 0.6 we can handle multiple Kafka instances with the same process and we can
-        #   have same topic name across multiple Kafkas
+        #   have same topic name across multiple consumer groups
         @id = "#{consumer_group.id}_#{@name}"
       end
 
-      # Initializes default values for all the options that support defaults if their values are
-      # not yet specified. This is need to be done (cannot be lazy loaded on first use) because
-      # everywhere except Karafka server command, those would not be initialized on time - for
-      # example for Sidekiq
-      def build
-        Karafka::AttributesMap.topic.each { |attr| send(attr) }
-        self
+      INHERITABLE_ATTRIBUTES.each do |attribute|
+        attr_writer attribute
+
+        define_method attribute do
+          current_value = instance_variable_get(:"@#{attribute}")
+
+          return current_value unless current_value.nil?
+
+          value = Karafka::App.config.send(attribute)
+
+          instance_variable_set(:"@#{attribute}", value)
+        end
+      end
+
+      # @return [Class] consumer class that we should use
+      def consumer
+        if Karafka::App.config.consumer_persistence
+          # When persistence of consumers is on, no need to reload them
+          @consumer
+        else
+          # In order to support code reload without having to change the topic api, we re-fetch the
+          # class of a consumer based on its class name. This will support all the cases where the
+          # consumer class is defined with a name. It won't support code reload for anonymous
+          # consumer classes, but this is an edge case
+          begin
+            ::Object.const_get(@consumer.to_s)
+          rescue NameError
+            # It will only fail if the in case of anonymous classes
+            @consumer
+          end
+        end
       end
 
-      # @return [Class, nil] Class (not an instance) of a responder that should respond from
-      #   consumer back to Kafka (useful for piping data flows)
-      def responder
-        @responder ||= Karafka::Responders::Builder.new(consumer).build
+      # @return [Class] consumer class that we should use
+      # @note This is just an alias to the `#consumer` method. We however want to use it internally
+      #   instead of referencing the `#consumer`. We use this to indicate that this method returns
+      #   class and not an instance. In the routing we want to keep the `#consumer Consumer`
+      #   routing syntax, but for references outside, we should use this one.
+      def consumer_class
+        consumer
       end
 
-      Karafka::AttributesMap.topic.each do |attribute|
-        config_retriever_for(attribute)
+      # @return [Boolean] true if this topic offset is handled by the end user
+      def manual_offset_management?
+        manual_offset_management
       end
 
       # @return [Hash] hash with all the topic attributes
       # @note This is being used when we validate the consumer_group and its topics
       def to_h
-        map = Karafka::AttributesMap.topic.map do |attribute|
+        map = INHERITABLE_ATTRIBUTES.map do |attribute|
           [attribute, public_send(attribute)]
         end
 
         Hash[map].merge!(
           id: id,
-          consumer: consumer
-        )
+          name: name,
+          consumer: consumer,
+          consumer_group_id: consumer_group.id
+        ).freeze
       end
     end
   end

data/lib/karafka/routing/topics.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # Abstraction layer on top of groups of topics
+    class Topics
+      include Enumerable
+      extend Forwardable
+
+      def_delegators :@accumulator, :[], :size, :empty?, :last, :<<
+
+      # @param topics_array [Array<Karafka::Routing::Topic>] array with topics
+      def initialize(topics_array)
+        @accumulator = topics_array.dup
+      end
+
+      # Yields each topic
+      #
+      # @param [Proc] block we want to yield with on each topic
+      def each(&block)
+        @accumulator.each(&block)
+      end
+
+      # Finds topic by its name
+      #
+      # @param topic_name [String] topic name
+      # @return [Karafka::Routing::Topic]
+      # @raise [Karafka::Errors::TopicNotFoundError] this should never happen. If you see it,
+      #   please create an issue.
+      def find(topic_name)
+        @accumulator.find { |topic| topic.name == topic_name } ||
+          raise(Karafka::Errors::TopicNotFoundError, topic_name)
+      end
+    end
+  end
+end