karafka 1.4.12 → 2.2.10

data/lib/karafka/active_job/dispatcher.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Karafka
+  module ActiveJob
+    # Dispatcher that sends the ActiveJob job to a proper topic based on the queue name
+    class Dispatcher
+      # Defaults for dispatching
+      # The can be updated by using `#karafka_options` on the job
+      DEFAULTS = {
+        dispatch_method: :produce_async,
+        dispatch_many_method: :produce_many_async
+      }.freeze
+
+      private_constant :DEFAULTS
+
+      # @param job [ActiveJob::Base] job
+      def dispatch(job)
+        ::Karafka.producer.public_send(
+          fetch_option(job, :dispatch_method, DEFAULTS),
+          topic: job.queue_name,
+          payload: ::ActiveSupport::JSON.encode(serialize_job(job))
+        )
+      end
+
+      # Bulk dispatches multiple jobs using the Rails 7.1+ API
+      # @param jobs [Array<ActiveJob::Base>] jobs we want to dispatch
+      def dispatch_many(jobs)
+        # Group jobs by their desired dispatch method
+        # It can be configured per job class, so we need to make sure we divide them
+        dispatches = Hash.new { |hash, key| hash[key] = [] }
+
+        jobs.each do |job|
+          d_method = fetch_option(job, :dispatch_many_method, DEFAULTS)
+
+          dispatches[d_method] << {
+            topic: job.queue_name,
+            payload: ::ActiveSupport::JSON.encode(serialize_job(job))
+          }
+        end
+
+        dispatches.each do |type, messages|
+          ::Karafka.producer.public_send(
+            type,
+            messages
+          )
+        end
+      end
+
+      private
+
+      # @param job [ActiveJob::Base] job
+      # @param key [Symbol] key we want to fetch
+      # @param defaults [Hash]
+      # @return [Object] options we are interested in
+      def fetch_option(job, key, defaults)
+        job
+          .class
+          .karafka_options
+          .fetch(key, defaults.fetch(key))
+      end
+
+      # @param job [ActiveJob::Base] job
+      # @return [Hash] json representation of the job
+      def serialize_job(job)
+        job.serialize
+      end
+    end
+  end
+end

data/lib/karafka/active_job/job_extensions.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Karafka
+  module ActiveJob
+    # Allows for setting karafka specific options in ActiveJob jobs
+    module JobExtensions
+      class << self
+        # Defines all the needed accessors and sets defaults
+        # @param klass [ActiveJob::Base] active job base
+        def extended(klass)
+          klass.class_attribute :_karafka_options
+          klass._karafka_options = {}
+        end
+      end
+
+      # @param new_options [Hash] additional options that allow for jobs Karafka related options
+      #   customization
+      # @return [Hash] karafka options
+      def karafka_options(new_options = {})
+        return _karafka_options if new_options.empty?
+
+        # Make sure, that karafka options that someone wants to use are valid before assigning
+        # them
+        App.config.internal.active_job.job_options_contract.validate!(new_options)
+
+        new_options.each do |name, value|
+          _karafka_options[name] = value
+        end
+
+        _karafka_options
+      end
+    end
+  end
+end

data/lib/karafka/active_job/job_options_contract.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Karafka
+  module ActiveJob
+    # Contract for validating the options that can be altered with `#karafka_options` per job class
+    # @note We keep this in the `Karafka::ActiveJob` namespace instead of `Karafka::Contracts` as
+    #   we want to keep ActiveJob related Karafka components outside of the core Karafka code and
+    #   all in the same place
+    class JobOptionsContract < Contracts::Base
+      configure do |config|
+        config.error_messages = YAML.safe_load(
+          File.read(
+            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+          )
+        ).fetch('en').fetch('validations').fetch('job_options')
+      end
+
+      optional(:dispatch_method) do |val|
+        %i[
+          produce_async
+          produce_sync
+        ].include?(val)
+      end
+      optional(:dispatch_many_method) do |val|
+        %i[
+          produce_many_async
+          produce_many_sync
+        ].include?(val)
+      end
+    end
+  end
+end

data/lib/karafka/admin.rb

@@ -0,0 +1,313 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Simple admin actions that we can perform via Karafka on our Kafka cluster
+  #
+  # @note It always initializes a new admin instance as we want to ensure it is always closed
+  #   Since admin actions are not performed that often, that should be ok.
+  #
+  # @note It always uses the primary defined cluster and does not support multi-cluster work.
+  #   Cluster on which operations are performed can be changed via `admin.kafka` config, however
+  #   there is no multi-cluster runtime support.
+  module Admin
+    class << self
+      # Allows us to read messages from the topic
+      #
+      # @param name [String, Symbol] topic name
+      # @param partition [Integer] partition
+      # @param count [Integer] how many messages we want to get at most
+      # @param start_offset [Integer, Time] offset from which we should start. If -1 is provided
+      #   (default) we will start from the latest offset. If time is provided, the appropriate
+      #   offset will be resolved. If negative beyond -1 is provided, we move backwards more.
+      # @param settings [Hash] kafka extra settings (optional)
+      #
+      # @return [Array<Karafka::Messages::Message>] array with messages
+      def read_topic(name, partition, count, start_offset = -1, settings = {})
+        messages = []
+        tpl = Rdkafka::Consumer::TopicPartitionList.new
+        low_offset, high_offset = nil
+
+        with_consumer(settings) do |consumer|
+          # Convert the time offset (if needed)
+          start_offset = resolve_offset(consumer, name.to_s, partition, start_offset)
+
+          low_offset, high_offset = consumer.query_watermark_offsets(name, partition)
+
+          # Select offset dynamically if -1 or less and move backwards with the negative
+          # offset, allowing to start from N messages back from high-watermark
+          start_offset = high_offset - count - start_offset.abs + 1 if start_offset.negative?
+          start_offset = low_offset if start_offset.negative?
+
+          # Build the requested range - since first element is on the start offset we need to
+          # subtract one from requested count to end up with expected number of elements
+          requested_range = (start_offset..start_offset + (count - 1))
+          # Establish theoretical available range. Note, that this does not handle cases related to
+          # log retention or compaction
+          available_range = (low_offset..(high_offset - 1))
+          # Select only offset that we can select. This will remove all the potential offsets that
+          # are below the low watermark offset
+          possible_range = requested_range.select { |offset| available_range.include?(offset) }
+
+          start_offset = possible_range.first
+          count = possible_range.count
+
+          tpl.add_topic_and_partitions_with_offsets(name, partition => start_offset)
+          consumer.assign(tpl)
+
+          # We should poll as long as we don't have all the messages that we need or as long as
+          # we do not read all the messages from the topic
+          loop do
+            # If we've got as many messages as we've wanted stop
+            break if messages.size >= count
+
+            message = consumer.poll(200)
+
+            next unless message
+
+            # If the message we've got is beyond the requested range, stop
+            break unless possible_range.include?(message.offset)
+
+            messages << message
+          rescue Rdkafka::RdkafkaError => e
+            # End of partition
+            break if e.code == :partition_eof
+
+            raise e
+          end
+        end
+
+        # Use topic from routes if we can match it or create a dummy one
+        # Dummy one is used in case we cannot match the topic with routes. This can happen
+        # when admin API is used to read topics that are not part of the routing
+        topic = ::Karafka::Routing::Router.find_or_initialize_by_name(name)
+
+        messages.map! do |message|
+          Messages::Builders::Message.call(
+            message,
+            topic,
+            Time.now
+          )
+        end
+      end
+
+      # Creates Kafka topic with given settings
+      #
+      # @param name [String] topic name
+      # @param partitions [Integer] number of partitions we expect
+      # @param replication_factor [Integer] number of replicas
+      # @param topic_config [Hash] topic config details as described here:
+      #   https://kafka.apache.org/documentation/#topicconfigs
+      def create_topic(name, partitions, replication_factor, topic_config = {})
+        with_admin do |admin|
+          handler = admin.create_topic(name, partitions, replication_factor, topic_config)
+
+          with_re_wait(
+            -> { handler.wait(max_wait_timeout: app_config.admin.max_wait_time) },
+            -> { topics_names.include?(name) }
+          )
+        end
+      end
+
+      # Deleted a given topic
+      #
+      # @param name [String] topic name
+      def delete_topic(name)
+        with_admin do |admin|
+          handler = admin.delete_topic(name)
+
+          with_re_wait(
+            -> { handler.wait(max_wait_timeout: app_config.admin.max_wait_time) },
+            -> { !topics_names.include?(name) }
+          )
+        end
+      end
+
+      # Creates more partitions for a given topic
+      #
+      # @param name [String] topic name
+      # @param partitions [Integer] total number of partitions we expect to end up with
+      def create_partitions(name, partitions)
+        with_admin do |admin|
+          handler = admin.create_partitions(name, partitions)
+
+          with_re_wait(
+            -> { handler.wait(max_wait_timeout: app_config.admin.max_wait_time) },
+            -> { topic(name).fetch(:partition_count) >= partitions }
+          )
+        end
+      end
+
+      # Fetches the watermark offsets for a given topic partition
+      #
+      # @param name [String, Symbol] topic name
+      # @param partition [Integer] partition
+      # @return [Array<Integer, Integer>] low watermark offset and high watermark offset
+      def read_watermark_offsets(name, partition)
+        with_consumer do |consumer|
+          # For newly created topics or in cases where we're trying to get them but there is no
+          # leader, this can fail. It happens more often for new topics under KRaft, however we
+          # still want to make sure things operate as expected even then
+          with_rdkafka_retry(codes: %i[not_leader_for_partition]) do
+            consumer.query_watermark_offsets(name, partition)
+          end
+        end
+      end
+
+      # @return [Rdkafka::Metadata] cluster metadata info
+      def cluster_info
+        with_admin do |admin|
+          admin.instance_variable_get('@native_kafka').with_inner do |inner|
+            Rdkafka::Metadata.new(inner)
+          end
+        end
+      end
+
+      # Creates consumer instance and yields it. After usage it closes the consumer instance
+      # This API can be used in other pieces of code and allows for low-level consumer usage
+      #
+      # @param settings [Hash] extra settings to customize consumer
+      #
+      # @note We always ship and yield a proxied consumer because admin API performance is not
+      #   that relevant. That is, there are no high frequency calls that would have to be delegated
+      def with_consumer(settings = {})
+        consumer = config(:consumer, settings).consumer
+        proxy = ::Karafka::Connection::Proxy.new(consumer)
+        yield(proxy)
+      ensure
+        # Always unsubscribe consumer just to be sure, that no metadata requests are running
+        # when we close the consumer. This in theory should prevent from some race-conditions
+        # that originate from librdkafka
+        begin
+          consumer&.unsubscribe
+        # Ignore any errors and continue to close consumer despite them
+        rescue Rdkafka::RdkafkaError
+          nil
+        end
+
+        consumer&.close
+      end
+
+      private
+
+      # @return [Array<String>] topics names
+      def topics_names
+        cluster_info.topics.map { |topic| topic.fetch(:topic_name) }
+      end
+
+      # Finds details about given topic
+      # @param name [String] topic name
+      # @return [Hash] topic details
+      def topic(name)
+        cluster_info.topics.find { |topic| topic[:topic_name] == name }
+      end
+
+      # Creates admin instance and yields it. After usage it closes the admin instance
+      def with_admin
+        admin = config(:producer, {}).admin
+        yield(admin)
+      ensure
+        admin&.close
+      end
+
+      # There are some cases where rdkafka admin operations finish successfully but without the
+      # callback being triggered to materialize the post-promise object. Until this is fixed we
+      # can figure out, that operation we wanted to do finished successfully by checking that the
+      # effect of the command (new topic, more partitions, etc) is handled. Exactly for that we
+      # use the breaker. It we get a timeout, we can check that what we wanted to achieve has
+      # happened via the breaker check, hence we do not need to wait any longer.
+      #
+      # @param handler [Proc] the wait handler operation
+      # @param breaker [Proc] extra condition upon timeout that indicates things were finished ok
+      def with_re_wait(handler, breaker)
+        attempt ||= 0
+        attempt += 1
+
+        handler.call
+
+        # If breaker does not operate, it means that the requested change was applied but is still
+        # not visible and we need to wait
+        raise(Errors::ResultNotVisibleError) unless breaker.call
+      rescue Rdkafka::AbstractHandle::WaitTimeoutError, Errors::ResultNotVisibleError
+        return if breaker.call
+
+        retry if attempt <= app_config.admin.max_attempts
+
+        raise
+      end
+
+      # Handles retries for rdkafka related errors that we specify in `:codes`.
+      #
+      # Some operations temporarily fail, especially for cases where we changed something fast
+      # like topic creation or repartitioning. In cases like this it is ok to retry operations that
+      # do not change the state as it will usually recover.
+      #
+      # @param codes [Array<Symbol>] librdkafka error codes on which we want to retry
+      # @param max_attempts [Integer] number of attempts (including initial) after which we should
+      #   give up
+      #
+      # @note This code implements a simple backoff that increases with each attempt.
+      def with_rdkafka_retry(codes:, max_attempts: 5)
+        attempt ||= 0
+        attempt += 1
+
+        yield
+      rescue Rdkafka::RdkafkaError => e
+        raise unless codes.include?(e.code)
+        raise if attempt >= max_attempts
+
+        sleep(max_attempts)
+
+        retry
+      end
+
+      # @param type [Symbol] type of config we want
+      # @param settings [Hash] extra settings for config (if needed)
+      # @return [::Rdkafka::Config] rdkafka config
+      def config(type, settings)
+        group_id = app_config.consumer_mapper.call(
+          app_config.admin.group_id
+        )
+
+        app_config
+          .kafka
+          .then(&:dup)
+          .merge(app_config.admin.kafka)
+          .merge!(settings)
+          .tap { |config| config[:'group.id'] = group_id }
+          .then { |config| Karafka::Setup::AttributesMap.public_send(type, config) }
+          .then { |config| ::Rdkafka::Config.new(config) }
+      end
+
+      # Resolves the offset if offset is in a time format. Otherwise returns the offset without
+      # resolving.
+      # @param consumer [::Rdkafka::Consumer]
+      # @param name [String, Symbol] expected topic name
+      # @param partition [Integer]
+      # @param offset [Integer, Time]
+      # @return [Integer] expected offset
+      def resolve_offset(consumer, name, partition, offset)
+        if offset.is_a?(Time)
+          tpl = ::Rdkafka::Consumer::TopicPartitionList.new
+          tpl.add_topic_and_partitions_with_offsets(
+            name, partition => offset
+          )
+
+          real_offsets = consumer.offsets_for_times(tpl)
+          detected_offset = real_offsets
+                            .to_h
+                            .fetch(name)
+                            .find { |p_data| p_data.partition == partition }
+
+          detected_offset&.offset || raise(Errors::InvalidTimeBasedOffsetError)
+        else
+          offset
+        end
+      end
+
+      # @return [Karafka::Core::Configurable::Node] root node config
+      def app_config
+        ::Karafka::App.config
+      end
+    end
+  end
+end

data/lib/karafka/app.rb

@@ -6,35 +6,57 @@ module Karafka
     extend Setup::Dsl
 
     class << self
-      # Sets up all the internal components and bootstrap whole app
-      # We need to know details about consumers in order to setup components,
-      # that's why we don't setup them after std setup is done
-      # @raise [Karafka::Errors::InvalidConfigurationError] raised when configuration
-      #   doesn't match with the config contract
-      def boot!
-        initialize!
-        Setup::Config.validate!
-        Setup::Config.setup_components
-        initialized!
+      # @return [Karafka::Routing::Builder] consumers builder instance alias
+      def consumer_groups
+        config
+          .internal
+          .routing
+          .builder
       end
 
-      # @return [Karafka::Routing::Builder] consumers builder instance
-      def consumer_groups
-        config.internal.routing_builder
+      # @return [Hash] active subscription groups grouped based on consumer group in a hash
+      def subscription_groups
+        # We first build all the subscription groups, so they all get the same position, despite
+        # later narrowing that. It allows us to maintain same position number for static members
+        # even when we want to run subset of consumer groups or subscription groups
+        #
+        # We then narrow this to active consumer groups from which we select active subscription
+        # groups.
+        consumer_groups
+          .map { |cg| [cg, cg.subscription_groups] }
+          .select { |cg, _| cg.active? }
+          .select { |_, sgs| sgs.delete_if { |sg| !sg.active? } }
+          .delete_if { |_, sgs| sgs.empty? }
+          .each { |_, sgs| sgs.each { |sg| sg.topics.delete_if { |top| !top.active? } } }
+          .each { |_, sgs| sgs.delete_if { |sg| sg.topics.empty? } }
+          .reject { |cg, _| cg.subscription_groups.empty? }
+          .to_h
       end
 
-      # Triggers reload of all cached Karafka app components, so we can use in-process
-      # in-development hot code reloading without Karafka process restart
-      def reload
-        Karafka::Persistence::Consumers.clear
-        Karafka::Persistence::Topics.clear
-        config.internal.routing_builder.reload
+      # Just a nicer name for the consumer groups
+      alias routes consumer_groups
+
+      # Allow for easier status management via `Karafka::App` by aliasing status methods here
+      Status::STATES.each do |state, transition|
+        class_eval <<~RUBY, __FILE__, __LINE__ + 1
+          def #{state}
+            App.config.internal.status.#{state}
+          end
+
+          def #{state}?
+            App.config.internal.status.#{state}?
+          end
+
+          def #{transition}
+            App.config.internal.status.#{transition}
+          end
+        RUBY
       end
 
-      Status.instance_methods(false).each do |delegated|
-        define_method(delegated) do
-          App.config.internal.status.send(delegated)
-        end
+      # @return [Boolean] true if we should be done in general with processing anything
+      # @note It is a meta status from the status object
+      def done?
+        App.config.internal.status.done?
       end
 
       # Methods that should be delegated to Karafka module
@@ -42,7 +64,9 @@ module Karafka
         root
         env
         logger
+        producer
         monitor
+        pro?
       ].each do |delegated|
         define_method(delegated) do
           Karafka.send(delegated)