karafka 1.4.4 → 2.1.10

data/lib/karafka/admin.rb

@@ -0,0 +1,286 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Simple admin actions that we can perform via Karafka on our Kafka cluster
+  #
+  # @note It always initializes a new admin instance as we want to ensure it is always closed
+  #   Since admin actions are not performed that often, that should be ok.
+  #
+  # @note It always uses the primary defined cluster and does not support multi-cluster work.
+  #   If you need this, just replace the cluster info for the time you use this
+  module Admin
+    # We wait only for this amount of time before raising error as we intercept this error and
+    # retry after checking that the operation was finished or failed using external factor.
+    MAX_WAIT_TIMEOUT = 1
+
+    # How many times should be try. 1 x 60 => 60 seconds wait in total
+    MAX_ATTEMPTS = 60
+
+    # Defaults for config
+    CONFIG_DEFAULTS = {
+      'group.id': 'karafka_admin',
+      # We want to know when there is no more data not to end up with an endless loop
+      'enable.partition.eof': true,
+      'statistics.interval.ms': 0,
+      # Fetch at most 5 MBs when using admin
+      'fetch.message.max.bytes': 5 * 1_048_576,
+      # Do not commit offset automatically, this prevents offset tracking for operations involving
+      # a consumer instance
+      'enable.auto.commit': false,
+      # Make sure that topic metadata lookups do not create topics accidentally
+      'allow.auto.create.topics': false
+    }.freeze
+
+    private_constant :CONFIG_DEFAULTS, :MAX_WAIT_TIMEOUT, :MAX_ATTEMPTS
+
+    class << self
+      # Allows us to read messages from the topic
+      #
+      # @param name [String, Symbol] topic name
+      # @param partition [Integer] partition
+      # @param count [Integer] how many messages we want to get at most
+      # @param start_offset [Integer, Time] offset from which we should start. If -1 is provided
+      #   (default) we will start from the latest offset. If time is provided, the appropriate
+      #   offset will be resolved.
+      # @param settings [Hash] kafka extra settings (optional)
+      #
+      # @return [Array<Karafka::Messages::Message>] array with messages
+      def read_topic(name, partition, count, start_offset = -1, settings = {})
+        messages = []
+        tpl = Rdkafka::Consumer::TopicPartitionList.new
+        low_offset, high_offset = nil
+
+        with_consumer(settings) do |consumer|
+          # Convert the time offset (if needed)
+          start_offset = resolve_offset(consumer, name.to_s, partition, start_offset)
+
+          low_offset, high_offset = consumer.query_watermark_offsets(name, partition)
+
+          # Select offset dynamically if -1 or less
+          start_offset = high_offset - count if start_offset.negative?
+
+          # Build the requested range - since first element is on the start offset we need to
+          # subtract one from requested count to end up with expected number of elements
+          requested_range = (start_offset..start_offset + (count - 1))
+          # Establish theoretical available range. Note, that this does not handle cases related to
+          # log retention or compaction
+          available_range = (low_offset..(high_offset - 1))
+          # Select only offset that we can select. This will remove all the potential offsets that
+          # are below the low watermark offset
+          possible_range = requested_range.select { |offset| available_range.include?(offset) }
+
+          start_offset = possible_range.first
+          count = possible_range.count
+
+          tpl.add_topic_and_partitions_with_offsets(name, partition => start_offset)
+          consumer.assign(tpl)
+
+          # We should poll as long as we don't have all the messages that we need or as long as
+          # we do not read all the messages from the topic
+          loop do
+            # If we've got as many messages as we've wanted stop
+            break if messages.size >= count
+
+            message = consumer.poll(200)
+
+            next unless message
+
+            # If the message we've got is beyond the requested range, stop
+            break unless possible_range.include?(message.offset)
+
+            messages << message
+          rescue Rdkafka::RdkafkaError => e
+            # End of partition
+            break if e.code == :partition_eof
+
+            raise e
+          end
+        end
+
+        # Use topic from routes if we can match it or create a dummy one
+        # Dummy one is used in case we cannot match the topic with routes. This can happen
+        # when admin API is used to read topics that are not part of the routing
+        topic = ::Karafka::Routing::Router.find_or_initialize_by_name(name)
+
+        messages.map! do |message|
+          Messages::Builders::Message.call(
+            message,
+            topic,
+            Time.now
+          )
+        end
+      end
+
+      # Creates Kafka topic with given settings
+      #
+      # @param name [String] topic name
+      # @param partitions [Integer] number of partitions we expect
+      # @param replication_factor [Integer] number of replicas
+      # @param topic_config [Hash] topic config details as described here:
+      #   https://kafka.apache.org/documentation/#topicconfigs
+      def create_topic(name, partitions, replication_factor, topic_config = {})
+        with_admin do |admin|
+          handler = admin.create_topic(name, partitions, replication_factor, topic_config)
+
+          with_re_wait(
+            -> { handler.wait(max_wait_timeout: MAX_WAIT_TIMEOUT) },
+            -> { topics_names.include?(name) }
+          )
+        end
+      end
+
+      # Deleted a given topic
+      #
+      # @param name [String] topic name
+      def delete_topic(name)
+        with_admin do |admin|
+          handler = admin.delete_topic(name)
+
+          with_re_wait(
+            -> { handler.wait(max_wait_timeout: MAX_WAIT_TIMEOUT) },
+            -> { !topics_names.include?(name) }
+          )
+        end
+      end
+
+      # Creates more partitions for a given topic
+      #
+      # @param name [String] topic name
+      # @param partitions [Integer] total number of partitions we expect to end up with
+      def create_partitions(name, partitions)
+        with_admin do |admin|
+          handler = admin.create_partitions(name, partitions)
+
+          with_re_wait(
+            -> { handler.wait(max_wait_timeout: MAX_WAIT_TIMEOUT) },
+            -> { topic(name).fetch(:partition_count) >= partitions }
+          )
+        end
+      end
+
+      # Fetches the watermark offsets for a given topic partition
+      #
+      # @param name [String, Symbol] topic name
+      # @param partition [Integer] partition
+      # @return [Array<Integer, Integer>] low watermark offset and high watermark offset
+      def read_watermark_offsets(name, partition)
+        with_consumer do |consumer|
+          consumer.query_watermark_offsets(name, partition)
+        end
+      end
+
+      # @return [Rdkafka::Metadata] cluster metadata info
+      def cluster_info
+        with_admin do |admin|
+          admin.instance_variable_get('@native_kafka').with_inner do |inner|
+            Rdkafka::Metadata.new(inner)
+          end
+        end
+      end
+
+      # Creates consumer instance and yields it. After usage it closes the consumer instance
+      # This API can be used in other pieces of code and allows for low-level consumer usage
+      #
+      # @param settings [Hash] extra settings to customize consumer
+      #
+      # @note We always ship and yield a proxied consumer because admin API performance is not
+      #   that relevant. That is, there are no high frequency calls that would have to be delegated
+      def with_consumer(settings = {})
+        consumer = config(:consumer, settings).consumer
+        proxy = ::Karafka::Connection::Proxy.new(consumer)
+        yield(proxy)
+      ensure
+        # Always unsubscribe consumer just to be sure, that no metadata requests are running
+        # when we close the consumer. This in theory should prevent from some race-conditions
+        # that originate from librdkafka
+        begin
+          consumer&.unsubscribe
+        # Ignore any errors and continue to close consumer despite them
+        rescue Rdkafka::RdkafkaError
+          nil
+        end
+
+        consumer&.close
+      end
+
+      private
+
+      # @return [Array<String>] topics names
+      def topics_names
+        cluster_info.topics.map { |topic| topic.fetch(:topic_name) }
+      end
+
+      # Finds details about given topic
+      # @param name [String] topic name
+      # @return [Hash] topic details
+      def topic(name)
+        cluster_info.topics.find { |topic| topic[:topic_name] == name }
+      end
+
+      # Creates admin instance and yields it. After usage it closes the admin instance
+      def with_admin
+        admin = config(:producer, {}).admin
+        yield(admin)
+      ensure
+        admin&.close
+      end
+
+      # There are some cases where rdkafka admin operations finish successfully but without the
+      # callback being triggered to materialize the post-promise object. Until this is fixed we
+      # can figure out, that operation we wanted to do finished successfully by checking that the
+      # effect of the command (new topic, more partitions, etc) is handled. Exactly for that we
+      # use the breaker. It we get a timeout, we can check that what we wanted to achieve has
+      # happened via the breaker check, hence we do not need to wait any longer.
+      #
+      # @param handler [Proc] the wait handler operation
+      # @param breaker [Proc] extra condition upon timeout that indicates things were finished ok
+      def with_re_wait(handler, breaker)
+        attempt ||= 0
+        attempt += 1
+
+        handler.call
+      rescue Rdkafka::AbstractHandle::WaitTimeoutError
+        return if breaker.call
+
+        retry if attempt <= MAX_ATTEMPTS
+
+        raise
+      end
+
+      # @param type [Symbol] type of config we want
+      # @param settings [Hash] extra settings for config (if needed)
+      # @return [::Rdkafka::Config] rdkafka config
+      def config(type, settings)
+        config_hash = Karafka::Setup::AttributesMap.public_send(
+          type,
+          Karafka::App.config.kafka.dup.merge(CONFIG_DEFAULTS).merge!(settings)
+        )
+
+        ::Rdkafka::Config.new(config_hash)
+      end
+
+      # Resolves the offset if offset is in a time format. Otherwise returns the offset without
+      # resolving.
+      # @param consumer [::Rdkafka::Consumer]
+      # @param name [String, Symbol] expected topic name
+      # @param partition [Integer]
+      # @param offset [Integer, Time]
+      # @return [Integer] expected offset
+      def resolve_offset(consumer, name, partition, offset)
+        if offset.is_a?(Time)
+          tpl = ::Rdkafka::Consumer::TopicPartitionList.new
+          tpl.add_topic_and_partitions_with_offsets(
+            name, partition => offset
+          )
+
+          real_offsets = consumer.offsets_for_times(tpl)
+          detected_offset = real_offsets.to_h.dig(name, partition)
+
+          detected_offset&.offset || raise(Errors::InvalidTimeBasedOffsetError)
+        else
+          offset
+        end
+      end
+    end
+  end
+end

data/lib/karafka/app.rb

@@ -6,35 +6,57 @@ module Karafka
     extend Setup::Dsl
 
     class << self
-      # Sets up all the internal components and bootstrap whole app
-      # We need to know details about consumers in order to setup components,
-      # that's why we don't setup them after std setup is done
-      # @raise [Karafka::Errors::InvalidConfigurationError] raised when configuration
-      #   doesn't match with the config contract
-      def boot!
-        initialize!
-        Setup::Config.validate!
-        Setup::Config.setup_components
-        initialized!
+      # @return [Karafka::Routing::Builder] consumers builder instance alias
+      def consumer_groups
+        config
+          .internal
+          .routing
+          .builder
       end
 
-      # @return [Karafka::Routing::Builder] consumers builder instance
-      def consumer_groups
-        config.internal.routing_builder
+      # @return [Hash] active subscription groups grouped based on consumer group in a hash
+      def subscription_groups
+        # We first build all the subscription groups, so they all get the same position, despite
+        # later narrowing that. It allows us to maintain same position number for static members
+        # even when we want to run subset of consumer groups or subscription groups
+        #
+        # We then narrow this to active consumer groups from which we select active subscription
+        # groups.
+        consumer_groups
+          .map { |cg| [cg, cg.subscription_groups] }
+          .select { |cg, _| cg.active? }
+          .select { |_, sgs| sgs.delete_if { |sg| !sg.active? } }
+          .delete_if { |_, sgs| sgs.empty? }
+          .each { |_, sgs| sgs.each { |sg| sg.topics.delete_if { |top| !top.active? } } }
+          .each { |_, sgs| sgs.delete_if { |sg| sg.topics.empty? } }
+          .reject { |cg, _| cg.subscription_groups.empty? }
+          .to_h
       end
 
-      # Triggers reload of all cached Karafka app components, so we can use in-process
-      # in-development hot code reloading without Karafka process restart
-      def reload
-        Karafka::Persistence::Consumers.clear
-        Karafka::Persistence::Topics.clear
-        config.internal.routing_builder.reload
+      # Just a nicer name for the consumer groups
+      alias routes consumer_groups
+
+      # Allow for easier status management via `Karafka::App` by aliasing status methods here
+      Status::STATES.each do |state, transition|
+        class_eval <<~RUBY, __FILE__, __LINE__ + 1
+          def #{state}
+            App.config.internal.status.#{state}
+          end
+
+          def #{state}?
+            App.config.internal.status.#{state}?
+          end
+
+          def #{transition}
+            App.config.internal.status.#{transition}
+          end
+        RUBY
       end
 
-      Status.instance_methods(false).each do |delegated|
-        define_method(delegated) do
-          App.config.internal.status.send(delegated)
-        end
+      # @return [Boolean] true if we should be done in general with processing anything
+      # @note It is a meta status from the status object
+      def done?
+        App.config.internal.status.done?
       end
 
       # Methods that should be delegated to Karafka module
@@ -42,7 +64,9 @@ module Karafka
         root
         env
         logger
+        producer
         monitor
+        pro?
       ].each do |delegated|
         define_method(delegated) do
           Karafka.send(delegated)

data/lib/karafka/base_consumer.rb

@@ -4,54 +4,272 @@
 module Karafka
   # Base consumer from which all Karafka consumers should inherit
   class BaseConsumer
+    # Allow for consumer instance tagging for instrumentation
+    include ::Karafka::Core::Taggable
+
     extend Forwardable
 
-    # Allows us to mark messages as consumed for non-automatic mode without having
-    # to use consumer client directly. We do this that way, because most of the people should not
-    # mess with the client instance directly (just in case)
-    %i[
-      mark_as_consumed
-      mark_as_consumed!
-      trigger_heartbeat
-      trigger_heartbeat!
-    ].each do |delegated_method_name|
-      def_delegator :client, delegated_method_name
+    def_delegators :@coordinator, :topic, :partition
+
+    # @return [String] id of the current consumer
+    attr_reader :id
+    # @return [Karafka::Routing::Topic] topic to which a given consumer is subscribed
+    attr_accessor :messages
+    # @return [Karafka::Connection::Client] kafka connection client
+    attr_accessor :client
+    # @return [Karafka::Processing::Coordinator] coordinator
+    attr_accessor :coordinator
+    # @return [Waterdrop::Producer] producer instance
+    attr_accessor :producer
 
-      private delegated_method_name
+    # Creates new consumer and assigns it an id
+    def initialize
+      @id = SecureRandom.hex(6)
+      @used = false
     end
 
-    # @return [Karafka::Routing::Topic] topic to which a given consumer is subscribed
-    attr_reader :topic
-    # @return [Karafka::Params:ParamsBatch] current params batch
-    attr_accessor :params_batch
+    # Can be used to run preparation code prior to the job being enqueued
+    #
+    # @private
+    # @note This should not be used by the end users as it is part of the lifecycle of things and
+    #   not as a part of the public api. This should not perform any extensive operations as it is
+    #   blocking and running in the listener thread.
+    def on_before_enqueue
+      @used = true
+      handle_before_enqueue
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.before_enqueue.error'
+      )
+    end
 
-    # Assigns a topic to a consumer and builds up proper consumer functionalities
-    #   so that it can cooperate with the topic settings
-    # @param topic [Karafka::Routing::Topic]
-    def initialize(topic)
-      @topic = topic
-      Consumers::Includer.call(self)
+    # Can be used to run preparation code in the worker
+    #
+    # @private
+    # @note This should not be used by the end users as it is part of the lifecycle of things and
+    #   not as part of the public api. This can act as a hook when creating non-blocking
+    #   consumers and doing other advanced stuff
+    def on_before_consume
+      messages.metadata.processed_at = Time.now
+      messages.metadata.freeze
+
+      # We run this after the full metadata setup, so we can use all the messages information
+      # if needed
+      handle_before_consume
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.before_consume.error'
+      )
     end
 
     # Executes the default consumer flow.
-    def call
-      process
+    #
+    # @private
+    #
+    # @return [Boolean] true if there was no exception, otherwise false.
+    # @note We keep the seek offset tracking, and use it to compensate for async offset flushing
+    #   that may not yet kick in when error occurs. That way we pause always on the last processed
+    #   message.
+    def on_consume
+      handle_consume
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        seek_offset: coordinator.seek_offset,
+        type: 'consumer.consume.error'
+      )
     end
 
-    private
+    # @private
+    # @note This should not be used by the end users as it is part of the lifecycle of things but
+    #   not as part of the public api.
+    def on_after_consume
+      handle_after_consume
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.after_consume.error'
+      )
+    end
+
+    # Trigger method for running on idle runs without messages
+    #
+    # @private
+    def on_idle
+      handle_idle
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.idle.error'
+      )
+    end
 
-    # @return [Karafka::Connection::Client] messages consuming client that can be used to
-    #    commit manually offset or pause / stop consumer based on the business logic
-    def client
-      Persistence::Client.read
+    # Trigger method for running on partition revocation.
+    #
+    # @private
+    def on_revoked
+      handle_revoked
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.revoked.error'
+      )
     end
 
+    # Trigger method for running on shutdown.
+    #
+    # @private
+    def on_shutdown
+      handle_shutdown
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.shutdown.error'
+      )
+    end
+
+    private
+
     # Method that will perform business logic and on data received from Kafka (it will consume
     #   the data)
-    # @note This method needs bo be implemented in a subclass. We stub it here as a failover if
+    # @note This method needs to be implemented in a subclass. We stub it here as a failover if
     #   someone forgets about it or makes on with typo
     def consume
       raise NotImplementedError, 'Implement this in a subclass'
     end
+
+    # Method that will be executed when a given topic partition is revoked. You can use it for
+    # some teardown procedures (closing file handler, etc).
+    def revoked; end
+
+    # Method that will be executed when the process is shutting down. You can use it for
+    # some teardown procedures (closing file handler, etc).
+    def shutdown; end
+
+    # @return [Boolean] was this consumer in active use. Active use means running `#consume` at
+    #   least once. Consumer may have to run `#revoked` or `#shutdown` despite not running
+    #   `#consume` previously in delayed job cases and other cases that potentially involve running
+    #   the `Jobs::Idle` for house-keeping
+    def used?
+      @used
+    end
+
+    # Pauses processing on a given offset for the current topic partition
+    #
+    # After given partition is resumed, it will continue processing from the given offset
+    # @param offset [Integer] offset from which we want to restart the processing
+    # @param timeout [Integer, nil] how long in milliseconds do we want to pause or nil to use the
+    #   default exponential pausing strategy defined for retries
+    # @param manual_pause [Boolean] Flag to differentiate between user pause and system/strategy
+    #   based pause. While they both pause in exactly the same way, the strategy application
+    #   may need to differentiate between them.
+    def pause(offset, timeout = nil, manual_pause = true)
+      timeout ? coordinator.pause_tracker.pause(timeout) : coordinator.pause_tracker.pause
+
+      client.pause(
+        topic.name,
+        partition,
+        offset
+      )
+
+      # Indicate, that user took a manual action of pausing
+      coordinator.manual_pause if manual_pause
+
+      Karafka.monitor.instrument(
+        'consumer.consuming.pause',
+        caller: self,
+        manual: manual_pause,
+        topic: topic.name,
+        partition: partition,
+        offset: offset,
+        timeout: coordinator.pause_tracker.current_timeout,
+        attempt: coordinator.pause_tracker.attempt
+      )
+    end
+
+    # Resumes processing of the current topic partition
+    def resume
+      return unless coordinator.pause_tracker.paused?
+
+      # This is sufficient to expire a partition pause, as with it will be resumed by the listener
+      # thread before the next poll.
+      coordinator.pause_tracker.expire
+    end
+
+    # Seeks in the context of current topic and partition
+    #
+    # @param offset [Integer, Time] offset where we want to seek or time of the offset where we
+    #   want to seek.
+    # @param manual_seek [Boolean] Flag to differentiate between user seek and system/strategy
+    #   based seek. User seek operations should take precedence over system actions, hence we need
+    #   to know who invoked it.
+    # @note Please note, that if you are seeking to a time offset, getting the offset is blocking
+    def seek(offset, manual_seek = true)
+      coordinator.manual_seek if manual_seek
+
+      client.seek(
+        Karafka::Messages::Seek.new(
+          topic.name,
+          partition,
+          offset
+        )
+      )
+    end
+
+    # @return [Boolean] true if partition was revoked from the current consumer
+    # @note There are two "levels" on which we can know that partition was revoked. First one is
+    #   when we loose the assignment involuntarily and second is when coordinator gets this info
+    #   after we poll with the rebalance callbacks. The first check allows us to get this notion
+    #   even before we poll but it gets reset when polling happens, hence we also need to switch
+    #   the coordinator state after the revocation (but prior to running more jobs)
+    def revoked?
+      return true if coordinator.revoked?
+      return false unless client.assignment_lost?
+
+      coordinator.revoke
+
+      true
+    end
+
+    # @return [Boolean] are we retrying processing after an error. This can be used to provide a
+    #   different flow after there is an error, for example for resources cleanup, small manual
+    #   backoff or different instrumentation tracking.
+    def retrying?
+      coordinator.pause_tracker.attempt > 1
+    end
+
+    # Pauses the processing from the last offset to retry on given message
+    # @private
+    def retry_after_pause
+      pause(coordinator.seek_offset, nil, false)
+
+      # Instrumentation needs to run **after** `#pause` invocation because we rely on the states
+      # set by `#pause`
+      Karafka.monitor.instrument(
+        'consumer.consuming.retry',
+        caller: self,
+        topic: topic.name,
+        partition: partition,
+        offset: coordinator.seek_offset,
+        timeout: coordinator.pause_tracker.current_timeout,
+        attempt: coordinator.pause_tracker.attempt
+      )
+    end
   end
 end