karafka 2.0.0 → 2.1.0

data/lib/karafka/app.rb

@@ -14,20 +14,43 @@ module Karafka
           .builder
       end
 
-      # @return [Array<Karafka::Routing::SubscriptionGroup>] active subscription groups
+      # @return [Hash] active subscription groups grouped based on consumer group in a hash
       def subscription_groups
+        # We first build all the subscription groups, so they all get the same position, despite
+        # later narrowing that. It allows us to maintain same position number for static members
+        # even when we want to run subset of consumer groups or subscription groups
+        #
+        # We then narrow this to active consumer groups from which we select active subscription
+        # groups.
         consumer_groups
-          .active
-          .flat_map(&:subscription_groups)
+          .map { |cg| [cg, cg.subscription_groups] }
+          .select { |cg, _| cg.active? }
+          .select { |_, sgs| sgs.delete_if { |sg| !sg.active? } }
+          .delete_if { |_, sgs| sgs.empty? }
+          .each { |_, sgs| sgs.each { |sg| sg.topics.delete_if { |top| !top.active? } } }
+          .each { |_, sgs| sgs.delete_if { |sg| sg.topics.empty? } }
+          .reject { |cg, _| cg.subscription_groups.empty? }
+          .to_h
       end
 
       # Just a nicer name for the consumer groups
       alias routes consumer_groups
 
-      Status.instance_methods(false).each do |delegated|
-        define_method(delegated) do
-          App.config.internal.status.send(delegated)
-        end
+      # Allow for easier status management via `Karafka::App` by aliasing status methods here
+      Status::STATES.each do |state, transition|
+        class_eval <<~RUBY, __FILE__, __LINE__ + 1
+          def #{state}
+            App.config.internal.status.#{state}
+          end
+
+          def #{state}?
+            App.config.internal.status.#{state}?
+          end
+
+          def #{transition}
+            App.config.internal.status.#{transition}
+          end
+        RUBY
       end
 
       # Methods that should be delegated to Karafka module

data/lib/karafka/base_consumer.rb

@@ -4,9 +4,16 @@
 module Karafka
   # Base consumer from which all Karafka consumers should inherit
   class BaseConsumer
+    # Allow for consumer instance tagging for instrumentation
+    include ::Karafka::Core::Taggable
+
+    extend Forwardable
+
+    def_delegators :@coordinator, :topic, :partition
+
+    # @return [String] id of the current consumer
+    attr_reader :id
     # @return [Karafka::Routing::Topic] topic to which a given consumer is subscribed
-    attr_accessor :topic
-    # @return [Karafka::Messages::Messages] current messages batch
     attr_accessor :messages
     # @return [Karafka::Connection::Client] kafka connection client
     attr_accessor :client
@@ -15,13 +22,49 @@ module Karafka
     # @return [Waterdrop::Producer] producer instance
     attr_accessor :producer
 
-    # Can be used to run preparation code
+    # Creates new consumer and assigns it an id
+    def initialize
+      @id = SecureRandom.hex(6)
+    end
+
+    # Can be used to run preparation code prior to the job being enqueued
     #
     # @private
-    # @note This should not be used by the end users as it is part of the lifecycle of things but
+    # @note This should not be used by the end users as it is part of the lifecycle of things and
+    #   not as a part of the public api. This should not perform any extensive operations as it is
+    #   blocking and running in the listener thread.
+    def on_before_enqueue
+      handle_before_enqueue
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.before_enqueue.error'
+      )
+    end
+
+    # Can be used to run preparation code in the worker
+    #
+    # @private
+    # @note This should not be used by the end users as it is part of the lifecycle of things and
     #   not as part of the public api. This can act as a hook when creating non-blocking
     #   consumers and doing other advanced stuff
-    def on_before_consume; end
+    def on_before_consume
+      messages.metadata.processed_at = Time.now
+      messages.metadata.freeze
+
+      # We run this after the full metadata setup, so we can use all the messages information
+      # if needed
+      handle_before_consume
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.before_consume.error'
+      )
+    end
 
     # Executes the default consumer flow.
     #
@@ -31,54 +74,50 @@ module Karafka
     #   that may not yet kick in when error occurs. That way we pause always on the last processed
     #   message.
     def on_consume
-      Karafka.monitor.instrument('consumer.consumed', caller: self) do
-        consume
-      end
-
-      coordinator.consumption(self).success!
+      handle_consume
     rescue StandardError => e
-      coordinator.consumption(self).failure!
-
       Karafka.monitor.instrument(
         'error.occurred',
         error: e,
         caller: self,
+        seek_offset: coordinator.seek_offset,
         type: 'consumer.consume.error'
       )
-    ensure
-      # We need to decrease number of jobs that this coordinator coordinates as it has finished
-      coordinator.decrement
     end
 
     # @private
     # @note This should not be used by the end users as it is part of the lifecycle of things but
     #   not as part of the public api.
     def on_after_consume
-      return if revoked?
-
-      if coordinator.success?
-        coordinator.pause_tracker.reset
-
-        # Mark as consumed only if manual offset management is not on
-        return if topic.manual_offset_management?
+      handle_after_consume
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.after_consume.error'
+      )
+    end
 
-        # We use the non-blocking one here. If someone needs the blocking one, can implement it
-        # with manual offset management
-        mark_as_consumed(messages.last)
-      else
-        pause(@seek_offset || messages.first.offset)
-      end
+    # Trigger method for running on idle runs without messages
+    #
+    # @private
+    def on_idle
+      handle_idle
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.idle.error'
+      )
     end
 
-    # Trigger method for running on shutdown.
+    # Trigger method for running on partition revocation.
     #
     # @private
     def on_revoked
-      coordinator.revoke
-
-      Karafka.monitor.instrument('consumer.revoked', caller: self) do
-        revoked
-      end
+      handle_revoked
     rescue StandardError => e
       Karafka.monitor.instrument(
         'error.occurred',
@@ -92,9 +131,7 @@ module Karafka
     #
     # @private
     def on_shutdown
-      Karafka.monitor.instrument('consumer.shutdown', caller: self) do
-        shutdown
-      end
+      handle_shutdown
     rescue StandardError => e
       Karafka.monitor.instrument(
         'error.occurred',
@@ -108,7 +145,7 @@ module Karafka
 
     # Method that will perform business logic and on data received from Kafka (it will consume
     #   the data)
-    # @note This method needs bo be implemented in a subclass. We stub it here as a failover if
+    # @note This method needs to be implemented in a subclass. We stub it here as a failover if
     #   someone forgets about it or makes on with typo
     def consume
       raise NotImplementedError, 'Implement this in a subclass'
@@ -122,63 +159,43 @@ module Karafka
     # some teardown procedures (closing file handler, etc).
     def shutdown; end
 
-    # Marks message as consumed in an async way.
-    #
-    # @param message [Messages::Message] last successfully processed message.
-    # @return [Boolean] true if we were able to mark the offset, false otherwise. False indicates
-    #   that we were not able and that we have lost the partition.
-    #
-    # @note We keep track of this offset in case we would mark as consumed and got error when
-    #   processing another message. In case like this we do not pause on the message we've already
-    #   processed but rather at the next one. This applies to both sync and async versions of this
-    #   method.
-    def mark_as_consumed(message)
-      unless client.mark_as_consumed(message)
-        coordinator.revoke
-
-        return false
-      end
-
-      @seek_offset = message.offset + 1
-
-      true
-    end
-
-    # Marks message as consumed in a sync way.
-    #
-    # @param message [Messages::Message] last successfully processed message.
-    # @return [Boolean] true if we were able to mark the offset, false otherwise. False indicates
-    #   that we were not able and that we have lost the partition.
-    def mark_as_consumed!(message)
-      unless client.mark_as_consumed!(message)
-        coordinator.revoke
-
-        return false
-      end
-
-      @seek_offset = message.offset + 1
-
-      true
-    end
-
     # Pauses processing on a given offset for the current topic partition
     #
     # After given partition is resumed, it will continue processing from the given offset
     # @param offset [Integer] offset from which we want to restart the processing
     # @param timeout [Integer, nil] how long in milliseconds do we want to pause or nil to use the
     #   default exponential pausing strategy defined for retries
-    def pause(offset, timeout = nil)
+    # @param manual_pause [Boolean] Flag to differentiate between user pause and system/strategy
+    #   based pause. While they both pause in exactly the same way, the strategy application
+    #   may need to differentiate between them.
+    def pause(offset, timeout = nil, manual_pause = true)
       timeout ? coordinator.pause_tracker.pause(timeout) : coordinator.pause_tracker.pause
 
       client.pause(
-        messages.metadata.topic,
-        messages.metadata.partition,
+        topic.name,
+        partition,
         offset
       )
+
+      # Indicate, that user took a manual action of pausing
+      coordinator.manual_pause if manual_pause
+
+      Karafka.monitor.instrument(
+        'consumer.consuming.pause',
+        caller: self,
+        manual: manual_pause,
+        topic: topic.name,
+        partition: partition,
+        offset: offset,
+        timeout: coordinator.pause_tracker.current_timeout,
+        attempt: coordinator.pause_tracker.attempt
+      )
     end
 
     # Resumes processing of the current topic partition
     def resume
+      return unless coordinator.pause_tracker.paused?
+
       # This is sufficient to expire a partition pause, as with it will be resumed by the listener
       # thread before the next poll.
       coordinator.pause_tracker.expire
@@ -190,8 +207,8 @@ module Karafka
     def seek(offset)
       client.seek(
         Karafka::Messages::Seek.new(
-          messages.metadata.topic,
-          messages.metadata.partition,
+          topic.name,
+          partition,
           offset
         )
       )
@@ -203,5 +220,30 @@ module Karafka
     def revoked?
       coordinator.revoked?
     end
+
+    # @return [Boolean] are we retrying processing after an error. This can be used to provide a
+    #   different flow after there is an error, for example for resources cleanup, small manual
+    #   backoff or different instrumentation tracking.
+    def retrying?
+      coordinator.pause_tracker.attempt.positive?
+    end
+
+    # Pauses the processing from the last offset to retry on given message
+    # @private
+    def retry_after_pause
+      pause(coordinator.seek_offset, nil, false)
+
+      # Instrumentation needs to run **after** `#pause` invocation because we rely on the states
+      # set by `#pause`
+      Karafka.monitor.instrument(
+        'consumer.consuming.retry',
+        caller: self,
+        topic: topic.name,
+        partition: partition,
+        offset: coordinator.seek_offset,
+        timeout: coordinator.pause_tracker.current_timeout,
+        attempt: coordinator.pause_tracker.attempt
+      )
+    end
   end
 end

data/lib/karafka/cli/base.rb

@@ -34,6 +34,26 @@ module Karafka
       end
 
       class << self
+        # Loads proper environment with what is needed to run the CLI
+        def load
+          # If there is a boot file, we need to require it as we expect it to contain
+          # Karafka app setup, routes, etc
+          if File.exist?(::Karafka.boot_file)
+            rails_env_rb = File.join(Dir.pwd, 'config/environment.rb')
+
+            # Load Rails environment file that starts Rails, so we can reference consumers and
+            # other things from `karafka.rb` file. This will work only for Rails, for non-rails
+            # a manual setup is needed
+            require rails_env_rb if Kernel.const_defined?(:Rails) && File.exist?(rails_env_rb)
+
+            require Karafka.boot_file.to_s
+          # However when it is unavailable, we still want to be able to run help command
+          # and install command as they don't require configured app itself to run
+          elsif %w[-h install].none? { |cmd| cmd == ARGV[0] }
+            raise ::Karafka::Errors::MissingBootFileError, ::Karafka.boot_file
+          end
+        end
+
         # Allows to set options for Thor cli
         # @see https://github.com/erikhuda/thor
         # @param option Single option details

data/lib/karafka/cli/console.rb

@@ -9,22 +9,27 @@ module Karafka
       option aliases: 'c'
 
       class << self
-        # @return [String] Console executing command
+        # @return [String] Console executing command for non-Rails setup
         # @example
         #   Karafka::Cli::Console.command #=> 'KARAFKA_CONSOLE=true bundle exec irb...'
-        def command
-          envs = [
-            "IRBRC='#{Karafka.gem_root}/.console_irbrc'",
-            'KARAFKA_CONSOLE=true'
-          ]
-          "#{envs.join(' ')} bundle exec irb -r #{Karafka.boot_file}"
+        def console
+          "IRBRC='#{Karafka.gem_root}/.console_irbrc' bundle exec irb -r #{Karafka.boot_file}"
+        end
+
+        # @return [String] Console executing command for Rails setup
+        # @note In case of Rails, it has its own console, hence we can just defer to it
+        def rails_console
+          'bundle exec rails console'
         end
       end
 
       # Start the Karafka console
       def call
         cli.info
-        exec self.class.command
+
+        command = ::Karafka.rails? ? self.class.rails_console : self.class.console
+
+        exec "KARAFKA_CONSOLE=true #{command}"
       end
     end
   end

data/lib/karafka/cli/info.rb

@@ -35,9 +35,10 @@ module Karafka
 
         [
           "Karafka version: #{Karafka::VERSION}#{postfix}",
-          "Ruby version: #{RUBY_VERSION}",
+          "Ruby version: #{RUBY_DESCRIPTION}",
           "Rdkafka version: #{::Rdkafka::VERSION}",
-          "Subscription groups count: #{Karafka::App.subscription_groups.size}",
+          "Consumer groups count: #{Karafka::App.consumer_groups.size}",
+          "Subscription groups count: #{Karafka::App.subscription_groups.values.flatten.size}",
           "Workers count: #{Karafka::App.config.concurrency}",
           "Application client id: #{config.client_id}",
           "Boot file: #{Karafka.boot_file}",

data/lib/karafka/cli/server.rb

@@ -7,37 +7,84 @@ module Karafka
     class Server < Base
       include Helpers::Colorize
 
+      # Types of things we can include / exclude from the routing via the CLI options
+      SUPPORTED_TYPES = ::Karafka::Routing::ActivityManager::SUPPORTED_TYPES
+
+      private_constant :SUPPORTED_TYPES
+
       desc 'Start the Karafka server (short-cut alias: "s")'
+
       option aliases: 's'
-      option :consumer_groups, type: :array, default: nil, aliases: :g
+
+      # Thor does not work well with many aliases combinations, hence we remap the aliases
+      # by ourselves in the code
+      option :consumer_groups, type: :array, default: [], aliases: :g
+      option :subscription_groups, type: :array, default: []
+      option :topics, type: :array, default: []
+
+      %i[
+        include
+        exclude
+      ].each do |action|
+        SUPPORTED_TYPES.each do |type|
+          option(
+            "#{action}_#{type}",
+            type: :array,
+            default: []
+          )
+        end
+      end
 
       # Start the Karafka server
       def call
         # Print our banner and info in the dev mode
         print_marketing_info if Karafka::App.env.development?
 
-        Contracts::ServerCliOptions.new.validate!(cli.options)
-
-        # We assign active topics on a server level, as only server is expected to listen on
-        # part of the topics
-        Karafka::Server.consumer_groups = cli.options[:consumer_groups]
+        register_inclusions(cli)
+        register_exclusions(cli)
 
         Karafka::Server.run
       end
 
       private
 
+      # Registers things we want to include (if defined)
+      # @param cli [Karafka::Cli] Thor cli handler
+      def register_inclusions(cli)
+        activities = ::Karafka::App.config.internal.routing.activity_manager
+
+        SUPPORTED_TYPES.each do |type|
+          v1 = cli.options[type] || []
+          v2 = cli.options[:"include_#{type}"] || []
+          names = v1 + v2
+
+          names.each { |name| activities.include(type, name) }
+        end
+      end
+
+      # Registers things we want to exclude (if defined)
+      # @param cli [Karafka::Cli] Thor cli handler
+      def register_exclusions(cli)
+        activities = ::Karafka::App.config.internal.routing.activity_manager
+
+        activities.class::SUPPORTED_TYPES.each do |type|
+          names = cli.options[:"exclude_#{type}"] || []
+
+          names.each { |name| activities.exclude(type, name) }
+        end
+      end
+
       # Prints marketing info
       def print_marketing_info
         Karafka.logger.info Info::BANNER
 
         if Karafka.pro?
           Karafka.logger.info(
-            green('Thank you for investing in the Karafka Pro subscription!')
+            green('Thank you for using Karafka Pro!')
           )
         else
           Karafka.logger.info(
-            red('You like Karafka? Please consider getting a Pro version!')
+            red('Upgrade to Karafka Pro for more features and support: https://karafka.io')
           )
         end
       end

data/lib/karafka/cli/topics.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+module Karafka
+  class Cli < Thor
+    # CLI actions related to Kafka cluster topics management
+    class Topics < Base
+      include Helpers::Colorize
+
+      desc 'Allows for the topics management (create, delete, reset, repartition)'
+      # @param action [String] action we want to take
+      def call(action = 'missing')
+        case action
+        when 'create'
+          create
+        when 'delete'
+          delete
+        when 'reset'
+          reset
+        when 'repartition'
+          repartition
+        when 'migrate'
+          migrate
+        else
+          raise ::ArgumentError, "Invalid topics action: #{action}"
+        end
+      end
+
+      private
+
+      # Creates topics based on the routing setup and configuration
+      def create
+        declaratives_routing_topics.each do |topic|
+          name = topic.name
+
+          if existing_topics_names.include?(name)
+            puts "#{yellow('Skipping')} because topic #{name} already exists."
+          else
+            puts "Creating topic #{name}..."
+            Admin.create_topic(
+              name,
+              topic.declaratives.partitions,
+              topic.declaratives.replication_factor,
+              topic.declaratives.details
+            )
+            puts "#{green('Created')} topic #{name}."
+          end
+        end
+      end
+
+      # Deletes routing based topics
+      def delete
+        declaratives_routing_topics.each do |topic|
+          name = topic.name
+
+          if existing_topics_names.include?(name)
+            puts "Deleting topic #{name}..."
+            Admin.delete_topic(name)
+            puts "#{green('Deleted')} topic #{name}."
+          else
+            puts "#{yellow('Skipping')} because topic #{name} does not exist."
+          end
+        end
+      end
+
+      # Deletes routing based topics and re-creates them
+      def reset
+        delete
+
+        # We need to invalidate the metadata cache, otherwise we will think, that the topic
+        # already exists
+        @existing_topics = nil
+
+        create
+      end
+
+      # Creates missing topics and aligns the partitions count
+      def migrate
+        create
+
+        @existing_topics = nil
+
+        repartition
+      end
+
+      # Increases number of partitions on topics that have less partitions than defined
+      # Will **not** create topics if missing.
+      def repartition
+        existing_partitions = existing_topics.map do |topic|
+          [topic.fetch(:topic_name), topic.fetch(:partition_count)]
+        end.to_h
+
+        declaratives_routing_topics.each do |topic|
+          name = topic.name
+
+          desired_count = topic.config.partitions
+          existing_count = existing_partitions.fetch(name, false)
+
+          if existing_count && existing_count < desired_count
+            puts "Increasing number of partitions to #{desired_count} on topic #{name}..."
+            Admin.create_partitions(name, desired_count)
+            change = desired_count - existing_count
+            puts "#{green('Created')} #{change} additional partitions on topic #{name}."
+          elsif existing_count
+            puts "#{yellow('Skipping')} because topic #{name} has #{existing_count} partitions."
+          else
+            puts "#{yellow('Skipping')} because topic #{name} does not exist."
+          end
+        end
+      end
+
+      # @return [Array<Karafka::Routing::Topic>] all available topics that can be managed
+      # @note If topic is defined in multiple consumer groups, first config will be used. This
+      #   means, that this CLI will not work for simultaneous management of multiple clusters from
+      #   a single CLI command execution flow.
+      def declaratives_routing_topics
+        return @declaratives_routing_topics if @declaratives_routing_topics
+
+        collected_topics = {}
+        default_servers = Karafka::App.config.kafka[:'bootstrap.servers']
+
+        App.consumer_groups.each do |consumer_group|
+          consumer_group.topics.each do |topic|
+            # Skip topics that were explicitly disabled from management
+            next unless topic.declaratives.active?
+            # If bootstrap servers are different, consider this a different cluster
+            next unless default_servers == topic.kafka[:'bootstrap.servers']
+
+            collected_topics[topic.name] ||= topic
+          end
+        end
+
+        @declaratives_routing_topics = collected_topics.values
+      end
+
+      # @return [Array<Hash>] existing topics details
+      def existing_topics
+        @existing_topics ||= Admin.cluster_info.topics
+      end
+
+      # @return [Array<String>] names of already existing topics
+      def existing_topics_names
+        existing_topics.map { |topic| topic.fetch(:topic_name) }
+      end
+    end
+  end
+end