karafka 1.4.12 → 2.2.10

data/lib/karafka/base_consumer.rb

@@ -4,54 +4,285 @@
 module Karafka
   # Base consumer from which all Karafka consumers should inherit
   class BaseConsumer
+    # Allow for consumer instance tagging for instrumentation
+    include ::Karafka::Core::Taggable
+
     extend Forwardable
 
-    # Allows us to mark messages as consumed for non-automatic mode without having
-    # to use consumer client directly. We do this that way, because most of the people should not
-    # mess with the client instance directly (just in case)
-    %i[
-      mark_as_consumed
-      mark_as_consumed!
-      trigger_heartbeat
-      trigger_heartbeat!
-    ].each do |delegated_method_name|
-      def_delegator :client, delegated_method_name
+    def_delegators :@coordinator, :topic, :partition
+
+    # @return [String] id of the current consumer
+    attr_reader :id
+    # @return [Karafka::Routing::Topic] topic to which a given consumer is subscribed
+    attr_accessor :messages
+    # @return [Karafka::Connection::Client] kafka connection client
+    attr_accessor :client
+    # @return [Karafka::Processing::Coordinator] coordinator
+    attr_accessor :coordinator
+    # @return [Waterdrop::Producer] producer instance
+    attr_accessor :producer
+
+    # Creates new consumer and assigns it an id
+    def initialize
+      @id = SecureRandom.hex(6)
+      @used = false
+    end
 
-      private delegated_method_name
+    # Can be used to run preparation code prior to the job being enqueued
+    #
+    # @private
+    # @note This should not be used by the end users as it is part of the lifecycle of things and
+    #   not as a part of the public api. This should not perform any extensive operations as it is
+    #   blocking and running in the listener thread.
+    def on_before_enqueue
+      @used = true
+      handle_before_enqueue
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.before_enqueue.error'
+      )
     end
 
-    # @return [Karafka::Routing::Topic] topic to which a given consumer is subscribed
-    attr_reader :topic
-    # @return [Karafka::Params:ParamsBatch] current params batch
-    attr_accessor :params_batch
+    # Can be used to run preparation code in the worker
+    #
+    # @private
+    # @note This should not be used by the end users as it is part of the lifecycle of things and
+    #   not as part of the public api. This can act as a hook when creating non-blocking
+    #   consumers and doing other advanced stuff
+    def on_before_consume
+      messages.metadata.processed_at = Time.now
+      messages.metadata.freeze
 
-    # Assigns a topic to a consumer and builds up proper consumer functionalities
-    #   so that it can cooperate with the topic settings
-    # @param topic [Karafka::Routing::Topic]
-    def initialize(topic)
-      @topic = topic
-      Consumers::Includer.call(self)
+      # We run this after the full metadata setup, so we can use all the messages information
+      # if needed
+      handle_before_consume
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.before_consume.error'
+      )
     end
 
     # Executes the default consumer flow.
-    def call
-      process
+    #
+    # @private
+    #
+    # @return [Boolean] true if there was no exception, otherwise false.
+    #
+    # @note We keep the seek offset tracking, and use it to compensate for async offset flushing
+    #   that may not yet kick in when error occurs. That way we pause always on the last processed
+    #   message.
+    def on_consume
+      handle_consume
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        seek_offset: coordinator.seek_offset,
+        type: 'consumer.consume.error'
+      )
     end
 
-    private
+    # @private
+    # @note This should not be used by the end users as it is part of the lifecycle of things but
+    #   not as part of the public api.
+    def on_after_consume
+      handle_after_consume
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.after_consume.error'
+      )
+    end
 
-    # @return [Karafka::Connection::Client] messages consuming client that can be used to
-    #    commit manually offset or pause / stop consumer based on the business logic
-    def client
-      Persistence::Client.read
+    # Trigger method for running on idle runs without messages
+    #
+    # @private
+    def on_idle
+      handle_idle
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.idle.error'
+      )
     end
 
+    # Trigger method for running on partition revocation.
+    #
+    # @private
+    def on_revoked
+      handle_revoked
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.revoked.error'
+      )
+    end
+
+    # Trigger method for running on shutdown.
+    #
+    # @private
+    def on_shutdown
+      handle_shutdown
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.shutdown.error'
+      )
+    end
+
+    private
+
     # Method that will perform business logic and on data received from Kafka (it will consume
     #   the data)
-    # @note This method needs bo be implemented in a subclass. We stub it here as a failover if
+    # @note This method needs to be implemented in a subclass. We stub it here as a failover if
     #   someone forgets about it or makes on with typo
     def consume
       raise NotImplementedError, 'Implement this in a subclass'
     end
+
+    # Method that will be executed when a given topic partition is revoked. You can use it for
+    # some teardown procedures (closing file handler, etc).
+    def revoked; end
+
+    # Method that will be executed when the process is shutting down. You can use it for
+    # some teardown procedures (closing file handler, etc).
+    def shutdown; end
+
+    # @return [Boolean] was this consumer in active use. Active use means running `#consume` at
+    #   least once. Consumer may have to run `#revoked` or `#shutdown` despite not running
+    #   `#consume` previously in delayed job cases and other cases that potentially involve running
+    #   the `Jobs::Idle` for house-keeping
+    def used?
+      @used
+    end
+
+    # Pauses processing on a given offset or consecutive offset for the current topic partition
+    #
+    # After given partition is resumed, it will continue processing from the given offset
+    # @param offset [Integer, Symbol] offset from which we want to restart the processing or
+    #  `:consecutive` if we want to pause and continue without changing the consecutive offset
+    #  (cursor position)
+    # @param timeout [Integer, nil] how long in milliseconds do we want to pause or nil to use the
+    #   default exponential pausing strategy defined for retries
+    # @param manual_pause [Boolean] Flag to differentiate between user pause and system/strategy
+    #   based pause. While they both pause in exactly the same way, the strategy application
+    #   may need to differentiate between them.
+    #
+    # @note It is **critical** to understand how pause with `:consecutive` offset operates. While
+    #   it provides benefit of not purging librdkafka buffer, in case of usage of filters, retries
+    #   or other advanced options the consecutive offset may not be the one you want to pause on.
+    #   Test it well to ensure, that this behaviour is expected by you.
+    def pause(offset, timeout = nil, manual_pause = true)
+      timeout ? coordinator.pause_tracker.pause(timeout) : coordinator.pause_tracker.pause
+
+      offset = nil if offset == :consecutive
+
+      client.pause(topic.name, partition, offset)
+
+      # Indicate, that user took a manual action of pausing
+      coordinator.manual_pause if manual_pause
+
+      Karafka.monitor.instrument(
+        'consumer.consuming.pause',
+        caller: self,
+        manual: manual_pause,
+        topic: topic.name,
+        partition: partition,
+        subscription_group: topic.subscription_group,
+        offset: offset,
+        timeout: coordinator.pause_tracker.current_timeout,
+        attempt: coordinator.pause_tracker.attempt
+      )
+    end
+
+    # Resumes processing of the current topic partition
+    def resume
+      return unless coordinator.pause_tracker.paused?
+
+      # This is sufficient to expire a partition pause, as with it will be resumed by the listener
+      # thread before the next poll.
+      coordinator.pause_tracker.expire
+    end
+
+    # Seeks in the context of current topic and partition
+    #
+    # @param offset [Integer, Time] offset where we want to seek or time of the offset where we
+    #   want to seek.
+    # @param manual_seek [Boolean] Flag to differentiate between user seek and system/strategy
+    #   based seek. User seek operations should take precedence over system actions, hence we need
+    #   to know who invoked it.
+    # @note Please note, that if you are seeking to a time offset, getting the offset is blocking
+    def seek(offset, manual_seek = true)
+      coordinator.manual_seek if manual_seek
+
+      client.seek(
+        Karafka::Messages::Seek.new(
+          topic.name,
+          partition,
+          offset
+        )
+      )
+    end
+
+    # @return [Boolean] true if partition was revoked from the current consumer
+    # @note There are two "levels" on which we can know that partition was revoked. First one is
+    #   when we loose the assignment involuntarily and second is when coordinator gets this info
+    #   after we poll with the rebalance callbacks. The first check allows us to get this notion
+    #   even before we poll but it gets reset when polling happens, hence we also need to switch
+    #   the coordinator state after the revocation (but prior to running more jobs)
+    def revoked?
+      return true if coordinator.revoked?
+      return false unless client.assignment_lost?
+
+      coordinator.revoke
+
+      true
+    end
+
+    # @return [Boolean] are we retrying processing after an error. This can be used to provide a
+    #   different flow after there is an error, for example for resources cleanup, small manual
+    #   backoff or different instrumentation tracking.
+    def retrying?
+      attempt > 1
+    end
+
+    # @return [Integer] attempt of processing given batch. 1 if this is the first attempt or higher
+    #  in case it is a retry
+    def attempt
+      coordinator.pause_tracker.attempt
+    end
+
+    # Pauses the processing from the last offset to retry on given message
+    # @private
+    def retry_after_pause
+      pause(coordinator.seek_offset, nil, false)
+
+      # Instrumentation needs to run **after** `#pause` invocation because we rely on the states
+      # set by `#pause`
+      Karafka.monitor.instrument(
+        'consumer.consuming.retry',
+        caller: self,
+        topic: topic.name,
+        partition: partition,
+        offset: coordinator.seek_offset,
+        timeout: coordinator.pause_tracker.current_timeout,
+        attempt: coordinator.pause_tracker.attempt
+      )
+    end
   end
 end

data/lib/karafka/cli/base.rb

@@ -1,31 +1,17 @@
 # frozen_string_literal: true
 
 module Karafka
-  class Cli < Thor
+  class Cli
     # Base class for all the command that we want to define
-    # This base class provides a nicer interface to Thor and allows to easier separate single
-    # independent commands
-    # In order to define a new command you need to:
-    #   - specify its desc
-    #   - implement call method
-    #
-    # @example Create a dummy command
-    #   class Dummy < Base
-    #     self.desc = 'Dummy command'
-    #
-    #     def call
-    #       puts 'I'm doing nothing!
-    #     end
-    #   end
+    # This base class provides an interface to easier separate single independent commands
     class Base
-      include Thor::Shell
+      # @return [Hash] given command cli options
+      attr_reader :options
 
-      # We can use it to call other cli methods via this object
-      attr_reader :cli
-
-      # @param cli [Karafka::Cli] current Karafka Cli instance
-      def initialize(cli)
-        @cli = cli
+      # Creates new CLI command instance
+      def initialize
+        # Parses the given command CLI options
+        @options = self.class.parse_options
       end
 
       # This method should implement proper cli action
@@ -34,6 +20,26 @@ module Karafka
       end
 
       class << self
+        # Loads proper environment with what is needed to run the CLI
+        def load
+          # If there is a boot file, we need to require it as we expect it to contain
+          # Karafka app setup, routes, etc
+          if File.exist?(::Karafka.boot_file)
+            rails_env_rb = File.join(Dir.pwd, 'config/environment.rb')
+
+            # Load Rails environment file that starts Rails, so we can reference consumers and
+            # other things from `karafka.rb` file. This will work only for Rails, for non-rails
+            # a manual setup is needed
+            require rails_env_rb if Kernel.const_defined?(:Rails) && File.exist?(rails_env_rb)
+
+            require Karafka.boot_file.to_s
+          # However when it is unavailable, we still want to be able to run help command
+          # and install command as they don't require configured app itself to run
+          elsif %w[-h install].none? { |cmd| cmd == ARGV[0] }
+            raise ::Karafka::Errors::MissingBootFileError, ::Karafka.boot_file
+          end
+        end
+
         # Allows to set options for Thor cli
         # @see https://github.com/erikhuda/thor
         # @param option Single option details
@@ -43,27 +49,47 @@ module Karafka
         end
 
         # Allows to set description of a given cli command
-        # @param args [Array] All the arguments that Thor desc method accepts
-        def desc(*args)
-          @desc ||= args
+        # @param desc [String] Description of a given cli command
+        def desc(desc = nil)
+          @desc ||= desc
         end
 
-        # This method will bind a given Cli command into Karafka Cli
-        # This method is a wrapper to way Thor defines its commands
-        # @param cli_class [Karafka::Cli] Karafka cli_class
-        def bind_to(cli_class)
-          cli_class.desc name, *@desc
+        # Allows to set aliases for a given cli command
+        # @param args [Array] list of aliases that we can use to run given cli command
+        def aliases(*args)
+          @aliases ||= []
+          @aliases << args.map(&:to_s)
+        end
 
-          (@options || []).each { |option| cli_class.option(*option) }
+        # Parses the CLI options
+        # @return [Hash] hash with parsed values
+        def parse_options
+          options = {}
 
-          context = self
+          OptionParser.new do |opts|
+            (@options || []).each do |option|
+              # Creates aliases for backwards compatibility
+              names = option[3].flat_map { |name| [name, name.tr('_', '-')] }
+              names.map! { |name| "#{name} value1,value2,valueN" } if option[2] == Array
+              names.uniq!
 
-          cli_class.send :define_method, name do |*args|
-            context.new(self).call(*args)
-          end
+              opts.on(
+                *[names, option[2], option[1]].flatten
+              ) { |value| options[option[0]] = value }
+            end
+          end.parse!
+
+          options
         end
 
-        private
+        # @return [Array<Class>] available commands
+        def commands
+          ObjectSpace
+            .each_object(Class)
+            .select { |klass| klass.superclass == Karafka::Cli::Base }
+            .reject { |klass| klass.to_s.end_with?('::Base') }
+            .sort_by(&:name)
+        end
 
         # @return [String] downcased current class name that we use to define name for
         #   given Cli command
@@ -72,6 +98,11 @@ module Karafka
         def name
           to_s.split('::').last.downcase
         end
+
+        # @return [Array<String>] names and aliases for command matching
+        def names
+          ((@aliases || []) << name).flatten.map(&:to_s)
+        end
       end
     end
   end

data/lib/karafka/cli/console.rb

@@ -2,29 +2,35 @@
 
 module Karafka
   # Karafka framework Cli
-  class Cli < Thor
+  class Cli
     # Console Karafka Cli action
     class Console < Base
-      desc 'Start the Karafka console (short-cut alias: "c")'
-      option aliases: 'c'
+      desc 'Starts the Karafka console (short-cut alias: "c")'
+
+      aliases :c
 
       class << self
-        # @return [String] Console executing command
+        # @return [String] Console executing command for non-Rails setup
         # @example
         #   Karafka::Cli::Console.command #=> 'KARAFKA_CONSOLE=true bundle exec irb...'
-        def command
-          envs = [
-            "IRBRC='#{Karafka.gem_root}/.console_irbrc'",
-            'KARAFKA_CONSOLE=true'
-          ]
-          "#{envs.join(' ')} bundle exec irb -r #{Karafka.boot_file}"
+        def console
+          "IRBRC='#{Karafka.gem_root}/.console_irbrc' bundle exec irb -r #{Karafka.boot_file}"
+        end
+
+        # @return [String] Console executing command for Rails setup
+        # @note In case of Rails, it has its own console, hence we can just defer to it
+        def rails_console
+          'bundle exec rails console'
         end
       end
 
       # Start the Karafka console
       def call
-        cli.info
-        exec self.class.command
+        Info.new.call
+
+        command = ::Karafka.rails? ? self.class.rails_console : self.class.console
+
+        exec "KARAFKA_CONSOLE=true #{command}"
       end
     end
   end

data/lib/karafka/cli/help.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Karafka framework Cli
+  class Cli
+    # Prints info with list of commands available
+    class Help < Base
+      desc 'Describes available commands'
+
+      # Print available commands
+      def call
+        # Find the longest command for alignment purposes
+        max_command_length = self.class.commands.map(&:name).map(&:size).max
+
+        puts 'Karafka commands:'
+
+        # Print each command formatted with its description
+        self.class.commands.each do |command|
+          puts "  #{command.name.ljust(max_command_length)}    # #{command.desc}"
+        end
+      end
+    end
+  end
+end

data/lib/karafka/cli/info.rb

@@ -2,29 +2,64 @@
 
 module Karafka
   # Karafka framework Cli
-  class Cli < Thor
+  class Cli
     # Info Karafka Cli action
     class Info < Base
-      desc 'Print configuration details and other options of your application'
+      desc 'Prints configuration details and other options of your application'
+
+      # Nice karafka banner
+      BANNER = <<~BANNER
+
+        @@@                                             @@@@@  @@@
+        @@@                                            @@@     @@@
+        @@@  @@@    @@@@@@@@@   @@@ @@@   @@@@@@@@@  @@@@@@@@  @@@  @@@@   @@@@@@@@@
+        @@@@@@     @@@    @@@   @@@@@    @@@    @@@    @@@     @@@@@@@    @@@    @@@
+        @@@@@@@    @@@    @@@   @@@     @@@@    @@@    @@@     @@@@@@@    @@@    @@@
+        @@@  @@@@  @@@@@@@@@@   @@@      @@@@@@@@@@    @@@     @@@  @@@@   @@@@@@@@@@
+
+      BANNER
 
       # Print configuration details and other options of your application
       def call
+        Karafka.logger.info(BANNER)
+        Karafka.logger.info((core_info + license_info).join("\n"))
+      end
+
+      private
+
+      # @return [Array<String>] core framework related info
+      def core_info
         config = Karafka::App.config
 
-        info = [
-          "Karafka version: #{Karafka::VERSION}",
-          "Ruby version: #{RUBY_VERSION}",
-          "Ruby-kafka version: #{::Kafka::VERSION}",
+        postfix = Karafka.pro? ? ' + Pro' : ''
+
+        [
+          "Karafka version: #{Karafka::VERSION}#{postfix}",
+          "Ruby version: #{RUBY_DESCRIPTION}",
+          "Rdkafka version: #{::Rdkafka::VERSION}",
+          "Consumer groups count: #{Karafka::App.consumer_groups.size}",
+          "Subscription groups count: #{Karafka::App.subscription_groups.values.flatten.size}",
+          "Workers count: #{Karafka::App.config.concurrency}",
           "Application client id: #{config.client_id}",
-          "Backend: #{config.backend}",
-          "Batch fetching: #{config.batch_fetching}",
-          "Batch consuming: #{config.batch_consuming}",
           "Boot file: #{Karafka.boot_file}",
-          "Environment: #{Karafka.env}",
-          "Kafka seed brokers: #{config.kafka.seed_brokers}"
+          "Environment: #{Karafka.env}"
         ]
+      end
+
+      # @return [Array<String>] license related info
+      def license_info
+        config = Karafka::App.config
 
-        Karafka.logger.info(info.join("\n"))
+        if Karafka.pro?
+          [
+            'License: Commercial',
+            "License entity: #{config.license.entity}"
+          ]
+        else
+          [
+            'License: LGPL-3.0'
+          ]
+        end
       end
     end
   end

data/lib/karafka/cli/install.rb

@@ -4,37 +4,37 @@ require 'erb'
 
 module Karafka
   # Karafka framework Cli
-  class Cli < Thor
+  class Cli
     # Install Karafka Cli action
     class Install < Base
-      desc 'Install all required things for Karafka application in current directory'
+      include Helpers::Colorize
+
+      desc 'Installs all required things for Karafka application in current directory'
 
       # Directories created by default
       INSTALL_DIRS = %w[
         app/consumers
-        app/responders
-        app/workers
         config
-        lib
         log
-        tmp/pids
+        lib
       ].freeze
 
       # Where should we map proper files from templates
       INSTALL_FILES_MAP = {
         'karafka.rb.erb' => Karafka.boot_file.basename,
         'application_consumer.rb.erb' => 'app/consumers/application_consumer.rb',
-        'application_responder.rb.erb' => 'app/responders/application_responder.rb'
+        'example_consumer.rb.erb' => 'app/consumers/example_consumer.rb'
       }.freeze
 
-      # @param args [Array] all the things that Thor CLI accepts
-      def initialize(*args)
+      def initialize
         super
+
         dependencies = Bundler::LockfileParser.new(
           Bundler.read_file(
             Bundler.default_lockfile
           )
         ).dependencies
+
         @rails = dependencies.key?('railties') || dependencies.key?('rails')
       end
 
@@ -44,16 +44,25 @@ module Karafka
           FileUtils.mkdir_p Karafka.root.join(dir)
         end
 
+        puts
+        puts 'Installing Karafka framework...'
+        puts 'Ruby on Rails detected...' if rails?
+        puts
+
         INSTALL_FILES_MAP.each do |source, target|
-          target = Karafka.root.join(target)
+          pathed_target = Karafka.root.join(target)
 
           template = File.read(Karafka.core_root.join("templates/#{source}"))
-          # @todo Replace with the keyword argument version once we don't have to support
-          # Ruby < 2.6
-          render = ::ERB.new(template, nil, '-').result(binding)
+          render = ::ERB.new(template, trim_mode: '-').result(binding)
 
-          File.open(target, 'w') { |file| file.write(render) }
+          File.open(pathed_target, 'w') { |file| file.write(render) }
+
+          puts "#{green('Created')} #{target}"
         end
+
+        puts
+        puts("Installation #{green('completed')}. Have fun!")
+        puts
       end
 
       # @return [Boolean] true if we have Rails loaded