karafka 1.4.0 → 2.0.10

data/lib/karafka/base_consumer.rb

@@ -4,54 +4,220 @@
 module Karafka
   # Base consumer from which all Karafka consumers should inherit
   class BaseConsumer
-    extend Forwardable
+    # @return [Karafka::Routing::Topic] topic to which a given consumer is subscribed
+    attr_accessor :topic
+    # @return [Karafka::Messages::Messages] current messages batch
+    attr_accessor :messages
+    # @return [Karafka::Connection::Client] kafka connection client
+    attr_accessor :client
+    # @return [Karafka::Processing::Coordinator] coordinator
+    attr_accessor :coordinator
+    # @return [Waterdrop::Producer] producer instance
+    attr_accessor :producer
 
-    # Allows us to mark messages as consumed for non-automatic mode without having
-    # to use consumer client directly. We do this that way, because most of the people should not
-    # mess with the client instance directly (just in case)
-    %i[
-      mark_as_consumed
-      mark_as_consumed!
-      trigger_heartbeat
-      trigger_heartbeat!
-    ].each do |delegated_method_name|
-      def_delegator :client, delegated_method_name
+    # Can be used to run preparation code prior to the job being enqueued
+    #
+    # @private
+    # @note This should not be used by the end users as it is part of the lifecycle of things and
+    #   not as a part of the public api. This should not perform any extensive operations as it is
+    #   blocking and running in the listener thread.
+    def on_before_enqueue; end
 
-      private delegated_method_name
+    # Can be used to run preparation code in the worker
+    #
+    # @private
+    # @note This should not be used by the end users as it is part of the lifecycle of things and
+    #   not as part of the public api. This can act as a hook when creating non-blocking
+    #   consumers and doing other advanced stuff
+    def on_before_consume
+      messages.metadata.processed_at = Time.now
+      messages.metadata.freeze
     end
 
-    # @return [Karafka::Routing::Topic] topic to which a given consumer is subscribed
-    attr_reader :topic
-    # @return [Karafka::Params:ParamsBatch] current params batch
-    attr_accessor :params_batch
+    # Executes the default consumer flow.
+    #
+    # @return [Boolean] true if there was no exception, otherwise false.
+    #
+    # @note We keep the seek offset tracking, and use it to compensate for async offset flushing
+    #   that may not yet kick in when error occurs. That way we pause always on the last processed
+    #   message.
+    def on_consume
+      Karafka.monitor.instrument('consumer.consumed', caller: self) do
+        consume
+      end
 
-    # Assigns a topic to a consumer and builds up proper consumer functionalities
-    #   so that it can cooperate with the topic settings
-    # @param topic [Karafka::Routing::Topic]
-    def initialize(topic)
-      @topic = topic
-      Consumers::Includer.call(self)
+      coordinator.consumption(self).success!
+    rescue StandardError => e
+      coordinator.consumption(self).failure!(e)
+
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.consume.error'
+      )
+    ensure
+      # We need to decrease number of jobs that this coordinator coordinates as it has finished
+      coordinator.decrement
     end
 
-    # Executes the default consumer flow.
-    def call
-      process
+    # @private
+    # @note This should not be used by the end users as it is part of the lifecycle of things but
+    #   not as part of the public api.
+    def on_after_consume
+      return if revoked?
+
+      if coordinator.success?
+        coordinator.pause_tracker.reset
+
+        # Mark as consumed only if manual offset management is not on
+        return if topic.manual_offset_management?
+
+        # We use the non-blocking one here. If someone needs the blocking one, can implement it
+        # with manual offset management
+        mark_as_consumed(messages.last)
+      else
+        pause(coordinator.seek_offset)
+      end
     end
 
-    private
+    # Trigger method for running on partition revocation.
+    #
+    # @private
+    def on_revoked
+      # We need to always un-pause the processing in case we have lost a given partition.
+      # Otherwise the underlying librdkafka would not know we may want to continue processing and
+      # the pause could in theory last forever
+      resume
 
-    # @return [Karafka::Connection::Client] messages consuming client that can be used to
-    #    commit manually offset or pause / stop consumer based on the business logic
-    def client
-      Persistence::Client.read
+      coordinator.revoke
+
+      Karafka.monitor.instrument('consumer.revoked', caller: self) do
+        revoked
+      end
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.revoked.error'
+      )
+    end
+
+    # Trigger method for running on shutdown.
+    #
+    # @private
+    def on_shutdown
+      Karafka.monitor.instrument('consumer.shutdown', caller: self) do
+        shutdown
+      end
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.shutdown.error'
+      )
     end
 
+    private
+
     # Method that will perform business logic and on data received from Kafka (it will consume
     #   the data)
-    # @note This method needs bo be implemented in a subclass. We stub it here as a failover if
+    # @note This method needs to be implemented in a subclass. We stub it here as a failover if
     #   someone forgets about it or makes on with typo
     def consume
       raise NotImplementedError, 'Implement this in a subclass'
     end
+
+    # Method that will be executed when a given topic partition is revoked. You can use it for
+    # some teardown procedures (closing file handler, etc).
+    def revoked; end
+
+    # Method that will be executed when the process is shutting down. You can use it for
+    # some teardown procedures (closing file handler, etc).
+    def shutdown; end
+
+    # Marks message as consumed in an async way.
+    #
+    # @param message [Messages::Message] last successfully processed message.
+    # @return [Boolean] true if we were able to mark the offset, false otherwise. False indicates
+    #   that we were not able and that we have lost the partition.
+    #
+    # @note We keep track of this offset in case we would mark as consumed and got error when
+    #   processing another message. In case like this we do not pause on the message we've already
+    #   processed but rather at the next one. This applies to both sync and async versions of this
+    #   method.
+    def mark_as_consumed(message)
+      unless client.mark_as_consumed(message)
+        coordinator.revoke
+
+        return false
+      end
+
+      coordinator.seek_offset = message.offset + 1
+
+      true
+    end
+
+    # Marks message as consumed in a sync way.
+    #
+    # @param message [Messages::Message] last successfully processed message.
+    # @return [Boolean] true if we were able to mark the offset, false otherwise. False indicates
+    #   that we were not able and that we have lost the partition.
+    def mark_as_consumed!(message)
+      unless client.mark_as_consumed!(message)
+        coordinator.revoke
+
+        return false
+      end
+
+      coordinator.seek_offset = message.offset + 1
+
+      true
+    end
+
+    # Pauses processing on a given offset for the current topic partition
+    #
+    # After given partition is resumed, it will continue processing from the given offset
+    # @param offset [Integer] offset from which we want to restart the processing
+    # @param timeout [Integer, nil] how long in milliseconds do we want to pause or nil to use the
+    #   default exponential pausing strategy defined for retries
+    def pause(offset, timeout = nil)
+      timeout ? coordinator.pause_tracker.pause(timeout) : coordinator.pause_tracker.pause
+
+      client.pause(
+        messages.metadata.topic,
+        messages.metadata.partition,
+        offset
+      )
+    end
+
+    # Resumes processing of the current topic partition
+    def resume
+      # This is sufficient to expire a partition pause, as with it will be resumed by the listener
+      # thread before the next poll.
+      coordinator.pause_tracker.expire
+    end
+
+    # Seeks in the context of current topic and partition
+    #
+    # @param offset [Integer] offset where we want to seek
+    def seek(offset)
+      client.seek(
+        Karafka::Messages::Seek.new(
+          messages.metadata.topic,
+          messages.metadata.partition,
+          offset
+        )
+      )
+    end
+
+    # @return [Boolean] true if partition was revoked from the current consumer
+    # @note We know that partition got revoked because when we try to mark message as consumed,
+    #   unless if is successful, it will return false
+    def revoked?
+      coordinator.revoked?
+    end
   end
 end

data/lib/karafka/cli/info.rb

@@ -7,24 +7,58 @@ module Karafka
     class Info < Base
       desc 'Print configuration details and other options of your application'
 
+      # Nice karafka banner
+      BANNER = <<~BANNER
+
+        @@@                                             @@@@@  @@@
+        @@@                                            @@@     @@@
+        @@@  @@@    @@@@@@@@@   @@@ @@@   @@@@@@@@@  @@@@@@@@  @@@  @@@@   @@@@@@@@@
+        @@@@@@     @@@    @@@   @@@@@    @@@    @@@    @@@     @@@@@@@    @@@    @@@
+        @@@@@@@    @@@    @@@   @@@     @@@@    @@@    @@@     @@@@@@@    @@@    @@@
+        @@@  @@@@  @@@@@@@@@@   @@@      @@@@@@@@@@    @@@     @@@  @@@@   @@@@@@@@@@
+
+      BANNER
+
       # Print configuration details and other options of your application
       def call
+        Karafka.logger.info(BANNER)
+        Karafka.logger.info((core_info + license_info).join("\n"))
+      end
+
+      private
+
+      # @return [Array<String>] core framework related info
+      def core_info
         config = Karafka::App.config
 
-        info = [
-          "Karafka version: #{Karafka::VERSION}",
-          "Ruby version: #{RUBY_VERSION}",
-          "Ruby-kafka version: #{::Kafka::VERSION}",
+        postfix = Karafka.pro? ? ' + Pro' : ''
+
+        [
+          "Karafka version: #{Karafka::VERSION}#{postfix}",
+          "Ruby version: #{RUBY_DESCRIPTION}",
+          "Rdkafka version: #{::Rdkafka::VERSION}",
+          "Subscription groups count: #{Karafka::App.subscription_groups.size}",
+          "Workers count: #{Karafka::App.config.concurrency}",
           "Application client id: #{config.client_id}",
-          "Backend: #{config.backend}",
-          "Batch fetching: #{config.batch_fetching}",
-          "Batch consuming: #{config.batch_consuming}",
           "Boot file: #{Karafka.boot_file}",
-          "Environment: #{Karafka.env}",
-          "Kafka seed brokers: #{config.kafka.seed_brokers}"
+          "Environment: #{Karafka.env}"
         ]
+      end
+
+      # @return [Array<String>] license related info
+      def license_info
+        config = Karafka::App.config
 
-        Karafka.logger.info(info.join("\n"))
+        if Karafka.pro?
+          [
+            'License: Commercial',
+            "License entity: #{config.license.entity}"
+          ]
+        else
+          [
+            'License: LGPL-3.0'
+          ]
+        end
       end
     end
   end

data/lib/karafka/cli/install.rb

@@ -7,34 +7,35 @@ module Karafka
   class Cli < Thor
     # Install Karafka Cli action
     class Install < Base
+      include Helpers::Colorize
+
       desc 'Install all required things for Karafka application in current directory'
 
       # Directories created by default
       INSTALL_DIRS = %w[
         app/consumers
-        app/responders
-        app/workers
         config
-        lib
         log
-        tmp/pids
+        lib
       ].freeze
 
       # Where should we map proper files from templates
       INSTALL_FILES_MAP = {
         'karafka.rb.erb' => Karafka.boot_file.basename,
         'application_consumer.rb.erb' => 'app/consumers/application_consumer.rb',
-        'application_responder.rb.erb' => 'app/responders/application_responder.rb'
+        'example_consumer.rb.erb' => 'app/consumers/example_consumer.rb'
       }.freeze
 
       # @param args [Array] all the things that Thor CLI accepts
       def initialize(*args)
         super
-        @rails = Bundler::LockfileParser.new(
+        dependencies = Bundler::LockfileParser.new(
           Bundler.read_file(
             Bundler.default_lockfile
           )
-        ).dependencies.key?('rails')
+        ).dependencies
+
+        @rails = dependencies.key?('railties') || dependencies.key?('rails')
       end
 
       # Install all required things for Karafka application in current directory
@@ -43,16 +44,25 @@ module Karafka
           FileUtils.mkdir_p Karafka.root.join(dir)
         end
 
+        puts
+        puts 'Installing Karafka framework...'
+        puts 'Ruby on Rails detected...' if rails?
+        puts
+
         INSTALL_FILES_MAP.each do |source, target|
-          target = Karafka.root.join(target)
+          pathed_target = Karafka.root.join(target)
 
           template = File.read(Karafka.core_root.join("templates/#{source}"))
-          # @todo Replace with the keyword argument version once we don't have to support
-          # Ruby < 2.6
-          render = ::ERB.new(template, nil, '-').result(binding)
+          render = ::ERB.new(template, trim_mode: '-').result(binding)
+
+          File.open(pathed_target, 'w') { |file| file.write(render) }
 
-          File.open(target, 'w') { |file| file.write(render) }
+          puts "#{green('Created')} #{target}"
         end
+
+        puts
+        puts("Installation #{green('completed')}. Have fun!")
+        puts
       end
 
       # @return [Boolean] true if we have Rails loaded

data/lib/karafka/cli/server.rb

@@ -5,66 +5,41 @@ module Karafka
   class Cli < Thor
     # Server Karafka Cli action
     class Server < Base
-      # Server config settings contract
-      CONTRACT = Contracts::ServerCliOptions.new.freeze
-
-      private_constant :CONTRACT
+      include Helpers::Colorize
 
       desc 'Start the Karafka server (short-cut alias: "s")'
       option aliases: 's'
-      option :daemon, default: false, type: :boolean, aliases: :d
-      option :pid, default: 'tmp/pids/karafka', type: :string, aliases: :p
       option :consumer_groups, type: :array, default: nil, aliases: :g
 
       # Start the Karafka server
       def call
-        cli.info
-
-        validate!
+        # Print our banner and info in the dev mode
+        print_marketing_info if Karafka::App.env.development?
 
-        if cli.options[:daemon]
-          FileUtils.mkdir_p File.dirname(cli.options[:pid])
-          daemonize
-        end
+        Contracts::ServerCliOptions.new.validate!(cli.options)
 
         # We assign active topics on a server level, as only server is expected to listen on
         # part of the topics
         Karafka::Server.consumer_groups = cli.options[:consumer_groups]
 
-        # Remove pidfile on stop, just before the server instance is going to be GCed
-        # We want to delay the moment in which the pidfile is removed as much as we can,
-        # so instead of removing it after the server stops running, we rely on the gc moment
-        # when this object gets removed (it is a bit later), so it is closer to the actual
-        # system process end. We do that, so monitoring and deployment tools that rely on a pid
-        # won't alarm or start new system process up until the current one is finished
-        ObjectSpace.define_finalizer(self, proc { send(:clean) })
-
         Karafka::Server.run
       end
 
       private
 
-      # Checks the server cli configuration
-      # options validations in terms of app setup (topics, pid existence, etc)
-      def validate!
-        result = CONTRACT.call(cli.options)
-        return if result.success?
-
-        raise Errors::InvalidConfigurationError, result.errors.to_h
-      end
-
-      # Detaches current process into background and writes its pidfile
-      def daemonize
-        ::Process.daemon(true)
-        File.open(
-          cli.options[:pid],
-          'w'
-        ) { |file| file.write(::Process.pid) }
-      end
-
-      # Removes a pidfile (if exist)
-      def clean
-        FileUtils.rm_f(cli.options[:pid]) if cli.options[:pid]
+      # Prints marketing info
+      def print_marketing_info
+        Karafka.logger.info Info::BANNER
+
+        if Karafka.pro?
+          Karafka.logger.info(
+            green('Thank you for using Karafka Pro!')
+          )
+        else
+          Karafka.logger.info(
+            red('Upgrade to Karafka Pro for more features and support: https://karafka.io')
+          )
+        end
       end
     end
   end

data/lib/karafka/cli.rb

@@ -2,6 +2,7 @@
 
 module Karafka
   # Karafka framework Cli
+  #
   # If you want to add/modify command that belongs to CLI, please review all commands
   # available in cli/ directory inside Karafka source code.
   #
@@ -11,9 +12,9 @@ module Karafka
     package_name 'Karafka'
 
     class << self
-      # Loads all Cli commands into Thor framework
+      # Loads all Cli commands into Thor framework.
       # This method should be executed before we run Karafka::Cli.start, otherwise we won't
-      # have any Cli commands available
+      # have any Cli commands available.
       def prepare
         cli_commands.each do |action|
           action.bind_to(self)
@@ -34,7 +35,7 @@ module Karafka
   end
 end
 
-# This is kinda trick - since we don't have a autoload and other magic stuff
+# This is kinda tricky - since we don't have an autoload and other magic stuff
 # like Rails does, so instead this method allows us to replace currently running
 # console with a new one via Kernel.exec. It will start console with new code loaded
 # Yes, we know that it is not turbo fast, however it is turbo convenient and small