karafka 1.4.12 → 2.0.0.alpha1

data/lib/karafka/base_consumer.rb

@@ -4,48 +4,80 @@
 module Karafka
   # Base consumer from which all Karafka consumers should inherit
   class BaseConsumer
-    extend Forwardable
-
-    # Allows us to mark messages as consumed for non-automatic mode without having
-    # to use consumer client directly. We do this that way, because most of the people should not
-    # mess with the client instance directly (just in case)
-    %i[
-      mark_as_consumed
-      mark_as_consumed!
-      trigger_heartbeat
-      trigger_heartbeat!
-    ].each do |delegated_method_name|
-      def_delegator :client, delegated_method_name
-
-      private delegated_method_name
-    end
-
     # @return [Karafka::Routing::Topic] topic to which a given consumer is subscribed
-    attr_reader :topic
-    # @return [Karafka::Params:ParamsBatch] current params batch
-    attr_accessor :params_batch
-
-    # Assigns a topic to a consumer and builds up proper consumer functionalities
-    #   so that it can cooperate with the topic settings
-    # @param topic [Karafka::Routing::Topic]
-    def initialize(topic)
-      @topic = topic
-      Consumers::Includer.call(self)
-    end
+    attr_accessor :topic
+    # @return [Karafka::Messages::Messages] current messages batch
+    attr_accessor :messages
+    # @return [Karafka::Connection::Client] kafka connection client
+    attr_accessor :client
+    # @return [Karafka::TimeTrackers::Pause] current topic partition pause
+    attr_accessor :pause
+    # @return [Waterdrop::Producer] producer instance
+    attr_accessor :producer
 
     # Executes the default consumer flow.
-    def call
-      process
+    #
+    # @note We keep the seek offset tracking, and use it to compensate for async offset flushing
+    #   that may not yet kick in when error occurs. That way we pause always on the last processed
+    #   message.
+    def on_consume
+      Karafka.monitor.instrument('consumer.consume', caller: self) do
+        consume
+      end
+
+      pause.reset
+
+      # Mark as consumed only if manual offset management is not on
+      return if topic.manual_offset_management
+
+      # We use the non-blocking one here. If someone needs the blocking one, can implement it with
+      # manual offset management
+      mark_as_consumed(messages.last)
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.consume.error'
+      )
+      client.pause(topic.name, messages.first.partition, @seek_offset || messages.first.offset)
+      pause.pause
     end
 
-    private
+    # Trigger method for running on shutdown.
+    #
+    # @private
+    def on_revoked
+      Karafka.monitor.instrument('consumer.revoked', caller: self) do
+        revoked
+      end
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.revoked.error'
+      )
+    end
 
-    # @return [Karafka::Connection::Client] messages consuming client that can be used to
-    #    commit manually offset or pause / stop consumer based on the business logic
-    def client
-      Persistence::Client.read
+    # Trigger method for running on shutdown.
+    #
+    # @private
+    def on_shutdown
+      Karafka.monitor.instrument('consumer.shutdown', caller: self) do
+        shutdown
+      end
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.shutdown.error'
+      )
     end
 
+    private
+
     # Method that will perform business logic and on data received from Kafka (it will consume
     #   the data)
     # @note This method needs bo be implemented in a subclass. We stub it here as a failover if
@@ -53,5 +85,42 @@ module Karafka
     def consume
       raise NotImplementedError, 'Implement this in a subclass'
     end
+
+    # Method that will be executed when a given topic partition is revoked. You can use it for
+    # some teardown procedures (closing file handler, etc).
+    def revoked; end
+
+    # Method that will be executed when the process is shutting down. You can use it for
+    # some teardown procedures (closing file handler, etc).
+    def shutdown; end
+
+    # Marks message as consumed in an async way.
+    #
+    # @param message [Messages::Message] last successfully processed message.
+    def mark_as_consumed(message)
+      client.mark_as_consumed(message)
+      @seek_offset = message.offset + 1
+    end
+
+    # Marks message as consumed in a sync way.
+    #
+    # @param message [Messages::Message] last successfully processed message.
+    def mark_as_consumed!(message)
+      client.mark_as_consumed!(message)
+      @seek_offset = message.offset + 1
+    end
+
+    # Seeks in the context of current topic and partition
+    #
+    # @param offset [Integer] offset where we want to seek
+    def seek(offset)
+      client.seek(
+        Karafka::Messages::Seek.new(
+          messages.metadata.topic,
+          messages.metadata.partition,
+          offset
+        )
+      )
+    end
   end
 end

data/lib/karafka/cli/base.rb

@@ -43,16 +43,16 @@ module Karafka
         end
 
         # Allows to set description of a given cli command
-        # @param args [Array] All the arguments that Thor desc method accepts
-        def desc(*args)
-          @desc ||= args
+        # @param desc [String] Description of a given cli command
+        def desc(desc)
+          @desc ||= desc
         end
 
         # This method will bind a given Cli command into Karafka Cli
         # This method is a wrapper to way Thor defines its commands
         # @param cli_class [Karafka::Cli] Karafka cli_class
         def bind_to(cli_class)
-          cli_class.desc name, *@desc
+          cli_class.desc name, @desc
 
           (@options || []).each { |option| cli_class.option(*option) }
 

data/lib/karafka/cli/info.rb

@@ -7,24 +7,59 @@ module Karafka
     class Info < Base
       desc 'Print configuration details and other options of your application'
 
+      # Nice karafka banner
+      BANNER = <<~BANNER
+
+        @@@                                             @@@@@  @@@
+        @@@                                            @@@     @@@
+        @@@  @@@    @@@@@@@@@   @@@ @@@   @@@@@@@@@  @@@@@@@@  @@@  @@@@   @@@@@@@@@
+        @@@@@@     @@@    @@@   @@@@@    @@@    @@@    @@@     @@@@@@@    @@@    @@@
+        @@@@@@@    @@@    @@@   @@@     @@@@    @@@    @@@     @@@@@@@    @@@    @@@
+        @@@  @@@@  @@@@@@@@@@   @@@      @@@@@@@@@@    @@@     @@@  @@@@   @@@@@@@@@@
+
+      BANNER
+
       # Print configuration details and other options of your application
       def call
+        Karafka.logger.info(BANNER)
+        Karafka.logger.info((core_info + license_info).join("\n"))
+      end
+
+      private
+
+      # @return [Array<String>] core framework related info
+      def core_info
         config = Karafka::App.config
 
+        postfix = config.license.token ? ' + Pro' : ''
+
         info = [
-          "Karafka version: #{Karafka::VERSION}",
+          "Karafka version: #{Karafka::VERSION}#{postfix}",
           "Ruby version: #{RUBY_VERSION}",
-          "Ruby-kafka version: #{::Kafka::VERSION}",
+          "Rdkafka version: #{::Rdkafka::VERSION}",
+          "Subscription groups count: #{Karafka::App.subscription_groups.size}",
+          "Workers count: #{Karafka::App.config.concurrency}",
           "Application client id: #{config.client_id}",
-          "Backend: #{config.backend}",
-          "Batch fetching: #{config.batch_fetching}",
-          "Batch consuming: #{config.batch_consuming}",
           "Boot file: #{Karafka.boot_file}",
-          "Environment: #{Karafka.env}",
-          "Kafka seed brokers: #{config.kafka.seed_brokers}"
+          "Environment: #{Karafka.env}"
         ]
+      end
+
+      # @return [Array<String>] license related info
+      def license_info
+        config = Karafka::App.config
 
-        Karafka.logger.info(info.join("\n"))
+        if config.license.token
+          [
+            'License: Commercial',
+            "License entity: #{config.license.entity}",
+            "License expires on: #{config.license.expires_on}"
+          ]
+        else
+          [
+            'License: LGPL-3.0'
+          ]
+        end
       end
     end
   end

data/lib/karafka/cli/install.rb

@@ -12,19 +12,15 @@ module Karafka
       # Directories created by default
       INSTALL_DIRS = %w[
         app/consumers
-        app/responders
-        app/workers
         config
-        lib
         log
-        tmp/pids
       ].freeze
 
       # Where should we map proper files from templates
       INSTALL_FILES_MAP = {
         'karafka.rb.erb' => Karafka.boot_file.basename,
         'application_consumer.rb.erb' => 'app/consumers/application_consumer.rb',
-        'application_responder.rb.erb' => 'app/responders/application_responder.rb'
+        'example_consumer.rb.erb' => 'app/consumers/example_consumer.rb'
       }.freeze
 
       # @param args [Array] all the things that Thor CLI accepts
@@ -35,6 +31,7 @@ module Karafka
             Bundler.default_lockfile
           )
         ).dependencies
+
         @rails = dependencies.key?('railties') || dependencies.key?('rails')
       end
 
@@ -48,9 +45,7 @@ module Karafka
           target = Karafka.root.join(target)
 
           template = File.read(Karafka.core_root.join("templates/#{source}"))
-          # @todo Replace with the keyword argument version once we don't have to support
-          # Ruby < 2.6
-          render = ::ERB.new(template, nil, '-').result(binding)
+          render = ::ERB.new(template, trim_mode: '-').result(binding)
 
           File.open(target, 'w') { |file| file.write(render) }
         end

data/lib/karafka/cli/server.rb

@@ -12,21 +12,15 @@ module Karafka
 
       desc 'Start the Karafka server (short-cut alias: "s")'
       option aliases: 's'
-      option :daemon, default: false, type: :boolean, aliases: :d
-      option :pid, default: 'tmp/pids/karafka', type: :string, aliases: :p
       option :consumer_groups, type: :array, default: nil, aliases: :g
 
       # Start the Karafka server
       def call
-        cli.info
+        # Print our banner and info in the dev mode
+        print_marketing_info if Karafka::App.env.development?
 
         validate!
 
-        if cli.options[:daemon]
-          FileUtils.mkdir_p File.dirname(cli.options[:pid])
-          daemonize
-        end
-
         # We assign active topics on a server level, as only server is expected to listen on
         # part of the topics
         Karafka::Server.consumer_groups = cli.options[:consumer_groups]
@@ -36,6 +30,21 @@ module Karafka
 
       private
 
+      # Prints marketing info
+      def print_marketing_info
+        Karafka.logger.info Info::BANNER
+
+        if Karafka::App.config.license.token
+          Karafka.logger.info(
+            "\033[0;32mThank you for investing in the Karafka Pro subscription!\033[0m\n"
+          )
+        else
+          Karafka.logger.info(
+            "\033[0;31mYou like Karafka? Please consider getting a Pro subscription!\033[0m\n"
+          )
+        end
+      end
+
       # Checks the server cli configuration
       # options validations in terms of app setup (topics, pid existence, etc)
       def validate!
@@ -44,28 +53,6 @@ module Karafka
 
         raise Errors::InvalidConfigurationError, result.errors.to_h
       end
-
-      # Detaches current process into background and writes its pidfile
-      def daemonize
-        ::Process.daemon(true)
-        File.open(
-          cli.options[:pid],
-          'w'
-        ) { |file| file.write(::Process.pid) }
-
-        # Remove pidfile on stop, just before the server instance is going to be GCed
-        # We want to delay the moment in which the pidfile is removed as much as we can,
-        # so instead of removing it after the server stops running, we rely on the gc moment
-        # when this object gets removed (it is a bit later), so it is closer to the actual
-        # system process end. We do that, so monitoring and deployment tools that rely on a pid
-        # won't alarm or start new system process up until the current one is finished
-        ObjectSpace.define_finalizer(self, proc { send(:clean) })
-      end
-
-      # Removes a pidfile (if exist)
-      def clean
-        FileUtils.rm_f(cli.options[:pid]) if cli.options[:pid]
-      end
     end
   end
 end

data/lib/karafka/cli.rb

@@ -2,6 +2,7 @@
 
 module Karafka
   # Karafka framework Cli
+  #
   # If you want to add/modify command that belongs to CLI, please review all commands
   # available in cli/ directory inside Karafka source code.
   #
@@ -10,24 +11,16 @@ module Karafka
   class Cli < Thor
     package_name 'Karafka'
 
-    default_task :missingno
-
     class << self
-      # Loads all Cli commands into Thor framework
+      # Loads all Cli commands into Thor framework.
       # This method should be executed before we run Karafka::Cli.start, otherwise we won't
-      # have any Cli commands available
+      # have any Cli commands available.
       def prepare
         cli_commands.each do |action|
           action.bind_to(self)
         end
       end
 
-      # When there is a CLI crash, exit
-      # @return [true]
-      def exit_on_failure?
-        true
-      end
-
       private
 
       # @return [Array<Class>] Array with Cli action classes that can be used as commands
@@ -42,7 +35,7 @@ module Karafka
   end
 end
 
-# This is kinda trick - since we don't have a autoload and other magic stuff
+# This is kinda tricky - since we don't have an autoload and other magic stuff
 # like Rails does, so instead this method allows us to replace currently running
 # console with a new one via Kernel.exec. It will start console with new code loaded
 # Yes, we know that it is not turbo fast, however it is turbo convenient and small