karafka 1.2.12 → 1.3.3

data/lib/karafka/cli.rb

@@ -37,7 +37,7 @@ end
 # This is kinda trick - since we don't have a autoload and other magic stuff
 # like Rails does, so instead this method allows us to replace currently running
 # console with a new one via Kernel.exec. It will start console with new code loaded
-# Yes we know that it is not turbofast, however it is turbo convinient and small
+# Yes, we know that it is not turbo fast, however it is turbo convenient and small
 #
 # Also - the KARAFKA_CONSOLE is used to detect that we're executing the irb session
 # so this method is only available when the Karafka console is running

data/lib/karafka/cli/console.rb

@@ -8,15 +8,17 @@ module Karafka
       desc 'Start the Karafka console (short-cut alias: "c")'
       option aliases: 'c'
 
-      # @return [String] Console executing command
-      # @example
-      #   Karafka::Cli::Console.command #=> 'KARAFKA_CONSOLE=true bundle exec irb...'
-      def self.command
-        envs = [
-          "IRBRC='#{Karafka.gem_root}/.console_irbrc'",
-          'KARAFKA_CONSOLE=true'
-        ]
-        "#{envs.join(' ')} bundle exec irb"
+      class << self
+        # @return [String] Console executing command
+        # @example
+        #   Karafka::Cli::Console.command #=> 'KARAFKA_CONSOLE=true bundle exec irb...'
+        def command
+          envs = [
+            "IRBRC='#{Karafka.gem_root}/.console_irbrc'",
+            'KARAFKA_CONSOLE=true'
+          ]
+          "#{envs.join(' ')} bundle exec irb -r #{Karafka.boot_file}"
+        end
       end
 
       # Start the Karafka console

data/lib/karafka/cli/flow.rb

@@ -18,7 +18,6 @@ module Karafka
             topic.responder.topics.each_value do |responder_topic|
               features = []
               features << (responder_topic.required? ? 'always' : 'conditionally')
-              features << (responder_topic.multiple_usage? ? 'one or more' : 'exactly once')
 
               print responder_topic.name, "(#{features.join(', ')})"
             end

data/lib/karafka/cli/info.rb

@@ -12,7 +12,9 @@ module Karafka
         config = Karafka::App.config
 
         info = [
-          "Karafka framework version: #{Karafka::VERSION}",
+          "Karafka version: #{Karafka::VERSION}",
+          "Ruby version: #{RUBY_VERSION}",
+          "Ruby-kafka version: #{::Kafka::VERSION}",
           "Application client id: #{config.client_id}",
           "Backend: #{config.backend}",
           "Batch fetching: #{config.batch_fetching}",

data/lib/karafka/cli/install.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'erb'
+
 module Karafka
   # Karafka framework Cli
   class Cli < Thor
@@ -11,18 +13,30 @@ module Karafka
       INSTALL_DIRS = %w[
         app/consumers
         app/responders
+        app/workers
         config
+        lib
         log
         tmp/pids
       ].freeze
 
       # Where should we map proper files from templates
       INSTALL_FILES_MAP = {
-        'karafka.rb.example' => Karafka.boot_file.basename,
-        'application_consumer.rb.example' => 'app/consumers/application_consumer.rb',
-        'application_responder.rb.example' => 'app/responders/application_responder.rb'
+        'karafka.rb.erb' => Karafka.boot_file.basename,
+        'application_consumer.rb.erb' => 'app/consumers/application_consumer.rb',
+        'application_responder.rb.erb' => 'app/responders/application_responder.rb'
       }.freeze
 
+      # @param args [Array] all the things that Thor CLI accepts
+      def initialize(*args)
+        super
+        @rails = Bundler::LockfileParser.new(
+          Bundler.read_file(
+            Bundler.default_lockfile
+          )
+        ).dependencies.key?('rails')
+      end
+
       # Install all required things for Karafka application in current directory
       def call
         INSTALL_DIRS.each do |dir|
@@ -31,12 +45,22 @@ module Karafka
 
         INSTALL_FILES_MAP.each do |source, target|
           target = Karafka.root.join(target)
-          next if File.exist?(target)
 
-          source = Karafka.core_root.join("templates/#{source}")
-          FileUtils.cp_r(source, target)
+          template = File.read(Karafka.core_root.join("templates/#{source}"))
+          # @todo Replace with the keyword argument version once we don't have to support
+          # Ruby < 2.6
+          render = ::ERB.new(template, nil, '-').result(binding)
+
+          File.open(target, 'w') { |file| file.write(render) }
         end
       end
+
+      # @return [Boolean] true if we have Rails loaded
+      # This allows us to generate customized karafka.rb template with some tweaks specific for
+      # Rails
+      def rails?
+        @rails
+      end
     end
   end
 end

data/lib/karafka/cli/server.rb

@@ -5,6 +5,11 @@ module Karafka
   class Cli < Thor
     # Server Karafka Cli action
     class Server < Base
+      # Server config settings contract
+      CONTRACT = Contracts::ServerCliOptions.new.freeze
+
+      private_constant :CONTRACT
+
       desc 'Start the Karafka server (short-cut alias: "s")'
       option aliases: 's'
       option :daemon, default: false, type: :boolean, aliases: :d
@@ -13,11 +18,10 @@ module Karafka
 
       # Start the Karafka server
       def call
-        validate!
-
-        puts 'Starting Karafka server'
         cli.info
 
+        validate!
+
         if cli.options[:daemon]
           FileUtils.mkdir_p File.dirname(cli.options[:pid])
           daemonize
@@ -31,7 +35,7 @@ module Karafka
         # We want to delay the moment in which the pidfile is removed as much as we can,
         # so instead of removing it after the server stops running, we rely on the gc moment
         # when this object gets removed (it is a bit later), so it is closer to the actual
-        # system process end. We do that, so monitoring and deployment tools that rely on pids
+        # system process end. We do that, so monitoring and deployment tools that rely on a pid
         # won't alarm or start new system process up until the current one is finished
         ObjectSpace.define_finalizer(self, proc { send(:clean) })
 
@@ -43,9 +47,10 @@ module Karafka
       # Checks the server cli configuration
       # options validations in terms of app setup (topics, pid existence, etc)
       def validate!
-        result = Schemas::ServerCliOptions.call(cli.options)
+        result = CONTRACT.call(cli.options)
         return if result.success?
-        raise Errors::InvalidConfiguration, result.errors
+
+        raise Errors::InvalidConfigurationError, result.errors.to_h
       end
 
       # Detaches current process into background and writes its pidfile

data/lib/karafka/code_reloader.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Special type of a listener, that is not an instrumentation one, but one that triggers
+  # code reload in the development mode after each fetched batch (or message)
+  #
+  # Please refer to the development code reload sections for details on the benefits and downsides
+  # of the in-process code reloading
+  class CodeReloader
+    # This mutex is needed as we might have an application that has multiple consumer groups
+    # running in separate threads and we should not trigger reload before fully reloading the app
+    # in previous thread
+    MUTEX = Mutex.new
+
+    private_constant :MUTEX
+
+    # @param reloaders [Array<Object>] any code loaders that we use in this app. Whether it is
+    #  the Rails loader, Zeitwerk or anything else that allows reloading triggering
+    # @param block [Proc] yields given block just before reloading. This can be used to hook custom
+    #   reloading stuff, that ain't reloaders (for example for resetting dry-events registry)
+    def initialize(*reloaders, &block)
+      @reloaders = reloaders
+      @block = block
+    end
+
+    # Binds to the instrumentation events and triggers reload
+    # @param _event [Dry::Event] empty dry event
+    # @note Since we de-register all the user defined objects and redraw routes, it means that
+    #   we won't be able to do a multi-batch buffering in the development mode as each of the
+    #   batches will be buffered on a newly created "per fetch" instance.
+    def on_connection_listener_fetch_loop(_event)
+      reload
+    end
+
+    private
+
+    # Triggers reload of both standard and Rails reloaders as well as expires all internals of
+    # Karafka, so it can be rediscovered and rebuilt
+    def reload
+      MUTEX.synchronize do
+        if @reloaders[0].respond_to?(:execute)
+          reload_with_rails
+        else
+          reload_without_rails
+        end
+      end
+    end
+
+    # Rails reloading procedure
+    def reload_with_rails
+      updatable = @reloaders.select(&:updated?)
+
+      return if updatable.empty?
+
+      updatable.each(&:execute)
+      @block&.call
+      Karafka::App.reload
+    end
+
+    # Zeitwerk and other reloaders
+    def reload_without_rails
+      @reloaders.each(&:reload)
+      @block&.call
+      Karafka::App.reload
+    end
+  end
+end

data/lib/karafka/connection/api_adapter.rb

@@ -87,7 +87,11 @@ module Karafka
           [
             Karafka::App.config.topic_mapper.outgoing(topic),
             partition,
-            { timeout: consumer_group.pause_timeout }
+            {
+              timeout: consumer_group.pause_timeout,
+              max_timeout: consumer_group.pause_max_timeout,
+              exponential_backoff: consumer_group.pause_exponential_backoff
+            }
           ]
         end
 
@@ -98,9 +102,10 @@ module Karafka
         # @note When default empty topic mapper is used, no need for any conversion as the
         #   internal and external format are exactly the same
         def mark_message_as_processed(params)
-          # Majority of non heroku users don't use custom topic mappers. No need to change
-          # anything when it is a default mapper that does not change anything
-          return [params] if Karafka::App.config.topic_mapper == Karafka::Routing::TopicMapper
+          # Majority of users don't use custom topic mappers. No need to change anything when it
+          # is a default mapper that does not change anything. Only some cloud providers require
+          # topics to be remapped
+          return [params] if Karafka::App.config.topic_mapper.is_a?(Karafka::Routing::TopicMapper)
 
           # @note We don't use tap as it is around 13% slower than non-dup version
           dupped = params.dup
@@ -119,9 +124,11 @@ module Karafka
           kafka_configs.each_key do |setting_name|
             # Ignore settings that are not related to our namespace
             next unless AttributesMap.api_adapter[namespace_key].include?(setting_name)
+
             # Ignore settings that are already initialized
             # In case they are in preexisting settings fetched differently
             next if preexisting_settings.key?(setting_name)
+
             # Fetch all the settings from a given layer object. Objects can handle the fallback
             # to the kafka settings, so
             preexisting_settings[setting_name] = route_layer.send(setting_name)

data/lib/karafka/connection/batch_delegator.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Class that delegates processing of batch received messages for which we listen to
+    # a proper processor
+    module BatchDelegator
+      class << self
+        # Delegates messages (does something with them)
+        # It will either schedule or run a proper processor action for messages
+        # @param group_id [String] group_id of a group from which a given message came
+        # @param kafka_batch [<Kafka::FetchedBatch>] raw messages fetched batch
+        # @note This should be looped to obtain a constant delegating of new messages
+        def call(group_id, kafka_batch)
+          topic = Persistence::Topics.fetch(group_id, kafka_batch.topic)
+          consumer = Persistence::Consumers.fetch(topic, kafka_batch.partition)
+
+          Karafka.monitor.instrument(
+            'connection.batch_delegator.call',
+            caller: self,
+            consumer: consumer,
+            kafka_batch: kafka_batch
+          ) do
+            # Due to how ruby-kafka is built, we have the metadata that is stored on the batch
+            # level only available for batch consuming
+            consumer.metadata = Params::Builders::Metadata.from_kafka_batch(kafka_batch, topic)
+            kafka_messages = kafka_batch.messages
+
+            # Depending on a case (persisted or not) we might use new consumer instance per
+            # each batch, or use the same one for all of them (for implementing buffering, etc.)
+            if topic.batch_consuming
+              consumer.params_batch = Params::Builders::ParamsBatch.from_kafka_messages(
+                kafka_messages,
+                topic
+              )
+              consumer.call
+            else
+              kafka_messages.each do |kafka_message|
+                consumer.params_batch = Params::Builders::ParamsBatch.from_kafka_messages(
+                  [kafka_message],
+                  topic
+                )
+                consumer.call
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/connection/builder.rb

@@ -5,7 +5,7 @@ module Karafka
     # Builder used to construct Kafka client
     module Builder
       class << self
-        # Builds a Kafka::Cient instance that we use to work with Kafka cluster
+        # Builds a Kafka::Client instance that we use to work with Kafka cluster
         # @return [::Kafka::Client] returns a Kafka client
         def call
           Kafka.new(*ApiAdapter.client)

data/lib/karafka/connection/client.rb

@@ -7,7 +7,13 @@ module Karafka
     class Client
       extend Forwardable
 
-      def_delegator :kafka_consumer, :seek
+      %i[
+        seek
+        trigger_heartbeat
+        trigger_heartbeat!
+      ].each do |delegated_method|
+        def_delegator :kafka_consumer, delegated_method
+      end
 
       # Creates a queue consumer client that will pull the data from Kafka
       # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group for which
@@ -20,30 +26,32 @@ module Karafka
       end
 
       # Opens connection, gets messages and calls a block for each of the incoming messages
-      # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
+      # @yieldparam [Array<Kafka::FetchedMessage>, Symbol] kafka response with an info about
+      #   the type of the fetcher that is being used
       # @note This will yield with raw messages - no preprocessing or reformatting.
       def fetch_loop
         settings = ApiAdapter.consumption(consumer_group)
 
         if consumer_group.batch_fetching
-          kafka_consumer.each_batch(*settings) { |batch| yield(batch.messages) }
+          kafka_consumer.each_batch(*settings) { |batch| yield(batch, :batch) }
         else
-          # always yield an array of messages, so we have consistent API (always a batch)
-          kafka_consumer.each_message(*settings) { |message| yield([message]) }
+          kafka_consumer.each_message(*settings) { |message| yield(message, :message) }
         end
-      rescue Kafka::ProcessingError => error
+      # @note We catch only the processing errors as any other are considered critical (exceptions)
+      #   and should require a client restart with a backoff
+      rescue Kafka::ProcessingError => e
         # If there was an error during consumption, we have to log it, pause current partition
         # and process other things
         Karafka.monitor.instrument(
           'connection.client.fetch_loop.error',
           caller: self,
-          error: error.cause
+          error: e.cause
         )
-        pause(error.topic, error.partition)
+        pause(e.topic, e.partition)
         retry
       end
 
-      # Gracefuly stops topic consumption
+      # Gracefully stops topic consumption
       # @note Stopping running consumers without a really important reason is not recommended
       #   as until all the consumers are stopped, the server will keep running serving only
       #   part of the messages
@@ -59,25 +67,27 @@ module Karafka
         kafka_consumer.pause(*ApiAdapter.pause(topic, partition, consumer_group))
       end
 
-      # Marks a given message as consumed and commit the offsets
-      # @note In opposite to ruby-kafka, we commit the offset for each manual marking to be sure
-      #   that offset commit happen asap in case of a crash
+      # Marks given message as consumed
       # @param [Karafka::Params::Params] params message that we want to mark as processed
+      # @note This method won't trigger automatic offsets commits, rather relying on the ruby-kafka
+      #   offsets time-interval based committing
       def mark_as_consumed(params)
         kafka_consumer.mark_message_as_processed(
           *ApiAdapter.mark_message_as_processed(params)
         )
+      end
+
+      # Marks a given message as consumed and commit the offsets in a blocking way
+      # @param [Karafka::Params::Params] params message that we want to mark as processed
+      # @note This method commits the offset for each manual marking to be sure
+      #   that offset commit happen asap in case of a crash
+      def mark_as_consumed!(params)
+        mark_as_consumed(params)
         # Trigger an immediate, blocking offset commit in order to minimize the risk of crashing
         # before the automatic triggers have kicked in.
         kafka_consumer.commit_offsets
       end
 
-      # Triggers a non-optional blocking heartbeat that notifies Kafka about the fact, that this
-      # consumer / client is still up and running
-      def trigger_heartbeat
-        kafka_consumer.trigger_heartbeat!
-      end
-
       private
 
       attr_reader :consumer_group
@@ -95,10 +105,10 @@ module Karafka
           end
         end
       rescue Kafka::ConnectionError
-        # If we would not wait it would totally spam log file with failed
+        # If we would not wait it will spam log file with failed
         # attempts if Kafka is down
         sleep(consumer_group.reconnect_timeout)
-        # We don't log and just reraise - this will be logged
+        # We don't log and just re-raise - this will be logged
         # down the road
         raise
       end

data/lib/karafka/connection/listener.rb

@@ -16,9 +16,10 @@ module Karafka
 
       # Runs prefetch callbacks and executes the main listener fetch loop
       def call
-        Karafka::Callbacks.before_fetch_loop(
-          @consumer_group,
-          client
+        Karafka.monitor.instrument(
+          'connection.listener.before_fetch_loop',
+          consumer_group: @consumer_group,
+          client: client
         )
         fetch_loop
       end
@@ -26,27 +27,37 @@ module Karafka
       private
 
       # Opens connection, gets messages and calls a block for each of the incoming messages
-      # @yieldparam [String] consumer group id
-      # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
-      # @note This will yield with a raw message - no preprocessing or reformatting
       # @note We catch all the errors here, so they don't affect other listeners (or this one)
       #   so we will be able to listen and consume other incoming messages.
       #   Since it is run inside Karafka::Connection::ActorCluster - catching all the exceptions
-      #   won't crash the whole cluster. Here we mostly focus on catchin the exceptions related to
+      #   won't crash the whole cluster. Here we mostly focus on catching the exceptions related to
       #   Kafka connections / Internet connection issues / Etc. Business logic problems should not
       #   propagate this far
       def fetch_loop
-        client.fetch_loop do |raw_messages|
-          # @note What happens here is a delegation of processing to a proper processor based
-          #   on the incoming messages characteristics
-          Karafka::Connection::Delegator.call(@consumer_group.id, raw_messages)
+        # @note What happens here is a delegation of processing to a proper processor based
+        #   on the incoming messages characteristics
+        client.fetch_loop do |raw_data, type|
+          Karafka.monitor.instrument('connection.listener.fetch_loop')
+
+          case type
+          when :message
+            MessageDelegator.call(@consumer_group.id, raw_data)
+          when :batch
+            BatchDelegator.call(@consumer_group.id, raw_data)
+          end
         end
         # This is on purpose - see the notes for this method
-        # rubocop:disable RescueException
+        # rubocop:disable Lint/RescueException
       rescue Exception => e
         Karafka.monitor.instrument('connection.listener.fetch_loop.error', caller: self, error: e)
-        # rubocop:enable RescueException
+        # rubocop:enable Lint/RescueException
+        # We can stop client without a problem, as it will reinitialize itself when running the
+        # `fetch_loop` again
         @client.stop
+        # We need to clear the consumers cache for current connection when fatal error happens and
+        # we reset the connection. Otherwise for consumers with manual offset management, the
+        # persistence might have stored some data that would be reprocessed
+        Karafka::Persistence::Consumers.clear
         sleep(@consumer_group.reconnect_timeout) && retry
       end