karafka 1.2.2 → 1.4.0.rc1

data/lib/karafka/cli/flow.rb

@@ -11,20 +11,22 @@ module Karafka
       def call
         topics.each do |topic|
           any_topics = !topic.responder&.topics.nil?
+          log_messages = []
 
           if any_topics
-            puts "#{topic.name} =>"
+            log_messages << "#{topic.name} =>"
 
             topic.responder.topics.each_value do |responder_topic|
               features = []
               features << (responder_topic.required? ? 'always' : 'conditionally')
-              features << (responder_topic.multiple_usage? ? 'one or more' : 'exactly once')
 
-              print responder_topic.name, "(#{features.join(', ')})"
+              log_messages << format(responder_topic.name, "(#{features.join(', ')})")
             end
           else
-            puts "#{topic.name} => (nothing)"
+            log_messages << "#{topic.name} => (nothing)"
           end
+
+          Karafka.logger.info(log_messages.join("\n"))
         end
       end
 
@@ -35,11 +37,11 @@ module Karafka
         Karafka::App.consumer_groups.map(&:topics).flatten.sort_by(&:name)
       end
 
-      # Prints a given value with label in a nice way
+      # Formats a given value with label in a nice way
       # @param label [String] label describing value
       # @param value [String] value that should be printed
-      def print(label, value)
-        printf "%-25s %s\n", "  - #{label}:", value
+      def format(label, value)
+        "  - #{label}:                #{value}"
       end
     end
   end

data/lib/karafka/cli/info.rb

@@ -12,7 +12,9 @@ module Karafka
         config = Karafka::App.config
 
         info = [
-          "Karafka framework version: #{Karafka::VERSION}",
+          "Karafka version: #{Karafka::VERSION}",
+          "Ruby version: #{RUBY_VERSION}",
+          "Ruby-kafka version: #{::Kafka::VERSION}",
           "Application client id: #{config.client_id}",
           "Backend: #{config.backend}",
           "Batch fetching: #{config.batch_fetching}",
@@ -22,7 +24,7 @@ module Karafka
           "Kafka seed brokers: #{config.kafka.seed_brokers}"
         ]
 
-        puts(info.join("\n"))
+        Karafka.logger.info(info.join("\n"))
       end
     end
   end

data/lib/karafka/cli/install.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'erb'
+
 module Karafka
   # Karafka framework Cli
   class Cli < Thor
@@ -11,18 +13,30 @@ module Karafka
       INSTALL_DIRS = %w[
         app/consumers
         app/responders
+        app/workers
         config
+        lib
         log
         tmp/pids
       ].freeze
 
       # Where should we map proper files from templates
       INSTALL_FILES_MAP = {
-        'karafka.rb.example' => Karafka.boot_file.basename,
-        'application_consumer.rb.example' => 'app/consumers/application_consumer.rb',
-        'application_responder.rb.example' => 'app/responders/application_responder.rb'
+        'karafka.rb.erb' => Karafka.boot_file.basename,
+        'application_consumer.rb.erb' => 'app/consumers/application_consumer.rb',
+        'application_responder.rb.erb' => 'app/responders/application_responder.rb'
       }.freeze
 
+      # @param args [Array] all the things that Thor CLI accepts
+      def initialize(*args)
+        super
+        @rails = Bundler::LockfileParser.new(
+          Bundler.read_file(
+            Bundler.default_lockfile
+          )
+        ).dependencies.key?('rails')
+      end
+
       # Install all required things for Karafka application in current directory
       def call
         INSTALL_DIRS.each do |dir|
@@ -31,12 +45,22 @@ module Karafka
 
         INSTALL_FILES_MAP.each do |source, target|
           target = Karafka.root.join(target)
-          next if File.exist?(target)
 
-          source = Karafka.core_root.join("templates/#{source}")
-          FileUtils.cp_r(source, target)
+          template = File.read(Karafka.core_root.join("templates/#{source}"))
+          # @todo Replace with the keyword argument version once we don't have to support
+          # Ruby < 2.6
+          render = ::ERB.new(template, nil, '-').result(binding)
+
+          File.open(target, 'w') { |file| file.write(render) }
         end
       end
+
+      # @return [Boolean] true if we have Rails loaded
+      # This allows us to generate customized karafka.rb template with some tweaks specific for
+      # Rails
+      def rails?
+        @rails
+      end
     end
   end
 end

data/lib/karafka/cli/server.rb

@@ -5,6 +5,11 @@ module Karafka
   class Cli < Thor
     # Server Karafka Cli action
     class Server < Base
+      # Server config settings contract
+      CONTRACT = Contracts::ServerCliOptions.new.freeze
+
+      private_constant :CONTRACT
+
       desc 'Start the Karafka server (short-cut alias: "s")'
       option aliases: 's'
       option :daemon, default: false, type: :boolean, aliases: :d
@@ -13,11 +18,10 @@ module Karafka
 
       # Start the Karafka server
       def call
-        validate!
-
-        puts 'Starting Karafka server'
         cli.info
 
+        validate!
+
         if cli.options[:daemon]
           FileUtils.mkdir_p File.dirname(cli.options[:pid])
           daemonize
@@ -31,7 +35,7 @@ module Karafka
         # We want to delay the moment in which the pidfile is removed as much as we can,
         # so instead of removing it after the server stops running, we rely on the gc moment
         # when this object gets removed (it is a bit later), so it is closer to the actual
-        # system process end. We do that, so monitoring and deployment tools that rely on pids
+        # system process end. We do that, so monitoring and deployment tools that rely on a pid
         # won't alarm or start new system process up until the current one is finished
         ObjectSpace.define_finalizer(self, proc { send(:clean) })
 
@@ -43,9 +47,10 @@ module Karafka
       # Checks the server cli configuration
       # options validations in terms of app setup (topics, pid existence, etc)
       def validate!
-        result = Schemas::ServerCliOptions.call(cli.options)
+        result = CONTRACT.call(cli.options)
         return if result.success?
-        raise Errors::InvalidConfiguration, result.errors
+
+        raise Errors::InvalidConfigurationError, result.errors.to_h
       end
 
       # Detaches current process into background and writes its pidfile

data/lib/karafka/code_reloader.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Special type of a listener, that is not an instrumentation one, but one that triggers
+  # code reload in the development mode after each fetched batch (or message)
+  #
+  # Please refer to the development code reload sections for details on the benefits and downsides
+  # of the in-process code reloading
+  class CodeReloader
+    # This mutex is needed as we might have an application that has multiple consumer groups
+    # running in separate threads and we should not trigger reload before fully reloading the app
+    # in previous thread
+    MUTEX = Mutex.new
+
+    private_constant :MUTEX
+
+    # @param reloaders [Array<Object>] any code loaders that we use in this app. Whether it is
+    #  the Rails loader, Zeitwerk or anything else that allows reloading triggering
+    # @param block [Proc] yields given block just before reloading. This can be used to hook custom
+    #   reloading stuff, that ain't reloaders (for example for resetting dry-events registry)
+    def initialize(*reloaders, &block)
+      @reloaders = reloaders
+      @block = block
+    end
+
+    # Binds to the instrumentation events and triggers reload
+    # @param _event [Dry::Event] empty dry event
+    # @note Since we de-register all the user defined objects and redraw routes, it means that
+    #   we won't be able to do a multi-batch buffering in the development mode as each of the
+    #   batches will be buffered on a newly created "per fetch" instance.
+    def on_connection_listener_fetch_loop(_event)
+      reload
+    end
+
+    private
+
+    # Triggers reload of both standard and Rails reloaders as well as expires all internals of
+    # Karafka, so it can be rediscovered and rebuilt
+    def reload
+      MUTEX.synchronize do
+        if @reloaders[0].respond_to?(:execute)
+          reload_with_rails
+        else
+          reload_without_rails
+        end
+      end
+    end
+
+    # Rails reloading procedure
+    def reload_with_rails
+      updatable = @reloaders.select(&:updated?)
+
+      return if updatable.empty?
+
+      updatable.each(&:execute)
+      @block&.call
+      Karafka::App.reload
+    end
+
+    # Zeitwerk and other reloaders
+    def reload_without_rails
+      @reloaders.each(&:reload)
+      @block&.call
+      Karafka::App.reload
+    end
+  end
+end

data/lib/karafka/connection/{config_adapter.rb → api_adapter.rb}

@@ -3,22 +3,23 @@
 module Karafka
   # Namespace for all the things related to Kafka connection
   module Connection
-    # Mapper used to convert our internal settings into ruby-kafka settings
+    # Mapper used to convert our internal settings into ruby-kafka settings based on their
+    # API requirements.
     # Since ruby-kafka has more and more options and there are few "levels" on which
     # we have to apply them (despite the fact, that in Karafka you configure all of it
     # in one place), we have to remap it into what ruby-kafka driver requires
     # @note The good thing about Kafka.new method is that it ignores all options that
     #   do nothing. So we don't have to worry about injecting our internal settings
     #   into the client and breaking stuff
-    module ConfigAdapter
+    module ApiAdapter
       class << self
         # Builds all the configuration settings for Kafka.new method
-        # @param _consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
+        # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
         # @return [Array<Hash>] Array with all the client arguments including hash with all
         #   the settings required by Kafka.new method
         # @note We return array, so we can inject any arguments we want, in case of changes in the
         #   raw driver
-        def client(_consumer_group)
+        def client(consumer_group)
           # This one is a default that takes all the settings except special
           # cases defined in the map
           settings = {
@@ -26,14 +27,17 @@ module Karafka
             client_id: ::Karafka::App.config.client_id
           }
 
-          kafka_configs.each do |setting_name, setting_value|
+          kafka_configs.each_key do |setting_name|
             # All options for config adapter should be ignored as we're just interested
             # in what is left, as we want to pass all the options that are "typical"
-            # and not listed in the config_adapter special cases mapping. All the values
-            # from the config_adapter mapping go somewhere else, not to the client directly
-            next if AttributesMap.config_adapter.values.flatten.include?(setting_name)
+            # and not listed in the api_adapter special cases mapping. All the values
+            # from the api_adapter mapping go somewhere else, not to the client directly
+            next if AttributesMap.api_adapter.values.flatten.include?(setting_name)
 
-            settings[setting_name] = setting_value
+            # Settings for each consumer group are either defined per consumer group or are
+            # inherited from the global/general settings level, thus we don't have to fetch them
+            # from the kafka settings as they are already on a consumer group level
+            settings[setting_name] = consumer_group.public_send(setting_name)
           end
 
           settings_hash = sanitize(settings)
@@ -58,26 +62,61 @@ module Karafka
         # @return [Array<Hash>] Array with all the arguments required by consuming method
         #   including hash with all the settings required by
         #   Kafka::Consumer#consume_each_message and Kafka::Consumer#consume_each_batch method
-        def consuming(consumer_group)
-          settings = {
-            automatically_mark_as_processed: consumer_group.automatically_mark_as_consumed
-          }
-          [sanitize(fetch_for(:consuming, consumer_group, settings))]
+        def consumption(consumer_group)
+          [
+            sanitize(
+              fetch_for(
+                :consumption,
+                consumer_group,
+                automatically_mark_as_processed: consumer_group.automatically_mark_as_consumed
+              )
+            )
+          ]
         end
 
         # Builds all the configuration settings for kafka consumer#subscribe method
         # @param topic [Karafka::Routing::Topic] topic that holds details for a given subscription
         # @return [Hash] hash with all the settings required by kafka consumer#subscribe method
-        def subscription(topic)
-          settings = fetch_for(:subscription, topic)
+        def subscribe(topic)
+          settings = fetch_for(:subscribe, topic)
           [Karafka::App.config.topic_mapper.outgoing(topic.name), sanitize(settings)]
         end
 
         # Builds all the configuration settings required by kafka consumer#pause method
+        # @param topic [String] topic that we want to pause
+        # @param partition [Integer] number partition that we want to pause
         # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
-        # @return [Hash] hash with all the settings required to pause kafka consumer
-        def pausing(consumer_group)
-          { timeout: consumer_group.pause_timeout }
+        # @return [Array] array with all the details required to pause kafka consumer
+        def pause(topic, partition, consumer_group)
+          [
+            Karafka::App.config.topic_mapper.outgoing(topic),
+            partition,
+            {
+              timeout: consumer_group.pause_timeout,
+              max_timeout: consumer_group.pause_max_timeout,
+              exponential_backoff: consumer_group.pause_exponential_backoff
+            }
+          ]
+        end
+
+        # Remaps topic details taking the topic mapper feature into consideration.
+        # @param params [Karafka::Params::Params] params instance
+        # @return [Array] array with all the details needed by ruby-kafka to mark message
+        #   as processed
+        # @note When default empty topic mapper is used, no need for any conversion as the
+        #   internal and external format are exactly the same
+        def mark_message_as_processed(params)
+          # Majority of users don't use custom topic mappers. No need to change anything when it
+          # is a default mapper that does not change anything. Only some cloud providers require
+          # topics to be remapped
+          return [params.metadata] if Karafka::App.config.topic_mapper.is_a?(
+            Karafka::Routing::TopicMapper
+          )
+
+          # @note We don't use tap as it is around 13% slower than non-dup version
+          dupped = params.metadata.dup
+          dupped['topic'] = Karafka::App.config.topic_mapper.outgoing(params.metadata.topic)
+          [dupped]
         end
 
         private
@@ -90,10 +129,12 @@ module Karafka
         def fetch_for(namespace_key, route_layer, preexisting_settings = {})
           kafka_configs.each_key do |setting_name|
             # Ignore settings that are not related to our namespace
-            next unless AttributesMap.config_adapter[namespace_key].include?(setting_name)
+            next unless AttributesMap.api_adapter[namespace_key].include?(setting_name)
+
             # Ignore settings that are already initialized
             # In case they are in preexisting settings fetched differently
-            next if preexisting_settings.keys.include?(setting_name)
+            next if preexisting_settings.key?(setting_name)
+
             # Fetch all the settings from a given layer object. Objects can handle the fallback
             # to the kafka settings, so
             preexisting_settings[setting_name] = route_layer.send(setting_name)

data/lib/karafka/connection/batch_delegator.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Class that delegates processing of batch received messages for which we listen to
+    # a proper processor
+    module BatchDelegator
+      class << self
+        # Delegates messages (does something with them)
+        # It will either schedule or run a proper processor action for messages
+        # @param group_id [String] group_id of a group from which a given message came
+        # @param kafka_batch [<Kafka::FetchedBatch>] raw messages fetched batch
+        # @note This should be looped to obtain a constant delegating of new messages
+        def call(group_id, kafka_batch)
+          topic = Persistence::Topics.fetch(group_id, kafka_batch.topic)
+          consumer = Persistence::Consumers.fetch(topic, kafka_batch.partition)
+
+          Karafka.monitor.instrument(
+            'connection.batch_delegator.call',
+            caller: self,
+            consumer: consumer,
+            kafka_batch: kafka_batch
+          ) do
+            # Due to how ruby-kafka is built, we have the metadata that is stored on the batch
+            # level only available for batch consuming
+            consumer.batch_metadata = Params::Builders::BatchMetadata.from_kafka_batch(
+              kafka_batch,
+              topic
+            )
+
+            kafka_messages = kafka_batch.messages
+
+            # Depending on a case (persisted or not) we might use new consumer instance per
+            # each batch, or use the same one for all of them (for implementing buffering, etc.)
+            if topic.batch_consuming
+              consumer.params_batch = Params::Builders::ParamsBatch.from_kafka_messages(
+                kafka_messages,
+                topic
+              )
+              consumer.call
+            else
+              kafka_messages.each do |kafka_message|
+                consumer.params_batch = Params::Builders::ParamsBatch.from_kafka_messages(
+                  [kafka_message],
+                  topic
+                )
+                consumer.call
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/connection/builder.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Builder used to construct Kafka client
+    module Builder
+      class << self
+        # Builds a Kafka::Client instance that we use to work with Kafka cluster
+        # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group for which we want
+        #   to have a new Kafka client
+        # @return [::Kafka::Client] returns a Kafka client
+        def call(consumer_group)
+          Kafka.new(*ApiAdapter.client(consumer_group))
+        end
+      end
+    end
+  end
+end