karafka 1.2.8 → 1.4.0

data/lib/karafka/app.rb

@@ -4,28 +4,36 @@ module Karafka
   # App class
   class App
     extend Setup::Dsl
-    extend Callbacks::Dsl
 
     class << self
       # Sets up all the internal components and bootstrap whole app
       # We need to know details about consumers in order to setup components,
       # that's why we don't setup them after std setup is done
-      # @raise [Karafka::Errors::InvalidConfiguration] raised when configuration
-      #   doesn't match with ConfigurationSchema
+      # @raise [Karafka::Errors::InvalidConfigurationError] raised when configuration
+      #   doesn't match with the config contract
       def boot!
+        initialize!
         Setup::Config.validate!
         Setup::Config.setup_components
-        Callbacks.after_init(Karafka::App.config)
+        initialized!
       end
 
       # @return [Karafka::Routing::Builder] consumers builder instance
       def consumer_groups
-        Routing::Builder.instance
+        config.internal.routing_builder
+      end
+
+      # Triggers reload of all cached Karafka app components, so we can use in-process
+      # in-development hot code reloading without Karafka process restart
+      def reload
+        Karafka::Persistence::Consumers.clear
+        Karafka::Persistence::Topics.clear
+        config.internal.routing_builder.reload
       end
 
       Status.instance_methods(false).each do |delegated|
         define_method(delegated) do
-          Status.instance.send(delegated)
+          App.config.internal.status.send(delegated)
         end
       end
 

data/lib/karafka/attributes_map.rb

@@ -11,9 +11,9 @@ module Karafka
   module AttributesMap
     class << self
       # What settings should go where in ruby-kafka
+      # @return [Hash] hash with proper sections on what to proxy where in Ruby-Kafka
       # @note All other settings will be passed to Kafka.new method invocation.
       #   All elements in this hash are just edge cases
-      # @return [Hash] hash with proper sections on what to proxy where in Ruby-Kafka
       def api_adapter
         {
           consumer: %i[
@@ -22,7 +22,7 @@ module Karafka
           ],
           subscribe: %i[start_from_beginning max_bytes_per_partition],
           consumption: %i[min_bytes max_bytes max_wait_time],
-          pause: %i[pause_timeout],
+          pause: %i[pause_timeout pause_max_timeout pause_exponential_backoff],
           # All the options that are under kafka config namespace, but are not used
           # directly with kafka api, but from the Karafka user perspective, they are
           # still related to kafka. They should not be proxied anywhere
@@ -35,10 +35,9 @@ module Karafka
         (api_adapter[:subscribe] + %i[
           backend
           name
-          parser
+          deserializer
           responder
           batch_consuming
-          persistent
         ]).uniq
       end
 
@@ -53,12 +52,8 @@ module Karafka
         ignored_settings = api_adapter[:subscribe]
         defined_settings = api_adapter.values.flatten
         karafka_settings = %i[batch_fetching]
-        # This is a drity and bad hack of dry-configurable to get keys before setting values
-        dynamically_proxied = Karafka::Setup::Config
-                              ._settings
-                              .find { |s| s.name == :kafka }
-                              .value
-                              .instance_variable_get('@klass').settings
+
+        dynamically_proxied = Karafka::Setup::Config.config.kafka.to_h.keys
 
         (defined_settings + dynamically_proxied).uniq + karafka_settings - ignored_settings
       end

data/lib/karafka/base_consumer.rb

@@ -4,41 +4,33 @@
 module Karafka
   # Base consumer from which all Karafka consumers should inherit
   class BaseConsumer
-    extend ActiveSupport::DescendantsTracker
     extend Forwardable
 
     # Allows us to mark messages as consumed for non-automatic mode without having
     # to use consumer client directly. We do this that way, because most of the people should not
     # mess with the client instance directly (just in case)
-    def_delegator :client, :mark_as_consumed
-
-    private :mark_as_consumed
-
-    class << self
-      attr_reader :topic
-
-      # Assigns a topic to a consumer and builds up proper consumer functionalities
-      #   so that it can cooperate with the topic settings
-      # @param topic [Karafka::Routing::Topic]
-      # @return [Karafka::Routing::Topic] assigned topic
-      def topic=(topic)
-        @topic = topic
-        Consumers::Includer.call(self)
-      end
+    %i[
+      mark_as_consumed
+      mark_as_consumed!
+      trigger_heartbeat
+      trigger_heartbeat!
+    ].each do |delegated_method_name|
+      def_delegator :client, delegated_method_name
+
+      private delegated_method_name
     end
 
     # @return [Karafka::Routing::Topic] topic to which a given consumer is subscribed
-    def topic
-      self.class.topic
-    end
-
-    # Creates lazy loaded params batch object
-    # @note Until first params usage, it won't parse data at all
-    # @param messages [Array<Kafka::FetchedMessage>, Array<Hash>] messages with raw
-    #   content (from Kafka) or messages inside a hash (from backend, etc)
-    # @return [Karafka::Params::ParamsBatch] lazy loaded params batch
-    def params_batch=(messages)
-      @params_batch = Karafka::Params::ParamsBatch.new(messages, topic.parser)
+    attr_reader :topic
+    # @return [Karafka::Params:ParamsBatch] current params batch
+    attr_accessor :params_batch
+
+    # Assigns a topic to a consumer and builds up proper consumer functionalities
+    #   so that it can cooperate with the topic settings
+    # @param topic [Karafka::Routing::Topic]
+    def initialize(topic)
+      @topic = topic
+      Consumers::Includer.call(self)
     end
 
     # Executes the default consumer flow.
@@ -48,9 +40,6 @@ module Karafka
 
     private
 
-    # We make it private as it should be accessible only from the inside of a consumer
-    attr_reader :params_batch
-
     # @return [Karafka::Connection::Client] messages consuming client that can be used to
     #    commit manually offset or pause / stop consumer based on the business logic
     def client

data/lib/karafka/base_responder.rb

@@ -39,7 +39,7 @@ module Karafka
   #
   # @example Multiple times used topic
   #   class Responder < BaseResponder
-  #     topic :required_topic, multiple_usage: true
+  #     topic :required_topic
   #
   #     def respond(data)
   #       data.each do |subset|
@@ -48,6 +48,17 @@ module Karafka
   #     end
   #   end
   #
+  # @example Specify serializer for a topic
+  #   class Responder < BaseResponder
+  #     topic :xml_topic, serializer: MyXMLSerializer
+  #
+  #     def respond(data)
+  #       data.each do |subset|
+  #         respond_to :xml_topic, subset
+  #       end
+  #     end
+  #   end
+  #
   # @example Accept multiple arguments to a respond method
   #   class Responder < BaseResponder
   #     topic :users_actions
@@ -59,31 +70,35 @@ module Karafka
   #     end
   #   end
   class BaseResponder
-    # Definitions of all topics that we want to be able to use in this responder should go here
-    class_attribute :topics
-
-    # Schema that we can use to control and/or require some additional details upon options
-    # that are being passed to the producer. This can be in particular useful if we want to make
-    # sure that for example partition_key is always present.
-    class_attribute :options_schema
+    # Responder usage contract
+    CONTRACT = Karafka::Contracts::ResponderUsage.new.freeze
 
-    attr_reader :messages_buffer
+    private_constant :CONTRACT
 
     class << self
+      # Definitions of all topics that we want to be able to use in this responder should go here
+      attr_accessor :topics
+      # Contract that we can use to control and/or require some additional details upon options
+      # that are being passed to the producer. This can be in particular useful if we want to make
+      # sure that for example partition_key is always present.
+      attr_accessor :options_contract
+
       # Registers a topic as on to which we will be able to respond
       # @param topic_name [Symbol, String] name of topic to which we want to respond
       # @param options [Hash] hash with optional configuration details
       def topic(topic_name, options = {})
+        options[:serializer] ||= Karafka::App.config.serializer
+        options[:registered] = true
         self.topics ||= {}
-        topic_obj = Responders::Topic.new(topic_name, options.merge(registered: true))
+        topic_obj = Responders::Topic.new(topic_name, options)
         self.topics[topic_obj.name] = topic_obj
       end
 
       # A simple alias for easier standalone responder usage.
-      # Instead of building it with new.call it allows (in case of usin JSON parser)
+      # Instead of building it with new.call it allows (in case of using JSON serializer)
       # to just run it directly from the class level
       # @param data Anything that we want to respond with
-      # @example Send user data with a responder (uses default Karafka::Parsers::Json parser)
+      # @example Send user data with a responder
       #   UsersCreatedResponder.call(@created_user)
       def call(*data)
         # Just in case there were no topics defined for a responder, we initialize with
@@ -93,12 +108,11 @@ module Karafka
       end
     end
 
+    attr_reader :messages_buffer
+
     # Creates a responder object
-    # @param parser_class [Class] parser class that we can use to generate appropriate string
-    #   or nothing if we want to default to Karafka::Parsers::Json
     # @return [Karafka::BaseResponder] base responder descendant responder
-    def initialize(parser_class = Karafka::App.config.parser)
-      @parser_class = parser_class
+    def initialize
       @messages_buffer = {}
     end
 
@@ -107,7 +121,7 @@ module Karafka
     # @note We know that validators should be executed also before sending data to topics, however
     #   the implementation gets way more complicated then, that's why we check after everything
     #   was sent using responder
-    # @example Send user data with a responder (uses default Karafka::Parsers::Json parser)
+    # @example Send user data with a responder
     #   UsersCreatedResponder.new.call(@created_user)
     # @example Send user data with a responder using non default Parser
     #   UsersCreatedResponder.new(MyParser).call(@created_user)
@@ -134,25 +148,26 @@ module Karafka
         topic.to_h.merge!(usage_count: usage.count)
       end
 
-      result = Karafka::Schemas::ResponderUsage.call(
+      result = CONTRACT.call(
         registered_topics: registered_topics,
         used_topics: used_topics
       )
 
       return if result.success?
 
-      raise Karafka::Errors::InvalidResponderUsage, result.errors
+      raise Karafka::Errors::InvalidResponderUsageError, result.errors.to_h
     end
 
     # Checks if we met all the options requirements before sending them to the producer.
     def validate_options!
-      return true unless self.class.options_schema
+      return true unless self.class.options_contract
 
       messages_buffer.each_value do |messages_set|
         messages_set.each do |message_data|
-          result = self.class.options_schema.call(message_data.last)
+          result = self.class.options_contract.call(message_data.last)
           next if result.success?
-          raise Karafka::Errors::InvalidResponderMessageOptions, result.errors
+
+          raise Karafka::Errors::InvalidResponderMessageOptionsError, result.errors.to_h
         end
       end
     end
@@ -174,6 +189,7 @@ module Karafka
 
     # Method that needs to be implemented in a subclass. It should handle responding
     #   on registered topics
+    # @param _data [Object] anything that we want to use to send to Kafka
     # @raise [NotImplementedError] This method needs to be implemented in a subclass
     def respond(*_data)
       raise NotImplementedError, 'Implement this in a subclass'
@@ -183,7 +199,7 @@ module Karafka
     # as many times as we need. Especially when we have 1:n flow
     # @param topic [Symbol, String] topic to which we want to respond
     # @param data [String, Object] string or object that we want to send
-    # @param options [Hash] options for waterdrop (e.g. partition_key)
+    # @param options [Hash] options for waterdrop (e.g. partition_key).
     # @note Respond to does not accept multiple data arguments.
     def respond_to(topic, data, options = {})
       # We normalize the format to string, as WaterDrop and Ruby-Kafka support only
@@ -192,7 +208,7 @@ module Karafka
 
       messages_buffer[topic] ||= []
       messages_buffer[topic] << [
-        @parser_class.generate(data),
+        self.class.topics[topic].serializer.call(data),
         options.merge(topic: topic)
       ]
     end
@@ -200,9 +216,11 @@ module Karafka
     # @param options [Hash] options for waterdrop
     # @return [Class] WaterDrop producer (sync or async based on the settings)
     def producer(options)
-      self.class.topics[
-        options[:topic]
-      ].async? ? WaterDrop::AsyncProducer : WaterDrop::SyncProducer
+      if self.class.topics[options[:topic]].async?
+        WaterDrop::AsyncProducer
+      else
+        WaterDrop::SyncProducer
+      end
     end
   end
 end

data/lib/karafka/cli.rb

@@ -37,7 +37,7 @@ end
 # This is kinda trick - since we don't have a autoload and other magic stuff
 # like Rails does, so instead this method allows us to replace currently running
 # console with a new one via Kernel.exec. It will start console with new code loaded
-# Yes we know that it is not turbofast, however it is turbo convinient and small
+# Yes, we know that it is not turbo fast, however it is turbo convenient and small
 #
 # Also - the KARAFKA_CONSOLE is used to detect that we're executing the irb session
 # so this method is only available when the Karafka console is running
@@ -47,7 +47,7 @@ end
 if ENV['KARAFKA_CONSOLE']
   # Reloads Karafka irb console session
   def reload!
-    puts "Reloading...\n"
+    Karafka.logger.info "Reloading...\n"
     Kernel.exec Karafka::Cli::Console.command
   end
 end

data/lib/karafka/cli/console.rb

@@ -8,15 +8,17 @@ module Karafka
       desc 'Start the Karafka console (short-cut alias: "c")'
       option aliases: 'c'
 
-      # @return [String] Console executing command
-      # @example
-      #   Karafka::Cli::Console.command #=> 'KARAFKA_CONSOLE=true bundle exec irb...'
-      def self.command
-        envs = [
-          "IRBRC='#{Karafka.gem_root}/.console_irbrc'",
-          'KARAFKA_CONSOLE=true'
-        ]
-        "#{envs.join(' ')} bundle exec irb"
+      class << self
+        # @return [String] Console executing command
+        # @example
+        #   Karafka::Cli::Console.command #=> 'KARAFKA_CONSOLE=true bundle exec irb...'
+        def command
+          envs = [
+            "IRBRC='#{Karafka.gem_root}/.console_irbrc'",
+            'KARAFKA_CONSOLE=true'
+          ]
+          "#{envs.join(' ')} bundle exec irb -r #{Karafka.boot_file}"
+        end
       end
 
       # Start the Karafka console

data/lib/karafka/cli/flow.rb

@@ -11,20 +11,22 @@ module Karafka
       def call
         topics.each do |topic|
           any_topics = !topic.responder&.topics.nil?
+          log_messages = []
 
           if any_topics
-            puts "#{topic.name} =>"
+            log_messages << "#{topic.name} =>"
 
             topic.responder.topics.each_value do |responder_topic|
               features = []
               features << (responder_topic.required? ? 'always' : 'conditionally')
-              features << (responder_topic.multiple_usage? ? 'one or more' : 'exactly once')
 
-              print responder_topic.name, "(#{features.join(', ')})"
+              log_messages << format(responder_topic.name, "(#{features.join(', ')})")
             end
           else
-            puts "#{topic.name} => (nothing)"
+            log_messages << "#{topic.name} => (nothing)"
           end
+
+          Karafka.logger.info(log_messages.join("\n"))
         end
       end
 
@@ -35,11 +37,11 @@ module Karafka
         Karafka::App.consumer_groups.map(&:topics).flatten.sort_by(&:name)
       end
 
-      # Prints a given value with label in a nice way
+      # Formats a given value with label in a nice way
       # @param label [String] label describing value
       # @param value [String] value that should be printed
-      def print(label, value)
-        printf "%-25s %s\n", "  - #{label}:", value
+      def format(label, value)
+        "  - #{label}:                #{value}"
       end
     end
   end

data/lib/karafka/cli/info.rb

@@ -12,7 +12,9 @@ module Karafka
         config = Karafka::App.config
 
         info = [
-          "Karafka framework version: #{Karafka::VERSION}",
+          "Karafka version: #{Karafka::VERSION}",
+          "Ruby version: #{RUBY_VERSION}",
+          "Ruby-kafka version: #{::Kafka::VERSION}",
           "Application client id: #{config.client_id}",
           "Backend: #{config.backend}",
           "Batch fetching: #{config.batch_fetching}",
@@ -22,7 +24,7 @@ module Karafka
           "Kafka seed brokers: #{config.kafka.seed_brokers}"
         ]
 
-        puts(info.join("\n"))
+        Karafka.logger.info(info.join("\n"))
       end
     end
   end

data/lib/karafka/cli/install.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'erb'
+
 module Karafka
   # Karafka framework Cli
   class Cli < Thor
@@ -11,18 +13,30 @@ module Karafka
       INSTALL_DIRS = %w[
         app/consumers
         app/responders
+        app/workers
         config
+        lib
         log
         tmp/pids
       ].freeze
 
       # Where should we map proper files from templates
       INSTALL_FILES_MAP = {
-        'karafka.rb.example' => Karafka.boot_file.basename,
-        'application_consumer.rb.example' => 'app/consumers/application_consumer.rb',
-        'application_responder.rb.example' => 'app/responders/application_responder.rb'
+        'karafka.rb.erb' => Karafka.boot_file.basename,
+        'application_consumer.rb.erb' => 'app/consumers/application_consumer.rb',
+        'application_responder.rb.erb' => 'app/responders/application_responder.rb'
       }.freeze
 
+      # @param args [Array] all the things that Thor CLI accepts
+      def initialize(*args)
+        super
+        @rails = Bundler::LockfileParser.new(
+          Bundler.read_file(
+            Bundler.default_lockfile
+          )
+        ).dependencies.key?('rails')
+      end
+
       # Install all required things for Karafka application in current directory
       def call
         INSTALL_DIRS.each do |dir|
@@ -31,12 +45,22 @@ module Karafka
 
         INSTALL_FILES_MAP.each do |source, target|
           target = Karafka.root.join(target)
-          next if File.exist?(target)
 
-          source = Karafka.core_root.join("templates/#{source}")
-          FileUtils.cp_r(source, target)
+          template = File.read(Karafka.core_root.join("templates/#{source}"))
+          # @todo Replace with the keyword argument version once we don't have to support
+          # Ruby < 2.6
+          render = ::ERB.new(template, nil, '-').result(binding)
+
+          File.open(target, 'w') { |file| file.write(render) }
         end
       end
+
+      # @return [Boolean] true if we have Rails loaded
+      # This allows us to generate customized karafka.rb template with some tweaks specific for
+      # Rails
+      def rails?
+        @rails
+      end
     end
   end
 end