karafka 1.2.2 → 1.4.0.rc1

data/lib/karafka.rb

@@ -1,20 +1,22 @@
 # frozen_string_literal: true
 
 %w[
+  delegate
   English
   waterdrop
   kafka
   envlogic
+  json
   thor
+  forwardable
   fileutils
-  multi_json
-  require_all
   dry-configurable
   dry-validation
+  dry/events/publisher
   dry/inflector
   dry/monitor/notifications
-  active_support/callbacks
-  karafka/loader
+  dry/core/constants
+  zeitwerk
 ].each(&method(:require))
 
 # Karafka library
@@ -62,17 +64,9 @@ module Karafka
   end
 end
 
-%w[
-  callbacks
-  callbacks/*
-  setup/dsl
-  setup/config
-  status
-  schemas/config
-  schemas/consumer_group_topic
-  schemas/consumer_group
-].each { |path| require_all File.join(Karafka.core_root, path + '.rb') }
+Zeitwerk::Loader
+  .for_gem
+  .tap(&:setup)
+  .tap(&:eager_load)
 
-Karafka::Loader.load!(Karafka.core_root)
 Kafka::Consumer.prepend(Karafka::Patches::RubyKafka)
-Dry::Configurable::Config.prepend(Karafka::Patches::DryConfigurable)

data/lib/karafka/app.rb

@@ -4,28 +4,36 @@ module Karafka
   # App class
   class App
     extend Setup::Dsl
-    extend Callbacks::Dsl
 
     class << self
       # Sets up all the internal components and bootstrap whole app
       # We need to know details about consumers in order to setup components,
       # that's why we don't setup them after std setup is done
-      # @raise [Karafka::Errors::InvalidConfiguration] raised when configuration
-      #   doesn't match with ConfigurationSchema
+      # @raise [Karafka::Errors::InvalidConfigurationError] raised when configuration
+      #   doesn't match with the config contract
       def boot!
+        initialize!
         Setup::Config.validate!
         Setup::Config.setup_components
-        Callbacks.after_init(Karafka::App.config)
+        initialized!
       end
 
       # @return [Karafka::Routing::Builder] consumers builder instance
       def consumer_groups
-        Routing::Builder.instance
+        config.internal.routing_builder
+      end
+
+      # Triggers reload of all cached Karafka app components, so we can use in-process
+      # in-development hot code reloading without Karafka process restart
+      def reload
+        Karafka::Persistence::Consumers.clear
+        Karafka::Persistence::Topics.clear
+        config.internal.routing_builder.reload
       end
 
       Status.instance_methods(false).each do |delegated|
         define_method(delegated) do
-          Status.instance.send(delegated)
+          App.config.internal.status.send(delegated)
         end
       end
 

data/lib/karafka/attributes_map.rb

@@ -11,18 +11,18 @@ module Karafka
   module AttributesMap
     class << self
       # What settings should go where in ruby-kafka
+      # @return [Hash] hash with proper sections on what to proxy where in Ruby-Kafka
       # @note All other settings will be passed to Kafka.new method invocation.
       #   All elements in this hash are just edge cases
-      # @return [Hash] hash with proper sections on what to proxy where in Ruby-Kafka
-      def config_adapter
+      def api_adapter
         {
           consumer: %i[
             session_timeout offset_commit_interval offset_commit_threshold
-            offset_retention_time heartbeat_interval
+            offset_retention_time heartbeat_interval fetcher_max_queue_size
           ],
-          subscription: %i[start_from_beginning max_bytes_per_partition],
-          consuming: %i[min_bytes max_bytes max_wait_time],
-          pausing: %i[pause_timeout],
+          subscribe: %i[start_from_beginning max_bytes_per_partition],
+          consumption: %i[min_bytes max_bytes max_wait_time],
+          pause: %i[pause_timeout pause_max_timeout pause_exponential_backoff],
           # All the options that are under kafka config namespace, but are not used
           # directly with kafka api, but from the Karafka user perspective, they are
           # still related to kafka. They should not be proxied anywhere
@@ -32,13 +32,12 @@ module Karafka
 
       # @return [Array<Symbol>] properties that can be set on a per topic level
       def topic
-        (config_adapter[:subscription] + %i[
+        (api_adapter[:subscribe] + %i[
           backend
           name
-          parser
+          deserializer
           responder
           batch_consuming
-          persistent
         ]).uniq
       end
 
@@ -48,17 +47,13 @@ module Karafka
       #   Thanks to this solution, if any new setting is available for ruby-kafka, we just need
       #   to add it to our configuration class and it will be handled automatically.
       def consumer_group
-        # @note We don't ignore the config_adapter[:ignored] values as they should be ignored
+        # @note We don't ignore the api_adapter[:ignored] values as they should be ignored
         #   only when proxying details go ruby-kafka. We use ignored fields internally in karafka
-        ignored_settings = config_adapter[:subscription]
-        defined_settings = config_adapter.values.flatten
+        ignored_settings = api_adapter[:subscribe]
+        defined_settings = api_adapter.values.flatten
         karafka_settings = %i[batch_fetching]
-        # This is a drity and bad hack of dry-configurable to get keys before setting values
-        dynamically_proxied = Karafka::Setup::Config
-                              ._settings
-                              .find { |s| s.name == :kafka }
-                              .value
-                              .instance_variable_get('@klass').settings
+
+        dynamically_proxied = Karafka::Setup::Config.config.kafka.to_h.keys
 
         (defined_settings + dynamically_proxied).uniq + karafka_settings - ignored_settings
       end

data/lib/karafka/base_consumer.rb

@@ -4,41 +4,33 @@
 module Karafka
   # Base consumer from which all Karafka consumers should inherit
   class BaseConsumer
-    extend ActiveSupport::DescendantsTracker
     extend Forwardable
 
     # Allows us to mark messages as consumed for non-automatic mode without having
     # to use consumer client directly. We do this that way, because most of the people should not
     # mess with the client instance directly (just in case)
-    def_delegator :client, :mark_as_consumed
-
-    private :mark_as_consumed
-
-    class << self
-      attr_reader :topic
-
-      # Assigns a topic to a consumer and builds up proper consumer functionalities
-      #   so that it can cooperate with the topic settings
-      # @param topic [Karafka::Routing::Topic]
-      # @return [Karafka::Routing::Topic] assigned topic
-      def topic=(topic)
-        @topic = topic
-        Consumers::Includer.call(self)
-      end
+    %i[
+      mark_as_consumed
+      mark_as_consumed!
+      trigger_heartbeat
+      trigger_heartbeat!
+    ].each do |delegated_method_name|
+      def_delegator :client, delegated_method_name
+
+      private delegated_method_name
     end
 
     # @return [Karafka::Routing::Topic] topic to which a given consumer is subscribed
-    def topic
-      self.class.topic
-    end
-
-    # Creates lazy loaded params batch object
-    # @note Until first params usage, it won't parse data at all
-    # @param messages [Array<Kafka::FetchedMessage>, Array<Hash>] messages with raw
-    #   content (from Kafka) or messages inside a hash (from backend, etc)
-    # @return [Karafka::Params::ParamsBatch] lazy loaded params batch
-    def params_batch=(messages)
-      @params_batch = Karafka::Params::ParamsBatch.new(messages, topic.parser)
+    attr_reader :topic
+    # @return [Karafka::Params:ParamsBatch] current params batch
+    attr_accessor :params_batch
+
+    # Assigns a topic to a consumer and builds up proper consumer functionalities
+    #   so that it can cooperate with the topic settings
+    # @param topic [Karafka::Routing::Topic]
+    def initialize(topic)
+      @topic = topic
+      Consumers::Includer.call(self)
     end
 
     # Executes the default consumer flow.
@@ -48,9 +40,6 @@ module Karafka
 
     private
 
-    # We make it private as it should be accessible only from the inside of a consumer
-    attr_reader :params_batch
-
     # @return [Karafka::Connection::Client] messages consuming client that can be used to
     #    commit manually offset or pause / stop consumer based on the business logic
     def client

data/lib/karafka/base_responder.rb

@@ -39,7 +39,7 @@ module Karafka
   #
   # @example Multiple times used topic
   #   class Responder < BaseResponder
-  #     topic :required_topic, multiple_usage: true
+  #     topic :required_topic
   #
   #     def respond(data)
   #       data.each do |subset|
@@ -48,6 +48,17 @@ module Karafka
   #     end
   #   end
   #
+  # @example Specify serializer for a topic
+  #   class Responder < BaseResponder
+  #     topic :xml_topic, serializer: MyXMLSerializer
+  #
+  #     def respond(data)
+  #       data.each do |subset|
+  #         respond_to :xml_topic, subset
+  #       end
+  #     end
+  #   end
+  #
   # @example Accept multiple arguments to a respond method
   #   class Responder < BaseResponder
   #     topic :users_actions
@@ -59,31 +70,35 @@ module Karafka
   #     end
   #   end
   class BaseResponder
-    # Definitions of all topics that we want to be able to use in this responder should go here
-    class_attribute :topics
-
-    # Schema that we can use to control and/or require some additional details upon options
-    # that are being passed to the producer. This can be in particular useful if we want to make
-    # sure that for example partition_key is always present.
-    class_attribute :options_schema
+    # Responder usage contract
+    CONTRACT = Karafka::Contracts::ResponderUsage.new.freeze
 
-    attr_reader :messages_buffer
+    private_constant :CONTRACT
 
     class << self
+      # Definitions of all topics that we want to be able to use in this responder should go here
+      attr_accessor :topics
+      # Contract that we can use to control and/or require some additional details upon options
+      # that are being passed to the producer. This can be in particular useful if we want to make
+      # sure that for example partition_key is always present.
+      attr_accessor :options_contract
+
       # Registers a topic as on to which we will be able to respond
       # @param topic_name [Symbol, String] name of topic to which we want to respond
       # @param options [Hash] hash with optional configuration details
       def topic(topic_name, options = {})
+        options[:serializer] ||= Karafka::App.config.serializer
+        options[:registered] = true
         self.topics ||= {}
-        topic_obj = Responders::Topic.new(topic_name, options.merge(registered: true))
+        topic_obj = Responders::Topic.new(topic_name, options)
         self.topics[topic_obj.name] = topic_obj
       end
 
       # A simple alias for easier standalone responder usage.
-      # Instead of building it with new.call it allows (in case of usin JSON parser)
+      # Instead of building it with new.call it allows (in case of using JSON serializer)
       # to just run it directly from the class level
       # @param data Anything that we want to respond with
-      # @example Send user data with a responder (uses default Karafka::Parsers::Json parser)
+      # @example Send user data with a responder
       #   UsersCreatedResponder.call(@created_user)
       def call(*data)
         # Just in case there were no topics defined for a responder, we initialize with
@@ -93,12 +108,11 @@ module Karafka
       end
     end
 
+    attr_reader :messages_buffer
+
     # Creates a responder object
-    # @param parser_class [Class] parser class that we can use to generate appropriate string
-    #   or nothing if we want to default to Karafka::Parsers::Json
     # @return [Karafka::BaseResponder] base responder descendant responder
-    def initialize(parser_class = Karafka::App.config.parser)
-      @parser_class = parser_class
+    def initialize
       @messages_buffer = {}
     end
 
@@ -107,7 +121,7 @@ module Karafka
     # @note We know that validators should be executed also before sending data to topics, however
     #   the implementation gets way more complicated then, that's why we check after everything
     #   was sent using responder
-    # @example Send user data with a responder (uses default Karafka::Parsers::Json parser)
+    # @example Send user data with a responder
     #   UsersCreatedResponder.new.call(@created_user)
     # @example Send user data with a responder using non default Parser
     #   UsersCreatedResponder.new(MyParser).call(@created_user)
@@ -134,25 +148,26 @@ module Karafka
         topic.to_h.merge!(usage_count: usage.count)
       end
 
-      result = Karafka::Schemas::ResponderUsage.call(
+      result = CONTRACT.call(
         registered_topics: registered_topics,
         used_topics: used_topics
       )
 
       return if result.success?
 
-      raise Karafka::Errors::InvalidResponderUsage, result.errors
+      raise Karafka::Errors::InvalidResponderUsageError, result.errors.to_h
     end
 
     # Checks if we met all the options requirements before sending them to the producer.
     def validate_options!
-      return true unless self.class.options_schema
+      return true unless self.class.options_contract
 
       messages_buffer.each_value do |messages_set|
         messages_set.each do |message_data|
-          result = self.class.options_schema.call(message_data.last)
+          result = self.class.options_contract.call(message_data.last)
           next if result.success?
-          raise Karafka::Errors::InvalidResponderMessageOptions, result.errors
+
+          raise Karafka::Errors::InvalidResponderMessageOptionsError, result.errors.to_h
         end
       end
     end
@@ -163,13 +178,18 @@ module Karafka
     def deliver!
       messages_buffer.each_value do |data_elements|
         data_elements.each do |data, options|
-          producer(options).call(data, options)
+          # We map this topic name, so it will match namespaced/etc topic in Kafka
+          # @note By default will not change topic (if default mapper used)
+          mapped_topic = Karafka::App.config.topic_mapper.outgoing(options[:topic])
+          external_options = options.merge(topic: mapped_topic)
+          producer(options).call(data, external_options)
         end
       end
     end
 
     # Method that needs to be implemented in a subclass. It should handle responding
     #   on registered topics
+    # @param _data [Object] anything that we want to use to send to Kafka
     # @raise [NotImplementedError] This method needs to be implemented in a subclass
     def respond(*_data)
       raise NotImplementedError, 'Implement this in a subclass'
@@ -179,7 +199,7 @@ module Karafka
     # as many times as we need. Especially when we have 1:n flow
     # @param topic [Symbol, String] topic to which we want to respond
     # @param data [String, Object] string or object that we want to send
-    # @param options [Hash] options for waterdrop (e.g. partition_key)
+    # @param options [Hash] options for waterdrop (e.g. partition_key).
     # @note Respond to does not accept multiple data arguments.
     def respond_to(topic, data, options = {})
       # We normalize the format to string, as WaterDrop and Ruby-Kafka support only
@@ -188,17 +208,19 @@ module Karafka
 
       messages_buffer[topic] ||= []
       messages_buffer[topic] << [
-        @parser_class.generate(data),
-        # We map this topic name, so it will match namespaced/etc topic in Kafka
-        # @note By default will not change topic (if default mapper used)
-        options.merge(topic: Karafka::App.config.topic_mapper.outgoing(topic))
+        self.class.topics[topic].serializer.call(data),
+        options.merge(topic: topic)
       ]
     end
 
     # @param options [Hash] options for waterdrop
     # @return [Class] WaterDrop producer (sync or async based on the settings)
     def producer(options)
-      options[:async] ? WaterDrop::AsyncProducer : WaterDrop::SyncProducer
+      if self.class.topics[options[:topic]].async?
+        WaterDrop::AsyncProducer
+      else
+        WaterDrop::SyncProducer
+      end
     end
   end
 end

data/lib/karafka/cli.rb

@@ -37,7 +37,7 @@ end
 # This is kinda trick - since we don't have a autoload and other magic stuff
 # like Rails does, so instead this method allows us to replace currently running
 # console with a new one via Kernel.exec. It will start console with new code loaded
-# Yes we know that it is not turbofast, however it is turbo convinient and small
+# Yes, we know that it is not turbo fast, however it is turbo convenient and small
 #
 # Also - the KARAFKA_CONSOLE is used to detect that we're executing the irb session
 # so this method is only available when the Karafka console is running
@@ -47,7 +47,7 @@ end
 if ENV['KARAFKA_CONSOLE']
   # Reloads Karafka irb console session
   def reload!
-    puts "Reloading...\n"
+    Karafka.logger.info "Reloading...\n"
     Kernel.exec Karafka::Cli::Console.command
   end
 end

data/lib/karafka/cli/console.rb

@@ -8,15 +8,17 @@ module Karafka
       desc 'Start the Karafka console (short-cut alias: "c")'
       option aliases: 'c'
 
-      # @return [String] Console executing command
-      # @example
-      #   Karafka::Cli::Console.command #=> 'KARAFKA_CONSOLE=true bundle exec irb...'
-      def self.command
-        envs = [
-          "IRBRC='#{Karafka.gem_root}/.console_irbrc'",
-          'KARAFKA_CONSOLE=true'
-        ]
-        "#{envs.join(' ')} bundle exec irb"
+      class << self
+        # @return [String] Console executing command
+        # @example
+        #   Karafka::Cli::Console.command #=> 'KARAFKA_CONSOLE=true bundle exec irb...'
+        def command
+          envs = [
+            "IRBRC='#{Karafka.gem_root}/.console_irbrc'",
+            'KARAFKA_CONSOLE=true'
+          ]
+          "#{envs.join(' ')} bundle exec irb -r #{Karafka.boot_file}"
+        end
       end
 
       # Start the Karafka console