karafka 1.1.0 → 1.3.0

data/lib/karafka/base_consumer.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+# Karafka module namespace
+module Karafka
+  # Base consumer from which all Karafka consumers should inherit
+  class BaseConsumer
+    extend Forwardable
+
+    # Allows us to mark messages as consumed for non-automatic mode without having
+    # to use consumer client directly. We do this that way, because most of the people should not
+    # mess with the client instance directly (just in case)
+    %i[
+      mark_as_consumed
+      mark_as_consumed!
+      trigger_heartbeat
+      trigger_heartbeat!
+    ].each do |delegated_method_name|
+      def_delegator :client, delegated_method_name
+
+      private delegated_method_name
+    end
+
+    # @return [Karafka::Routing::Topic] topic to which a given consumer is subscribed
+    attr_reader :topic
+    # @return [Karafka::Params:ParamsBatch] current params batch
+    attr_accessor :params_batch
+
+    # Assigns a topic to a consumer and builds up proper consumer functionalities
+    #   so that it can cooperate with the topic settings
+    # @param topic [Karafka::Routing::Topic]
+    def initialize(topic)
+      @topic = topic
+      Consumers::Includer.call(self)
+    end
+
+    # Executes the default consumer flow.
+    def call
+      process
+    end
+
+    private
+
+    # @return [Karafka::Connection::Client] messages consuming client that can be used to
+    #    commit manually offset or pause / stop consumer based on the business logic
+    def client
+      Persistence::Client.read
+    end
+
+    # Method that will perform business logic and on data received from Kafka (it will consume
+    #   the data)
+    # @note This method needs bo be implemented in a subclass. We stub it here as a failover if
+    #   someone forgets about it or makes on with typo
+    def consume
+      raise NotImplementedError, 'Implement this in a subclass'
+    end
+  end
+end

data/lib/karafka/base_responder.rb

@@ -39,7 +39,7 @@ module Karafka
   #
   # @example Multiple times used topic
   #   class Responder < BaseResponder
-  #     topic :required_topic, multiple_usage: true
+  #     topic :required_topic
   #
   #     def respond(data)
   #       data.each do |subset|
@@ -48,6 +48,17 @@ module Karafka
   #     end
   #   end
   #
+  # @example Specify serializer for a topic
+  #   class Responder < BaseResponder
+  #     topic :xml_topic, serializer: MyXMLSerializer
+  #
+  #     def respond(data)
+  #       data.each do |subset|
+  #         respond_to :xml_topic, subset
+  #       end
+  #     end
+  #   end
+  #
   # @example Accept multiple arguments to a respond method
   #   class Responder < BaseResponder
   #     topic :users_actions
@@ -59,26 +70,35 @@ module Karafka
   #     end
   #   end
   class BaseResponder
-    # Definitions of all topics that we want to be able to use in this responder should go here
-    class_attribute :topics
+    # Responder usage contract
+    CONTRACT = Karafka::Contracts::ResponderUsage.new.freeze
 
-    attr_reader :messages_buffer
+    private_constant :CONTRACT
 
     class << self
+      # Definitions of all topics that we want to be able to use in this responder should go here
+      attr_accessor :topics
+      # Contract that we can use to control and/or require some additional details upon options
+      # that are being passed to the producer. This can be in particular useful if we want to make
+      # sure that for example partition_key is always present.
+      attr_accessor :options_contract
+
       # Registers a topic as on to which we will be able to respond
       # @param topic_name [Symbol, String] name of topic to which we want to respond
       # @param options [Hash] hash with optional configuration details
       def topic(topic_name, options = {})
+        options[:serializer] ||= Karafka::App.config.serializer
+        options[:registered] = true
         self.topics ||= {}
-        topic_obj = Responders::Topic.new(topic_name, options.merge(registered: true))
+        topic_obj = Responders::Topic.new(topic_name, options)
         self.topics[topic_obj.name] = topic_obj
       end
 
       # A simple alias for easier standalone responder usage.
-      # Instead of building it with new.call it allows (in case of usin JSON parser)
+      # Instead of building it with new.call it allows (in case of using JSON serializer)
       # to just run it directly from the class level
       # @param data Anything that we want to respond with
-      # @example Send user data with a responder (uses default Karafka::Parsers::Json parser)
+      # @example Send user data with a responder
       #   UsersCreatedResponder.call(@created_user)
       def call(*data)
         # Just in case there were no topics defined for a responder, we initialize with
@@ -88,12 +108,11 @@ module Karafka
       end
     end
 
+    attr_reader :messages_buffer
+
     # Creates a responder object
-    # @param parser_class [Class] parser class that we can use to generate appropriate string
-    #   or nothing if we want to default to Karafka::Parsers::Json
     # @return [Karafka::BaseResponder] base responder descendant responder
-    def initialize(parser_class = Karafka::Parsers::Json)
-      @parser_class = parser_class
+    def initialize
       @messages_buffer = {}
     end
 
@@ -102,13 +121,14 @@ module Karafka
     # @note We know that validators should be executed also before sending data to topics, however
     #   the implementation gets way more complicated then, that's why we check after everything
     #   was sent using responder
-    # @example Send user data with a responder (uses default Karafka::Parsers::Json parser)
+    # @example Send user data with a responder
     #   UsersCreatedResponder.new.call(@created_user)
     # @example Send user data with a responder using non default Parser
     #   UsersCreatedResponder.new(MyParser).call(@created_user)
     def call(*data)
       respond(*data)
-      validate!
+      validate_usage!
+      validate_options!
       deliver!
     end
 
@@ -116,7 +136,7 @@ module Karafka
 
     # Checks if we met all the topics requirements. It will fail if we didn't send a message to
     # a registered required topic, etc.
-    def validate!
+    def validate_usage!
       registered_topics = self.class.topics.map do |name, topic|
         topic.to_h.merge!(
           usage_count: messages_buffer[name]&.count || 0
@@ -128,36 +148,48 @@ module Karafka
         topic.to_h.merge!(usage_count: usage.count)
       end
 
-      result = Karafka::Schemas::ResponderUsage.call(
+      result = CONTRACT.call(
         registered_topics: registered_topics,
         used_topics: used_topics
       )
 
       return if result.success?
 
-      raise Karafka::Errors::InvalidResponderUsage, result.errors
+      raise Karafka::Errors::InvalidResponderUsageError, result.errors.to_h
+    end
+
+    # Checks if we met all the options requirements before sending them to the producer.
+    def validate_options!
+      return true unless self.class.options_contract
+
+      messages_buffer.each_value do |messages_set|
+        messages_set.each do |message_data|
+          result = self.class.options_contract.call(message_data.last)
+          next if result.success?
+
+          raise Karafka::Errors::InvalidResponderMessageOptionsError, result.errors.to_h
+        end
+      end
     end
 
     # Takes all the messages from the buffer and delivers them one by one
     # @note This method is executed after the validation, so we're sure that
     #   what we send is legit and it will go to a proper topics
     def deliver!
-      messages_buffer.each do |topic, data_elements|
-        # We map this topic name, so it will match namespaced/etc topic in Kafka
-        # @note By default will not change topic (if default mapper used)
-        mapped_topic = Karafka::App.config.topic_mapper.outgoing(topic)
-
+      messages_buffer.each_value do |data_elements|
         data_elements.each do |data, options|
-          producer(options).call(
-            data,
-            options.merge(topic: mapped_topic)
-          )
+          # We map this topic name, so it will match namespaced/etc topic in Kafka
+          # @note By default will not change topic (if default mapper used)
+          mapped_topic = Karafka::App.config.topic_mapper.outgoing(options[:topic])
+          external_options = options.merge(topic: mapped_topic)
+          producer(options).call(data, external_options)
         end
       end
     end
 
     # Method that needs to be implemented in a subclass. It should handle responding
     #   on registered topics
+    # @param _data [Object] anything that we want to use to send to Kafka
     # @raise [NotImplementedError] This method needs to be implemented in a subclass
     def respond(*_data)
       raise NotImplementedError, 'Implement this in a subclass'
@@ -167,19 +199,28 @@ module Karafka
     # as many times as we need. Especially when we have 1:n flow
     # @param topic [Symbol, String] topic to which we want to respond
     # @param data [String, Object] string or object that we want to send
-    # @param options [Hash] options for waterdrop (e.g. partition_key)
+    # @param options [Hash] options for waterdrop (e.g. partition_key).
     # @note Respond to does not accept multiple data arguments.
     def respond_to(topic, data, options = {})
-      Karafka.monitor.notice(self.class, topic: topic, data: data, options: options)
-
-      messages_buffer[topic.to_s] ||= []
-      messages_buffer[topic.to_s] << [@parser_class.generate(data), options]
+      # We normalize the format to string, as WaterDrop and Ruby-Kafka support only
+      # string topics
+      topic = topic.to_s
+
+      messages_buffer[topic] ||= []
+      messages_buffer[topic] << [
+        self.class.topics[topic].serializer.call(data),
+        options.merge(topic: topic)
+      ]
     end
 
     # @param options [Hash] options for waterdrop
     # @return [Class] WaterDrop producer (sync or async based on the settings)
     def producer(options)
-      options[:async] ? WaterDrop::AsyncProducer : WaterDrop::SyncProducer
+      if self.class.topics[options[:topic]].async?
+        WaterDrop::AsyncProducer
+      else
+        WaterDrop::SyncProducer
+      end
     end
   end
 end

data/lib/karafka/cli.rb

@@ -37,7 +37,7 @@ end
 # This is kinda trick - since we don't have a autoload and other magic stuff
 # like Rails does, so instead this method allows us to replace currently running
 # console with a new one via Kernel.exec. It will start console with new code loaded
-# Yes we know that it is not turbofast, however it is turbo convinient and small
+# Yes, we know that it is not turbo fast, however it is turbo convenient and small
 #
 # Also - the KARAFKA_CONSOLE is used to detect that we're executing the irb session
 # so this method is only available when the Karafka console is running

data/lib/karafka/cli/console.rb

@@ -8,15 +8,17 @@ module Karafka
       desc 'Start the Karafka console (short-cut alias: "c")'
       option aliases: 'c'
 
-      # @return [String] Console executing command
-      # @example
-      #   Karafka::Cli::Console.command #=> 'KARAFKA_CONSOLE=true bundle exec irb...'
-      def self.command
-        envs = [
-          "IRBRC='#{Karafka.gem_root}/.console_irbrc'",
-          'KARAFKA_CONSOLE=true'
-        ]
-        "#{envs.join(' ')} bundle exec irb"
+      class << self
+        # @return [String] Console executing command
+        # @example
+        #   Karafka::Cli::Console.command #=> 'KARAFKA_CONSOLE=true bundle exec irb...'
+        def command
+          envs = [
+            "IRBRC='#{Karafka.gem_root}/.console_irbrc'",
+            'KARAFKA_CONSOLE=true'
+          ]
+          "#{envs.join(' ')} bundle exec irb -r #{Karafka.boot_file}"
+        end
       end
 
       # Start the Karafka console

data/lib/karafka/cli/flow.rb

@@ -18,7 +18,6 @@ module Karafka
             topic.responder.topics.each_value do |responder_topic|
               features = []
               features << (responder_topic.required? ? 'always' : 'conditionally')
-              features << (responder_topic.multiple_usage? ? 'one or more' : 'exactly once')
 
               print responder_topic.name, "(#{features.join(', ')})"
             end

data/lib/karafka/cli/info.rb

@@ -12,7 +12,9 @@ module Karafka
         config = Karafka::App.config
 
         info = [
-          "Karafka framework version: #{Karafka::VERSION}",
+          "Karafka version: #{Karafka::VERSION}",
+          "Ruby version: #{RUBY_VERSION}",
+          "Ruby-kafka version: #{::Kafka::VERSION}",
           "Application client id: #{config.client_id}",
           "Backend: #{config.backend}",
           "Batch fetching: #{config.batch_fetching}",

data/lib/karafka/cli/install.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'erb'
+
 module Karafka
   # Karafka framework Cli
   class Cli < Thor
@@ -9,8 +11,7 @@ module Karafka
 
       # Directories created by default
       INSTALL_DIRS = %w[
-        app/models
-        app/controllers
+        app/consumers
         app/responders
         config
         log
@@ -19,11 +20,21 @@ module Karafka
 
       # Where should we map proper files from templates
       INSTALL_FILES_MAP = {
-        'karafka.rb.example' => Karafka.boot_file.basename,
-        'application_controller.rb.example' => 'app/controllers/application_controller.rb',
-        'application_responder.rb.example' => 'app/responders/application_responder.rb'
+        'karafka.rb.erb' => Karafka.boot_file.basename,
+        'application_consumer.rb.erb' => 'app/consumers/application_consumer.rb',
+        'application_responder.rb.erb' => 'app/responders/application_responder.rb'
       }.freeze
 
+      # @param args [Array] all the things that Thor CLI accepts
+      def initialize(*args)
+        super
+        @rails = Bundler::LockfileParser.new(
+          Bundler.read_file(
+            Bundler.default_lockfile
+          )
+        ).dependencies.key?('rails')
+      end
+
       # Install all required things for Karafka application in current directory
       def call
         INSTALL_DIRS.each do |dir|
@@ -32,12 +43,22 @@ module Karafka
 
         INSTALL_FILES_MAP.each do |source, target|
           target = Karafka.root.join(target)
-          next if File.exist?(target)
 
-          source = Karafka.core_root.join("templates/#{source}")
-          FileUtils.cp_r(source, target)
+          template = File.read(Karafka.core_root.join("templates/#{source}"))
+          # @todo Replace with the keyword argument version once we don't have to support
+          # Ruby < 2.6
+          render = ::ERB.new(template, nil, '-').result(binding)
+
+          File.open(target, 'w') { |file| file.write(render) }
         end
       end
+
+      # @return [Boolean] true if we have Rails loaded
+      # This allows us to generate customized karafka.rb template with some tweaks specific for
+      # Rails
+      def rails?
+        @rails
+      end
     end
   end
 end

data/lib/karafka/cli/server.rb

@@ -5,6 +5,11 @@ module Karafka
   class Cli < Thor
     # Server Karafka Cli action
     class Server < Base
+      # Server config settings contract
+      CONTRACT = Contracts::ServerCliOptions.new.freeze
+
+      private_constant :CONTRACT
+
       desc 'Start the Karafka server (short-cut alias: "s")'
       option aliases: 's'
       option :daemon, default: false, type: :boolean, aliases: :d
@@ -13,11 +18,10 @@ module Karafka
 
       # Start the Karafka server
       def call
-        validate!
-
-        puts 'Starting Karafka server'
         cli.info
 
+        validate!
+
         if cli.options[:daemon]
           FileUtils.mkdir_p File.dirname(cli.options[:pid])
           daemonize
@@ -31,11 +35,10 @@ module Karafka
         # We want to delay the moment in which the pidfile is removed as much as we can,
         # so instead of removing it after the server stops running, we rely on the gc moment
         # when this object gets removed (it is a bit later), so it is closer to the actual
-        # system process end. We do that, so monitoring and deployment tools that rely on pids
+        # system process end. We do that, so monitoring and deployment tools that rely on a pid
         # won't alarm or start new system process up until the current one is finished
         ObjectSpace.define_finalizer(self, proc { send(:clean) })
 
-        # After we fork, we can boot celluloid again
         Karafka::Server.run
       end
 
@@ -44,9 +47,10 @@ module Karafka
       # Checks the server cli configuration
       # options validations in terms of app setup (topics, pid existence, etc)
       def validate!
-        result = Schemas::ServerCliOptions.call(cli.options)
+        result = CONTRACT.call(cli.options)
         return if result.success?
-        raise Errors::InvalidConfiguration, result.errors
+
+        raise Errors::InvalidConfigurationError, result.errors.to_h
       end
 
       # Detaches current process into background and writes its pidfile

data/lib/karafka/code_reloader.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Special type of a listener, that is not an instrumentation one, but one that triggers
+  # code reload in the development mode after each fetched batch (or message)
+  #
+  # Please refer to the development code reload sections for details on the benefits and downsides
+  # of the in-process code reloading
+  class CodeReloader
+    # This mutex is needed as we might have an application that has multiple consumer groups
+    # running in separate threads and we should not trigger reload before fully reloading the app
+    # in previous thread
+    MUTEX = Mutex.new
+
+    private_constant :MUTEX
+
+    # @param reloaders [Array<Object>] any code loaders that we use in this app. Whether it is
+    #  the Rails loader, Zeitwerk or anything else that allows reloading triggering
+    # @param block [Proc] yields given block just before reloading. This can be used to hook custom
+    #   reloading stuff, that ain't reloaders (for example for resetting dry-events registry)
+    def initialize(*reloaders, &block)
+      @reloaders = reloaders
+      @block = block
+    end
+
+    # Binds to the instrumentation events and triggers reload
+    # @param _event [Dry::Event] empty dry event
+    # @note Since we de-register all the user defined objects and redraw routes, it means that
+    #   we won't be able to do a multi-batch buffering in the development mode as each of the
+    #   batches will be buffered on a newly created "per fetch" instance.
+    def on_connection_listener_fetch_loop(_event)
+      reload
+    end
+
+    private
+
+    # Triggers reload of both standard and Rails reloaders as well as expires all internals of
+    # Karafka, so it can be rediscovered and rebuilt
+    def reload
+      MUTEX.synchronize do
+        if @reloaders[0].respond_to?(:execute)
+          reload_with_rails
+        else
+          reload_without_rails
+        end
+      end
+    end
+
+    # Rails reloading procedure
+    def reload_with_rails
+      updatable = @reloaders.select(&:updated?)
+
+      return if updatable.empty?
+
+      updatable.each(&:execute)
+      @block&.call
+      Karafka::App.reload
+    end
+
+    # Zeitwerk and other reloaders
+    def reload_without_rails
+      @reloaders.each(&:reload)
+      @block&.call
+      Karafka::App.reload
+    end
+  end
+end