karafka 1.0.1 → 1.4.14

data/lib/karafka/assignment_strategies/round_robin.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Strategies for Kafka partitions assignments
+  module AssignmentStrategies
+    # Standard RoundRobin strategy
+    class RoundRobin < SimpleDelegator
+      def initialize
+        super(Kafka::RoundRobinAssignmentStrategy.new)
+      end
+    end
+  end
+end

data/lib/karafka/attributes_map.rb

@@ -11,34 +11,34 @@ module Karafka
   module AttributesMap
     class << self
       # What settings should go where in ruby-kafka
+      # @return [Hash] hash with proper sections on what to proxy where in Ruby-Kafka
       # @note All other settings will be passed to Kafka.new method invocation.
       #   All elements in this hash are just edge cases
-      # @return [Hash] hash with proper sections on what to proxy where in Ruby-Kafka
-      def config_adapter
+      def api_adapter
         {
           consumer: %i[
             session_timeout offset_commit_interval offset_commit_threshold
-            offset_retention_time heartbeat_interval
+            offset_retention_time heartbeat_interval fetcher_max_queue_size
+            assignment_strategy
           ],
-          subscription: %i[start_from_beginning max_bytes_per_partition],
-          consuming: %i[min_bytes max_wait_time],
-          pausing: %i[pause_timeout],
+          subscribe: %i[start_from_beginning max_bytes_per_partition],
+          consumption: %i[min_bytes max_bytes max_wait_time],
+          pause: %i[pause_timeout pause_max_timeout pause_exponential_backoff],
           # All the options that are under kafka config namespace, but are not used
           # directly with kafka api, but from the Karafka user perspective, they are
           # still related to kafka. They should not be proxied anywhere
-          ignored: %i[reconnect_timeout]
+          ignored: %i[reconnect_timeout automatically_mark_as_consumed]
         }
       end
 
       # @return [Array<Symbol>] properties that can be set on a per topic level
       def topic
-        (config_adapter[:subscription] + %i[
+        (api_adapter[:subscribe] + %i[
           backend
           name
-          parser
+          deserializer
           responder
-          batch_processing
-          persistent
+          batch_consuming
         ]).uniq
       end
 
@@ -48,17 +48,13 @@ module Karafka
       #   Thanks to this solution, if any new setting is available for ruby-kafka, we just need
       #   to add it to our configuration class and it will be handled automatically.
       def consumer_group
-        # @note We don't ignore the config_adapter[:ignored] values as they should be ignored
+        # @note We don't ignore the api_adapter[:ignored] values as they should be ignored
         #   only when proxying details go ruby-kafka. We use ignored fields internally in karafka
-        ignored_settings = config_adapter[:subscription]
-        defined_settings = config_adapter.values.flatten
-        karafka_settings = %i[batch_consuming]
-        # This is a drity and bad hack of dry-configurable to get keys before setting values
-        dynamically_proxied = Karafka::Setup::Config
-                              ._settings
-                              .find { |s| s.name == :kafka }
-                              .value
-                              .instance_variable_get('@klass').settings
+        ignored_settings = api_adapter[:subscribe]
+        defined_settings = api_adapter.values.flatten
+        karafka_settings = %i[batch_fetching]
+
+        dynamically_proxied = Karafka::Setup::Config.config.kafka.to_h.keys
 
         (defined_settings + dynamically_proxied).uniq + karafka_settings - ignored_settings
       end

data/lib/karafka/backends/inline.rb

@@ -7,10 +7,9 @@ module Karafka
     module Inline
       private
 
-      # Executes perform code immediately (without enqueuing)
+      # Executes consume code immediately (without enqueuing)
       def process
-        Karafka.monitor.notice(self.class, params_batch)
-        perform
+        Karafka.monitor.instrument('backends.inline.process', caller: self) { consume }
       end
     end
   end

data/lib/karafka/base_consumer.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+# Karafka module namespace
+module Karafka
+  # Base consumer from which all Karafka consumers should inherit
+  class BaseConsumer
+    extend Forwardable
+
+    # Allows us to mark messages as consumed for non-automatic mode without having
+    # to use consumer client directly. We do this that way, because most of the people should not
+    # mess with the client instance directly (just in case)
+    %i[
+      mark_as_consumed
+      mark_as_consumed!
+      trigger_heartbeat
+      trigger_heartbeat!
+    ].each do |delegated_method_name|
+      def_delegator :client, delegated_method_name
+
+      private delegated_method_name
+    end
+
+    # @return [Karafka::Routing::Topic] topic to which a given consumer is subscribed
+    attr_reader :topic
+    # @return [Karafka::Params:ParamsBatch] current params batch
+    attr_accessor :params_batch
+
+    # Assigns a topic to a consumer and builds up proper consumer functionalities
+    #   so that it can cooperate with the topic settings
+    # @param topic [Karafka::Routing::Topic]
+    def initialize(topic)
+      @topic = topic
+      Consumers::Includer.call(self)
+    end
+
+    # Executes the default consumer flow.
+    def call
+      process
+    end
+
+    private
+
+    # @return [Karafka::Connection::Client] messages consuming client that can be used to
+    #    commit manually offset or pause / stop consumer based on the business logic
+    def client
+      Persistence::Client.read
+    end
+
+    # Method that will perform business logic and on data received from Kafka (it will consume
+    #   the data)
+    # @note This method needs bo be implemented in a subclass. We stub it here as a failover if
+    #   someone forgets about it or makes on with typo
+    def consume
+      raise NotImplementedError, 'Implement this in a subclass'
+    end
+  end
+end

data/lib/karafka/base_responder.rb

@@ -39,7 +39,7 @@ module Karafka
   #
   # @example Multiple times used topic
   #   class Responder < BaseResponder
-  #     topic :required_topic, multiple_usage: true
+  #     topic :required_topic
   #
   #     def respond(data)
   #       data.each do |subset|
@@ -48,6 +48,17 @@ module Karafka
   #     end
   #   end
   #
+  # @example Specify serializer for a topic
+  #   class Responder < BaseResponder
+  #     topic :xml_topic, serializer: MyXMLSerializer
+  #
+  #     def respond(data)
+  #       data.each do |subset|
+  #         respond_to :xml_topic, subset
+  #       end
+  #     end
+  #   end
+  #
   # @example Accept multiple arguments to a respond method
   #   class Responder < BaseResponder
   #     topic :users_actions
@@ -59,26 +70,35 @@ module Karafka
   #     end
   #   end
   class BaseResponder
-    # Definitions of all topics that we want to be able to use in this responder should go here
-    class_attribute :topics
+    # Responder usage contract
+    CONTRACT = Karafka::Contracts::ResponderUsage.new.freeze
 
-    attr_reader :messages_buffer
+    private_constant :CONTRACT
 
     class << self
+      # Definitions of all topics that we want to be able to use in this responder should go here
+      attr_accessor :topics
+      # Contract that we can use to control and/or require some additional details upon options
+      # that are being passed to the producer. This can be in particular useful if we want to make
+      # sure that for example partition_key is always present.
+      attr_accessor :options_contract
+
       # Registers a topic as on to which we will be able to respond
       # @param topic_name [Symbol, String] name of topic to which we want to respond
       # @param options [Hash] hash with optional configuration details
       def topic(topic_name, options = {})
+        options[:serializer] ||= Karafka::App.config.serializer
+        options[:registered] = true
         self.topics ||= {}
-        topic_obj = Responders::Topic.new(topic_name, options.merge(registered: true))
+        topic_obj = Responders::Topic.new(topic_name, options)
         self.topics[topic_obj.name] = topic_obj
       end
 
       # A simple alias for easier standalone responder usage.
-      # Instead of building it with new.call it allows (in case of usin JSON parser)
+      # Instead of building it with new.call it allows (in case of using JSON serializer)
       # to just run it directly from the class level
       # @param data Anything that we want to respond with
-      # @example Send user data with a responder (uses default Karafka::Parsers::Json parser)
+      # @example Send user data with a responder
       #   UsersCreatedResponder.call(@created_user)
       def call(*data)
         # Just in case there were no topics defined for a responder, we initialize with
@@ -88,12 +108,11 @@ module Karafka
       end
     end
 
+    attr_reader :messages_buffer
+
     # Creates a responder object
-    # @param parser_class [Class] parser class that we can use to generate appropriate string
-    #   or nothing if we want to default to Karafka::Parsers::Json
     # @return [Karafka::BaseResponder] base responder descendant responder
-    def initialize(parser_class = Karafka::Parsers::Json)
-      @parser_class = parser_class
+    def initialize
       @messages_buffer = {}
     end
 
@@ -102,13 +121,14 @@ module Karafka
     # @note We know that validators should be executed also before sending data to topics, however
     #   the implementation gets way more complicated then, that's why we check after everything
     #   was sent using responder
-    # @example Send user data with a responder (uses default Karafka::Parsers::Json parser)
+    # @example Send user data with a responder
     #   UsersCreatedResponder.new.call(@created_user)
     # @example Send user data with a responder using non default Parser
     #   UsersCreatedResponder.new(MyParser).call(@created_user)
     def call(*data)
       respond(*data)
-      validate!
+      validate_usage!
+      validate_options!
       deliver!
     end
 
@@ -116,7 +136,7 @@ module Karafka
 
     # Checks if we met all the topics requirements. It will fail if we didn't send a message to
     # a registered required topic, etc.
-    def validate!
+    def validate_usage!
       registered_topics = self.class.topics.map do |name, topic|
         topic.to_h.merge!(
           usage_count: messages_buffer[name]&.count || 0
@@ -128,37 +148,48 @@ module Karafka
         topic.to_h.merge!(usage_count: usage.count)
       end
 
-      result = Karafka::Schemas::ResponderUsage.call(
+      result = CONTRACT.call(
         registered_topics: registered_topics,
         used_topics: used_topics
       )
 
       return if result.success?
 
-      raise Karafka::Errors::InvalidResponderUsage, result.errors
+      raise Karafka::Errors::InvalidResponderUsageError, result.errors.to_h
+    end
+
+    # Checks if we met all the options requirements before sending them to the producer.
+    def validate_options!
+      return true unless self.class.options_contract
+
+      messages_buffer.each_value do |messages_set|
+        messages_set.each do |message_data|
+          result = self.class.options_contract.call(message_data.last)
+          next if result.success?
+
+          raise Karafka::Errors::InvalidResponderMessageOptionsError, result.errors.to_h
+        end
+      end
     end
 
     # Takes all the messages from the buffer and delivers them one by one
     # @note This method is executed after the validation, so we're sure that
     #   what we send is legit and it will go to a proper topics
     def deliver!
-      messages_buffer.each do |topic, data_elements|
-        # We map this topic name, so it will match namespaced/etc topic in Kafka
-        # @note By default will not change topic (if default mapper used)
-        mapped_topic = Karafka::App.config.topic_mapper.outgoing(topic)
-
-        data_elements.each do |(data, options)|
-          ::WaterDrop::Message.new(
-            mapped_topic,
-            data,
-            options
-          ).send!
+      messages_buffer.each_value do |data_elements|
+        data_elements.each do |data, options|
+          # We map this topic name, so it will match namespaced/etc topic in Kafka
+          # @note By default will not change topic (if default mapper used)
+          mapped_topic = Karafka::App.config.topic_mapper.outgoing(options[:topic])
+          external_options = options.merge(topic: mapped_topic)
+          producer(options).call(data, external_options)
         end
       end
     end
 
     # Method that needs to be implemented in a subclass. It should handle responding
     #   on registered topics
+    # @param _data [Object] anything that we want to use to send to Kafka
     # @raise [NotImplementedError] This method needs to be implemented in a subclass
     def respond(*_data)
       raise NotImplementedError, 'Implement this in a subclass'
@@ -168,13 +199,28 @@ module Karafka
     # as many times as we need. Especially when we have 1:n flow
     # @param topic [Symbol, String] topic to which we want to respond
     # @param data [String, Object] string or object that we want to send
-    # @param options [Hash] options for waterdrop (e.g. partition_key)
+    # @param options [Hash] options for waterdrop (e.g. partition_key).
     # @note Respond to does not accept multiple data arguments.
     def respond_to(topic, data, options = {})
-      Karafka.monitor.notice(self.class, topic: topic, data: data, options: options)
+      # We normalize the format to string, as WaterDrop and Ruby-Kafka support only
+      # string topics
+      topic = topic.to_s
+
+      messages_buffer[topic] ||= []
+      messages_buffer[topic] << [
+        self.class.topics[topic].serializer.call(data),
+        options.merge(topic: topic)
+      ]
+    end
 
-      messages_buffer[topic.to_s] ||= []
-      messages_buffer[topic.to_s] << [@parser_class.generate(data), options]
+    # @param options [Hash] options for waterdrop
+    # @return [Class] WaterDrop producer (sync or async based on the settings)
+    def producer(options)
+      if self.class.topics[options[:topic]].async?
+        WaterDrop::AsyncProducer
+      else
+        WaterDrop::SyncProducer
+      end
     end
   end
 end

data/lib/karafka/cli/base.rb

@@ -43,16 +43,16 @@ module Karafka
         end
 
         # Allows to set description of a given cli command
-        # @param desc [String] Description of a given cli command
-        def desc(desc)
-          @desc ||= desc
+        # @param args [Array] All the arguments that Thor desc method accepts
+        def desc(*args)
+          @desc ||= args
         end
 
         # This method will bind a given Cli command into Karafka Cli
         # This method is a wrapper to way Thor defines its commands
         # @param cli_class [Karafka::Cli] Karafka cli_class
         def bind_to(cli_class)
-          cli_class.desc name, @desc
+          cli_class.desc name, *@desc
 
           (@options || []).each { |option| cli_class.option(*option) }
 

data/lib/karafka/cli/console.rb

@@ -8,15 +8,17 @@ module Karafka
       desc 'Start the Karafka console (short-cut alias: "c")'
       option aliases: 'c'
 
-      # @return [String] Console executing command
-      # @example
-      #   Karafka::Cli::Console.command #=> 'KARAFKA_CONSOLE=true bundle exec irb...'
-      def self.command
-        envs = [
-          "IRBRC='#{Karafka.gem_root}/.console_irbrc'",
-          'KARAFKA_CONSOLE=true'
-        ]
-        "#{envs.join(' ')} bundle exec irb"
+      class << self
+        # @return [String] Console executing command
+        # @example
+        #   Karafka::Cli::Console.command #=> 'KARAFKA_CONSOLE=true bundle exec irb...'
+        def command
+          envs = [
+            "IRBRC='#{Karafka.gem_root}/.console_irbrc'",
+            'KARAFKA_CONSOLE=true'
+          ]
+          "#{envs.join(' ')} bundle exec irb -r #{Karafka.boot_file}"
+        end
       end
 
       # Start the Karafka console

data/lib/karafka/cli/flow.rb

@@ -11,20 +11,22 @@ module Karafka
       def call
         topics.each do |topic|
           any_topics = !topic.responder&.topics.nil?
+          log_messages = []
 
           if any_topics
-            puts "#{topic.name} =>"
+            log_messages << "#{topic.name} =>"
 
             topic.responder.topics.each_value do |responder_topic|
               features = []
               features << (responder_topic.required? ? 'always' : 'conditionally')
-              features << (responder_topic.multiple_usage? ? 'one or more' : 'exactly once')
 
-              print responder_topic.name, "(#{features.join(', ')})"
+              log_messages << format(responder_topic.name, "(#{features.join(', ')})")
             end
           else
-            puts "#{topic.name} => (nothing)"
+            log_messages << "#{topic.name} => (nothing)"
           end
+
+          Karafka.logger.info(log_messages.join("\n"))
         end
       end
 
@@ -35,11 +37,11 @@ module Karafka
         Karafka::App.consumer_groups.map(&:topics).flatten.sort_by(&:name)
       end
 
-      # Prints a given value with label in a nice way
+      # Formats a given value with label in a nice way
       # @param label [String] label describing value
       # @param value [String] value that should be printed
-      def print(label, value)
-        printf "%-25s %s\n", "  - #{label}:", value
+      def format(label, value)
+        "  - #{label}:                #{value}"
       end
     end
   end

data/lib/karafka/cli/info.rb

@@ -12,18 +12,19 @@ module Karafka
         config = Karafka::App.config
 
         info = [
-          "Karafka framework version: #{Karafka::VERSION}",
+          "Karafka version: #{Karafka::VERSION}",
+          "Ruby version: #{RUBY_VERSION}",
+          "Ruby-kafka version: #{::Kafka::VERSION}",
           "Application client id: #{config.client_id}",
           "Backend: #{config.backend}",
+          "Batch fetching: #{config.batch_fetching}",
           "Batch consuming: #{config.batch_consuming}",
-          "Batch processing: #{config.batch_processing}",
-          "Number of threads: #{config.concurrency}",
           "Boot file: #{Karafka.boot_file}",
           "Environment: #{Karafka.env}",
           "Kafka seed brokers: #{config.kafka.seed_brokers}"
         ]
 
-        puts(info.join("\n"))
+        Karafka.logger.info(info.join("\n"))
       end
     end
   end

data/lib/karafka/cli/install.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'erb'
+
 module Karafka
   # Karafka framework Cli
   class Cli < Thor
@@ -9,21 +11,33 @@ module Karafka
 
       # Directories created by default
       INSTALL_DIRS = %w[
-        app/models
-        app/controllers
+        app/consumers
         app/responders
+        app/workers
         config
+        lib
         log
         tmp/pids
       ].freeze
 
       # Where should we map proper files from templates
       INSTALL_FILES_MAP = {
-        'karafka.rb.example' => Karafka.boot_file.basename,
-        'application_controller.rb.example' => 'app/controllers/application_controller.rb',
-        'application_responder.rb.example' => 'app/responders/application_responder.rb'
+        'karafka.rb.erb' => Karafka.boot_file.basename,
+        'application_consumer.rb.erb' => 'app/consumers/application_consumer.rb',
+        'application_responder.rb.erb' => 'app/responders/application_responder.rb'
       }.freeze
 
+      # @param args [Array] all the things that Thor CLI accepts
+      def initialize(*args)
+        super
+        dependencies = Bundler::LockfileParser.new(
+          Bundler.read_file(
+            Bundler.default_lockfile
+          )
+        ).dependencies
+        @rails = dependencies.key?('railties') || dependencies.key?('rails')
+      end
+
       # Install all required things for Karafka application in current directory
       def call
         INSTALL_DIRS.each do |dir|
@@ -32,12 +46,22 @@ module Karafka
 
         INSTALL_FILES_MAP.each do |source, target|
           target = Karafka.root.join(target)
-          next if File.exist?(target)
 
-          source = Karafka.core_root.join("templates/#{source}")
-          FileUtils.cp_r(source, target)
+          template = File.read(Karafka.core_root.join("templates/#{source}"))
+          # @todo Replace with the keyword argument version once we don't have to support
+          # Ruby < 2.6
+          render = ::ERB.new(template, nil, '-').result(binding)
+
+          File.open(target, 'w') { |file| file.write(render) }
         end
       end
+
+      # @return [Boolean] true if we have Rails loaded
+      # This allows us to generate customized karafka.rb template with some tweaks specific for
+      # Rails
+      def rails?
+        @rails
+      end
     end
   end
 end

data/lib/karafka/cli/missingno.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Karafka
+  class Cli < Thor
+    # Command that gets invoked when no method is provided when running the CLI
+    # It allows us to exit with exit code 1 instead of default 0 to indicate that something
+    #   was missing
+    # @see https://github.com/karafka/karafka/issues/619
+    class Missingno < Base
+      desc 'Hidden command that gets invoked when no command is provided', hide: true
+
+      # Prints an error about the lack of command (nothing selected)
+      def call
+        Karafka.logger.error('No command provided')
+        exit 1
+      end
+    end
+  end
+end

data/lib/karafka/cli/server.rb

@@ -5,6 +5,11 @@ module Karafka
   class Cli < Thor
     # Server Karafka Cli action
     class Server < Base
+      # Server config settings contract
+      CONTRACT = Contracts::ServerCliOptions.new.freeze
+
+      private_constant :CONTRACT
+
       desc 'Start the Karafka server (short-cut alias: "s")'
       option aliases: 's'
       option :daemon, default: false, type: :boolean, aliases: :d
@@ -13,31 +18,19 @@ module Karafka
 
       # Start the Karafka server
       def call
-        validate!
-
-        puts 'Starting Karafka server'
         cli.info
 
+        validate!
+
         if cli.options[:daemon]
           FileUtils.mkdir_p File.dirname(cli.options[:pid])
-          # For some reason Celluloid spins threads that break forking
-          # Threads are not shutdown immediately so deamonization will stale until
-          # those threads are killed by Celluloid manager (via timeout)
-          # There's nothing initialized here yet, so instead we shutdown celluloid
-          # and run it again when we need (after fork)
-          Celluloid.shutdown
           daemonize
-          Celluloid.boot
         end
 
         # We assign active topics on a server level, as only server is expected to listen on
         # part of the topics
         Karafka::Server.consumer_groups = cli.options[:consumer_groups]
 
-        # Remove pidfile on shutdown, just before the server instance is going to be GCed
-        ObjectSpace.define_finalizer(self, proc { send(:clean) })
-
-        # After we fork, we can boot celluloid again
         Karafka::Server.run
       end
 
@@ -46,9 +39,10 @@ module Karafka
       # Checks the server cli configuration
       # options validations in terms of app setup (topics, pid existence, etc)
       def validate!
-        result = Schemas::ServerCliOptions.call(cli.options)
+        result = CONTRACT.call(cli.options)
         return if result.success?
-        raise Errors::InvalidConfiguration, result.errors
+
+        raise Errors::InvalidConfigurationError, result.errors.to_h
       end
 
       # Detaches current process into background and writes its pidfile
@@ -58,6 +52,14 @@ module Karafka
           cli.options[:pid],
           'w'
         ) { |file| file.write(::Process.pid) }
+
+        # Remove pidfile on stop, just before the server instance is going to be GCed
+        # We want to delay the moment in which the pidfile is removed as much as we can,
+        # so instead of removing it after the server stops running, we rely on the gc moment
+        # when this object gets removed (it is a bit later), so it is closer to the actual
+        # system process end. We do that, so monitoring and deployment tools that rely on a pid
+        # won't alarm or start new system process up until the current one is finished
+        ObjectSpace.define_finalizer(self, proc { send(:clean) })
       end
 
       # Removes a pidfile (if exist)