karafka 1.3.0

data/karafka.gemspec

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require 'karafka/version'
+
+# rubocop:disable Metrics/BlockLength
+Gem::Specification.new do |spec|
+  spec.name        = 'karafka'
+  spec.version     = ::Karafka::VERSION
+  spec.platform    = Gem::Platform::RUBY
+  spec.authors     = ['Maciej Mensfeld', 'Pavlo Vavruk', 'Adam Gwozdowski']
+  spec.email       = %w[maciej@coditsu.io pavlo.vavruk@gmail.com adam99g@gmail.com]
+  spec.homepage    = 'https://github.com/karafka/karafka'
+  spec.summary     = 'Ruby based framework for working with Apache Kafka'
+  spec.description = 'Framework used to simplify Apache Kafka based Ruby applications development'
+  spec.license     = 'MIT'
+
+  spec.add_dependency 'dry-configurable', '~> 0.8'
+  spec.add_dependency 'dry-inflector', '~> 0.1'
+  spec.add_dependency 'dry-monitor', '~> 0.3'
+  spec.add_dependency 'dry-validation', '~> 1.2'
+  spec.add_dependency 'envlogic', '~> 1.1'
+  spec.add_dependency 'irb', '~> 1.0'
+  spec.add_dependency 'multi_json', '>= 1.12'
+  spec.add_dependency 'rake', '>= 11.3'
+  spec.add_dependency 'ruby-kafka', '>= 0.7.8'
+  spec.add_dependency 'thor', '~> 0.20'
+  spec.add_dependency 'waterdrop', '~> 1.3.0'
+  spec.add_dependency 'zeitwerk', '~> 2.1'
+
+  spec.required_ruby_version = '>= 2.4.0'
+
+  if $PROGRAM_NAME.end_with?('gem')
+    spec.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')
+  end
+
+  spec.cert_chain    = %w[certs/mensfeld.pem]
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = %w[lib]
+end
+# rubocop:enable Metrics/BlockLength

data/lib/karafka.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+%w[
+  English
+  waterdrop
+  kafka
+  envlogic
+  thor
+  forwardable
+  fileutils
+  multi_json
+  dry-configurable
+  dry-validation
+  dry/events/publisher
+  dry/inflector
+  dry/monitor/notifications
+  dry/core/constants
+  zeitwerk
+].each(&method(:require))
+
+# Karafka library
+module Karafka
+  extend Envlogic
+
+  class << self
+    # @return [Logger] logger that we want to use. Will use ::Karafka::Logger by default
+    def logger
+      @logger ||= App.config.logger
+    end
+
+    # @return [::Karafka::Monitor] monitor that we want to use
+    def monitor
+      @monitor ||= App.config.monitor
+    end
+
+    # @return [String] root path of this gem
+    def gem_root
+      Pathname.new(File.expand_path('..', __dir__))
+    end
+
+    # @return [String] Karafka app root path (user application path)
+    def root
+      Pathname.new(ENV['KARAFKA_ROOT_DIR'] || File.dirname(ENV['BUNDLE_GEMFILE']))
+    end
+
+    # @return [String] path to Karafka gem root core
+    def core_root
+      Pathname.new(File.expand_path('karafka', __dir__))
+    end
+
+    # @return [String] path to a default file that contains booting procedure etc
+    # @note By default it is a file called 'karafka.rb' but it can be specified as you wish if you
+    #   have Karafka that is merged into a Sinatra/Rails app and karafka.rb is taken.
+    #   It will be used for console/consumers/etc
+    # @example Standard only-Karafka case
+    #   Karafka.boot_file #=> '/home/app_path/karafka.rb'
+    # @example Non standard case
+    #   KARAFKA_BOOT_FILE='/home/app_path/app.rb'
+    #   Karafka.boot_file #=> '/home/app_path/app.rb'
+    def boot_file
+      Pathname.new(ENV['KARAFKA_BOOT_FILE'] || File.join(Karafka.root, 'karafka.rb'))
+    end
+  end
+end
+
+Zeitwerk::Loader
+  .for_gem
+  .tap(&:setup)
+  .tap(&:eager_load)
+
+Kafka::Consumer.prepend(Karafka::Patches::RubyKafka)

data/lib/karafka/app.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Karafka
+  # App class
+  class App
+    extend Setup::Dsl
+
+    class << self
+      # Sets up all the internal components and bootstrap whole app
+      # We need to know details about consumers in order to setup components,
+      # that's why we don't setup them after std setup is done
+      # @raise [Karafka::Errors::InvalidConfigurationError] raised when configuration
+      #   doesn't match with the config contract
+      def boot!
+        initialize!
+        Setup::Config.validate!
+        Setup::Config.setup_components
+        initialized!
+      end
+
+      # @return [Karafka::Routing::Builder] consumers builder instance
+      def consumer_groups
+        config.internal.routing_builder
+      end
+
+      # Triggers reload of all cached Karafka app components, so we can use in-process
+      # in-development hot code reloading without Karafka process restart
+      def reload
+        Karafka::Persistence::Consumers.clear
+        Karafka::Persistence::Topics.clear
+        config.internal.routing_builder.reload
+      end
+
+      Status.instance_methods(false).each do |delegated|
+        define_method(delegated) do
+          App.config.internal.status.send(delegated)
+        end
+      end
+
+      # Methods that should be delegated to Karafka module
+      %i[
+        root
+        env
+        logger
+        monitor
+      ].each do |delegated|
+        define_method(delegated) do
+          Karafka.send(delegated)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/attributes_map.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Both Karafka and Ruby-Kafka contain a lot of settings that can be applied on multiple
+  # levels. In Karafka that is on consumer group and on the topic level. In Ruby-Kafka it
+  # is on consumer, subscription and consumption levels. In order to maintain an order
+  # in managing those settings, this module was created. It contains details on what setting
+  # where should go and which layer (both on Karafka and Ruby-Kafka) is responsible for
+  # setting it and sending it forward
+  # @note Settings presented here cover all the settings that are being used across Karafka
+  module AttributesMap
+    class << self
+      # What settings should go where in ruby-kafka
+      # @return [Hash] hash with proper sections on what to proxy where in Ruby-Kafka
+      # @note All other settings will be passed to Kafka.new method invocation.
+      #   All elements in this hash are just edge cases
+      def api_adapter
+        {
+          consumer: %i[
+            session_timeout offset_commit_interval offset_commit_threshold
+            offset_retention_time heartbeat_interval fetcher_max_queue_size
+          ],
+          subscribe: %i[start_from_beginning max_bytes_per_partition],
+          consumption: %i[min_bytes max_bytes max_wait_time],
+          pause: %i[pause_timeout pause_max_timeout pause_exponential_backoff],
+          # All the options that are under kafka config namespace, but are not used
+          # directly with kafka api, but from the Karafka user perspective, they are
+          # still related to kafka. They should not be proxied anywhere
+          ignored: %i[reconnect_timeout automatically_mark_as_consumed]
+        }
+      end
+
+      # @return [Array<Symbol>] properties that can be set on a per topic level
+      def topic
+        (api_adapter[:subscribe] + %i[
+          backend
+          name
+          deserializer
+          responder
+          batch_consuming
+        ]).uniq
+      end
+
+      # @return [Array<Symbol>] properties that can be set on a per consumer group level
+      # @note Note that there are settings directly extracted from the config kafka namespace
+      #   I did this that way, so I won't have to repeat same setting keys over and over again
+      #   Thanks to this solution, if any new setting is available for ruby-kafka, we just need
+      #   to add it to our configuration class and it will be handled automatically.
+      def consumer_group
+        # @note We don't ignore the api_adapter[:ignored] values as they should be ignored
+        #   only when proxying details go ruby-kafka. We use ignored fields internally in karafka
+        ignored_settings = api_adapter[:subscribe]
+        defined_settings = api_adapter.values.flatten
+        karafka_settings = %i[batch_fetching]
+        # This is a dirty and bad hack of dry-configurable to get keys before setting values
+        dynamically_proxied = Karafka::Setup::Config
+                              ._settings
+                              .settings
+                              .find { |s| s.name == :kafka }
+                              .value
+                              .names
+                              .to_a
+
+        (defined_settings + dynamically_proxied).uniq + karafka_settings - ignored_settings
+      end
+    end
+  end
+end

data/lib/karafka/backends/inline.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace for all different backends Karafka supports
+  module Backends
+    # Backend that just runs stuff asap without any scheduling
+    module Inline
+      private
+
+      # Executes consume code immediately (without enqueuing)
+      def process
+        Karafka.monitor.instrument('backends.inline.process', caller: self) { consume }
+      end
+    end
+  end
+end

data/lib/karafka/base_consumer.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+# Karafka module namespace
+module Karafka
+  # Base consumer from which all Karafka consumers should inherit
+  class BaseConsumer
+    extend Forwardable
+
+    # Allows us to mark messages as consumed for non-automatic mode without having
+    # to use consumer client directly. We do this that way, because most of the people should not
+    # mess with the client instance directly (just in case)
+    %i[
+      mark_as_consumed
+      mark_as_consumed!
+      trigger_heartbeat
+      trigger_heartbeat!
+    ].each do |delegated_method_name|
+      def_delegator :client, delegated_method_name
+
+      private delegated_method_name
+    end
+
+    # @return [Karafka::Routing::Topic] topic to which a given consumer is subscribed
+    attr_reader :topic
+    # @return [Karafka::Params:ParamsBatch] current params batch
+    attr_accessor :params_batch
+
+    # Assigns a topic to a consumer and builds up proper consumer functionalities
+    #   so that it can cooperate with the topic settings
+    # @param topic [Karafka::Routing::Topic]
+    def initialize(topic)
+      @topic = topic
+      Consumers::Includer.call(self)
+    end
+
+    # Executes the default consumer flow.
+    def call
+      process
+    end
+
+    private
+
+    # @return [Karafka::Connection::Client] messages consuming client that can be used to
+    #    commit manually offset or pause / stop consumer based on the business logic
+    def client
+      Persistence::Client.read
+    end
+
+    # Method that will perform business logic and on data received from Kafka (it will consume
+    #   the data)
+    # @note This method needs bo be implemented in a subclass. We stub it here as a failover if
+    #   someone forgets about it or makes on with typo
+    def consume
+      raise NotImplementedError, 'Implement this in a subclass'
+    end
+  end
+end

data/lib/karafka/base_responder.rb

@@ -0,0 +1,226 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Base responder from which all Karafka responders should inherit
+  # Similar to Rails responders concept. It allows us to design flow from one app to another
+  # by isolating what responses should be sent (and where) based on a given action
+  # It differs from Rails responders in the way it works: in std http request we can have one
+  # response, here we can have unlimited number of them
+  #
+  # It has a simple API for defining where should we respond (and if it is required)
+  #
+  # @example Basic usage (each registered topic is required to be used by default)
+  #   class Responder < BaseResponder
+  #     topic :new_action
+  #
+  #     def respond(data)
+  #       respond_to :new_action, data
+  #     end
+  #   end
+  #
+  # @example Responding to a topic with extra options
+  #   class Responder < BaseResponder
+  #     topic :new_action
+  #
+  #     def respond(data)
+  #       respond_to :new_action, data, partition_key: 'thing'
+  #     end
+  #   end
+  #
+  # @example Marking topic as not required (we won't have to use it)
+  #   class Responder < BaseResponder
+  #     topic :required_topic
+  #     topic :new_action, required: false
+  #
+  #     def respond(data)
+  #       respond_to :required_topic, data
+  #     end
+  #   end
+  #
+  # @example Multiple times used topic
+  #   class Responder < BaseResponder
+  #     topic :required_topic
+  #
+  #     def respond(data)
+  #       data.each do |subset|
+  #         respond_to :required_topic, subset
+  #       end
+  #     end
+  #   end
+  #
+  # @example Specify serializer for a topic
+  #   class Responder < BaseResponder
+  #     topic :xml_topic, serializer: MyXMLSerializer
+  #
+  #     def respond(data)
+  #       data.each do |subset|
+  #         respond_to :xml_topic, subset
+  #       end
+  #     end
+  #   end
+  #
+  # @example Accept multiple arguments to a respond method
+  #   class Responder < BaseResponder
+  #     topic :users_actions
+  #     topic :articles_viewed
+  #
+  #     def respond(user, article)
+  #       respond_to :users_actions, user
+  #       respond_to :articles_viewed, article
+  #     end
+  #   end
+  class BaseResponder
+    # Responder usage contract
+    CONTRACT = Karafka::Contracts::ResponderUsage.new.freeze
+
+    private_constant :CONTRACT
+
+    class << self
+      # Definitions of all topics that we want to be able to use in this responder should go here
+      attr_accessor :topics
+      # Contract that we can use to control and/or require some additional details upon options
+      # that are being passed to the producer. This can be in particular useful if we want to make
+      # sure that for example partition_key is always present.
+      attr_accessor :options_contract
+
+      # Registers a topic as on to which we will be able to respond
+      # @param topic_name [Symbol, String] name of topic to which we want to respond
+      # @param options [Hash] hash with optional configuration details
+      def topic(topic_name, options = {})
+        options[:serializer] ||= Karafka::App.config.serializer
+        options[:registered] = true
+        self.topics ||= {}
+        topic_obj = Responders::Topic.new(topic_name, options)
+        self.topics[topic_obj.name] = topic_obj
+      end
+
+      # A simple alias for easier standalone responder usage.
+      # Instead of building it with new.call it allows (in case of using JSON serializer)
+      # to just run it directly from the class level
+      # @param data Anything that we want to respond with
+      # @example Send user data with a responder
+      #   UsersCreatedResponder.call(@created_user)
+      def call(*data)
+        # Just in case there were no topics defined for a responder, we initialize with
+        # empty hash not to handle a nil case
+        self.topics ||= {}
+        new.call(*data)
+      end
+    end
+
+    attr_reader :messages_buffer
+
+    # Creates a responder object
+    # @return [Karafka::BaseResponder] base responder descendant responder
+    def initialize
+      @messages_buffer = {}
+    end
+
+    # Performs respond and validates that all the response requirement were met
+    # @param data Anything that we want to respond with
+    # @note We know that validators should be executed also before sending data to topics, however
+    #   the implementation gets way more complicated then, that's why we check after everything
+    #   was sent using responder
+    # @example Send user data with a responder
+    #   UsersCreatedResponder.new.call(@created_user)
+    # @example Send user data with a responder using non default Parser
+    #   UsersCreatedResponder.new(MyParser).call(@created_user)
+    def call(*data)
+      respond(*data)
+      validate_usage!
+      validate_options!
+      deliver!
+    end
+
+    private
+
+    # Checks if we met all the topics requirements. It will fail if we didn't send a message to
+    # a registered required topic, etc.
+    def validate_usage!
+      registered_topics = self.class.topics.map do |name, topic|
+        topic.to_h.merge!(
+          usage_count: messages_buffer[name]&.count || 0
+        )
+      end
+
+      used_topics = messages_buffer.map do |name, usage|
+        topic = self.class.topics[name] || Responders::Topic.new(name, registered: false)
+        topic.to_h.merge!(usage_count: usage.count)
+      end
+
+      result = CONTRACT.call(
+        registered_topics: registered_topics,
+        used_topics: used_topics
+      )
+
+      return if result.success?
+
+      raise Karafka::Errors::InvalidResponderUsageError, result.errors.to_h
+    end
+
+    # Checks if we met all the options requirements before sending them to the producer.
+    def validate_options!
+      return true unless self.class.options_contract
+
+      messages_buffer.each_value do |messages_set|
+        messages_set.each do |message_data|
+          result = self.class.options_contract.call(message_data.last)
+          next if result.success?
+
+          raise Karafka::Errors::InvalidResponderMessageOptionsError, result.errors.to_h
+        end
+      end
+    end
+
+    # Takes all the messages from the buffer and delivers them one by one
+    # @note This method is executed after the validation, so we're sure that
+    #   what we send is legit and it will go to a proper topics
+    def deliver!
+      messages_buffer.each_value do |data_elements|
+        data_elements.each do |data, options|
+          # We map this topic name, so it will match namespaced/etc topic in Kafka
+          # @note By default will not change topic (if default mapper used)
+          mapped_topic = Karafka::App.config.topic_mapper.outgoing(options[:topic])
+          external_options = options.merge(topic: mapped_topic)
+          producer(options).call(data, external_options)
+        end
+      end
+    end
+
+    # Method that needs to be implemented in a subclass. It should handle responding
+    #   on registered topics
+    # @param _data [Object] anything that we want to use to send to Kafka
+    # @raise [NotImplementedError] This method needs to be implemented in a subclass
+    def respond(*_data)
+      raise NotImplementedError, 'Implement this in a subclass'
+    end
+
+    # This method allow us to respond to a single topic with a given data. It can be used
+    # as many times as we need. Especially when we have 1:n flow
+    # @param topic [Symbol, String] topic to which we want to respond
+    # @param data [String, Object] string or object that we want to send
+    # @param options [Hash] options for waterdrop (e.g. partition_key).
+    # @note Respond to does not accept multiple data arguments.
+    def respond_to(topic, data, options = {})
+      # We normalize the format to string, as WaterDrop and Ruby-Kafka support only
+      # string topics
+      topic = topic.to_s
+
+      messages_buffer[topic] ||= []
+      messages_buffer[topic] << [
+        self.class.topics[topic].serializer.call(data),
+        options.merge(topic: topic)
+      ]
+    end
+
+    # @param options [Hash] options for waterdrop
+    # @return [Class] WaterDrop producer (sync or async based on the settings)
+    def producer(options)
+      if self.class.topics[options[:topic]].async?
+        WaterDrop::AsyncProducer
+      else
+        WaterDrop::SyncProducer
+      end
+    end
+  end
+end