karafka 1.1.0

data/lib/karafka/app.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Karafka
+  # App class
+  class App
+    class << self
+      # Sets up the whole configuration
+      # @param [Block] block configuration block
+      def setup(&block)
+        Setup::Config.setup(&block)
+        initialize!
+      end
+
+      # Sets up all the internal components and bootstrap whole app
+      # We need to know details about consumers in order to setup components,
+      # that's why we don't setup them after std setup is done
+      # @raise [Karafka::Errors::InvalidConfiguration] raised when configuration
+      #   doesn't match with ConfigurationSchema
+      def boot!
+        Setup::Config.validate!
+        Setup::Config.setup_components
+      end
+
+      # @return [Karafka::Config] config instance
+      def config
+        Setup::Config.config
+      end
+
+      # @return [Karafka::Routing::Builder] consumers builder instance
+      def consumer_groups
+        Routing::Builder.instance
+      end
+
+      Status.instance_methods(false).each do |delegated|
+        define_method(delegated) do
+          Status.instance.public_send(delegated)
+        end
+      end
+
+      # Methods that should be delegated to Karafka module
+      %i[
+        root
+        env
+        logger monitor
+      ].each do |delegated|
+        define_method(delegated) do
+          Karafka.public_send(delegated)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/attributes_map.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Both Karafka and Ruby-Kafka contain a lot of settings that can be applied on multiple
+  # levels. In Karafka that is on consumer group and on the topic level. In Ruby-Kafka it
+  # is on consumer, subscription and consumption levels. In order to maintain an order
+  # in managing those settings, this module was created. It contains details on what setting
+  # where should go and which layer (both on Karafka and Ruby-Kafka) is responsible for
+  # setting it and sending it forward
+  # @note Settings presented here cover all the settings that are being used across Karafka
+  module AttributesMap
+    class << self
+      # What settings should go where in ruby-kafka
+      # @note All other settings will be passed to Kafka.new method invocation.
+      #   All elements in this hash are just edge cases
+      # @return [Hash] hash with proper sections on what to proxy where in Ruby-Kafka
+      def config_adapter
+        {
+          consumer: %i[
+            session_timeout offset_commit_interval offset_commit_threshold
+            offset_retention_time heartbeat_interval
+          ],
+          subscription: %i[start_from_beginning max_bytes_per_partition],
+          consuming: %i[min_bytes max_wait_time],
+          pausing: %i[pause_timeout],
+          # All the options that are under kafka config namespace, but are not used
+          # directly with kafka api, but from the Karafka user perspective, they are
+          # still related to kafka. They should not be proxied anywhere
+          ignored: %i[reconnect_timeout automatically_mark_as_consumed]
+        }
+      end
+
+      # @return [Array<Symbol>] properties that can be set on a per topic level
+      def topic
+        (config_adapter[:subscription] + %i[
+          backend
+          name
+          parser
+          responder
+          batch_consuming
+          persistent
+        ]).uniq
+      end
+
+      # @return [Array<Symbol>] properties that can be set on a per consumer group level
+      # @note Note that there are settings directly extracted from the config kafka namespace
+      #   I did this that way, so I won't have to repeat same setting keys over and over again
+      #   Thanks to this solution, if any new setting is available for ruby-kafka, we just need
+      #   to add it to our configuration class and it will be handled automatically.
+      def consumer_group
+        # @note We don't ignore the config_adapter[:ignored] values as they should be ignored
+        #   only when proxying details go ruby-kafka. We use ignored fields internally in karafka
+        ignored_settings = config_adapter[:subscription]
+        defined_settings = config_adapter.values.flatten
+        karafka_settings = %i[batch_fetching]
+        # This is a drity and bad hack of dry-configurable to get keys before setting values
+        dynamically_proxied = Karafka::Setup::Config
+                              ._settings
+                              .find { |s| s.name == :kafka }
+                              .value
+                              .instance_variable_get('@klass').settings
+
+        (defined_settings + dynamically_proxied).uniq + karafka_settings - ignored_settings
+      end
+    end
+  end
+end

data/lib/karafka/backends/inline.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace for all different backends Karafka supports
+  module Backends
+    # Backend that just runs stuff asap without any scheduling
+    module Inline
+      private
+
+      # Executes consume code immediately (without enqueuing)
+      def process
+        Karafka.monitor.notice(self.class, params_batch)
+        consume
+      end
+    end
+  end
+end

data/lib/karafka/base_controller.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+# Karafka module namespace
+module Karafka
+  # Base controller from which all Karafka controllers should inherit
+  class BaseController
+    extend ActiveSupport::DescendantsTracker
+
+    class << self
+      attr_reader :topic
+
+      # Assigns a topic to a controller and build up proper controller functionalities, so it can
+      #   cooperate with the topic settings
+      # @param topic [Karafka::Routing::Topic]
+      # @return [Karafka::Routing::Topic] assigned topic
+      def topic=(topic)
+        @topic = topic
+        Controllers::Includer.call(self)
+      end
+    end
+
+    # @return [Karafka::Routing::Topic] topic to which a given controller is subscribed
+    def topic
+      self.class.topic
+    end
+
+    # Creates lazy loaded params batch object
+    # @note Until first params usage, it won't parse data at all
+    # @param messages [Array<Kafka::FetchedMessage>, Array<Hash>] messages with raw
+    #   content (from Kafka) or messages inside a hash (from backend, etc)
+    # @return [Karafka::Params::ParamsBatch] lazy loaded params batch
+    def params_batch=(messages)
+      @params_batch = Karafka::Params::ParamsBatch.new(messages, topic.parser)
+    end
+
+    # Executes the default controller flow.
+    def call
+      process
+    end
+
+    private
+
+    # We make it private as it should be accesible only from the inside of a controller
+    attr_reader :params_batch
+
+    # @return [Karafka::Connection::Consumer] messages consumer that can be used to
+    #    commit manually offset or pause / stop consumer based on the business logic
+    def consumer
+      Persistence::Consumer.read
+    end
+
+    # Method that will perform business logic and on data received from Kafka (it will consume
+    #   the data)
+    # @note This method needs bo be implemented in a subclass. We stub it here as a failover if
+    #   someone forgets about it or makes on with typo
+    def consume
+      raise NotImplementedError, 'Implement this in a subclass'
+    end
+  end
+end

data/lib/karafka/base_responder.rb

@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Base responder from which all Karafka responders should inherit
+  # Similar to Rails responders concept. It allows us to design flow from one app to another
+  # by isolating what responses should be sent (and where) based on a given action
+  # It differs from Rails responders in the way it works: in std http request we can have one
+  # response, here we can have unlimited number of them
+  #
+  # It has a simple API for defining where should we respond (and if it is required)
+  #
+  # @example Basic usage (each registered topic is required to be used by default)
+  #   class Responder < BaseResponder
+  #     topic :new_action
+  #
+  #     def respond(data)
+  #       respond_to :new_action, data
+  #     end
+  #   end
+  #
+  # @example Responding to a topic with extra options
+  #   class Responder < BaseResponder
+  #     topic :new_action
+  #
+  #     def respond(data)
+  #       respond_to :new_action, data, partition_key: 'thing'
+  #     end
+  #   end
+  #
+  # @example Marking topic as not required (we won't have to use it)
+  #   class Responder < BaseResponder
+  #     topic :required_topic
+  #     topic :new_action, required: false
+  #
+  #     def respond(data)
+  #       respond_to :required_topic, data
+  #     end
+  #   end
+  #
+  # @example Multiple times used topic
+  #   class Responder < BaseResponder
+  #     topic :required_topic, multiple_usage: true
+  #
+  #     def respond(data)
+  #       data.each do |subset|
+  #         respond_to :required_topic, subset
+  #       end
+  #     end
+  #   end
+  #
+  # @example Accept multiple arguments to a respond method
+  #   class Responder < BaseResponder
+  #     topic :users_actions
+  #     topic :articles_viewed
+  #
+  #     def respond(user, article)
+  #       respond_to :users_actions, user
+  #       respond_to :articles_viewed, article
+  #     end
+  #   end
+  class BaseResponder
+    # Definitions of all topics that we want to be able to use in this responder should go here
+    class_attribute :topics
+
+    attr_reader :messages_buffer
+
+    class << self
+      # Registers a topic as on to which we will be able to respond
+      # @param topic_name [Symbol, String] name of topic to which we want to respond
+      # @param options [Hash] hash with optional configuration details
+      def topic(topic_name, options = {})
+        self.topics ||= {}
+        topic_obj = Responders::Topic.new(topic_name, options.merge(registered: true))
+        self.topics[topic_obj.name] = topic_obj
+      end
+
+      # A simple alias for easier standalone responder usage.
+      # Instead of building it with new.call it allows (in case of usin JSON parser)
+      # to just run it directly from the class level
+      # @param data Anything that we want to respond with
+      # @example Send user data with a responder (uses default Karafka::Parsers::Json parser)
+      #   UsersCreatedResponder.call(@created_user)
+      def call(*data)
+        # Just in case there were no topics defined for a responder, we initialize with
+        # empty hash not to handle a nil case
+        self.topics ||= {}
+        new.call(*data)
+      end
+    end
+
+    # Creates a responder object
+    # @param parser_class [Class] parser class that we can use to generate appropriate string
+    #   or nothing if we want to default to Karafka::Parsers::Json
+    # @return [Karafka::BaseResponder] base responder descendant responder
+    def initialize(parser_class = Karafka::Parsers::Json)
+      @parser_class = parser_class
+      @messages_buffer = {}
+    end
+
+    # Performs respond and validates that all the response requirement were met
+    # @param data Anything that we want to respond with
+    # @note We know that validators should be executed also before sending data to topics, however
+    #   the implementation gets way more complicated then, that's why we check after everything
+    #   was sent using responder
+    # @example Send user data with a responder (uses default Karafka::Parsers::Json parser)
+    #   UsersCreatedResponder.new.call(@created_user)
+    # @example Send user data with a responder using non default Parser
+    #   UsersCreatedResponder.new(MyParser).call(@created_user)
+    def call(*data)
+      respond(*data)
+      validate!
+      deliver!
+    end
+
+    private
+
+    # Checks if we met all the topics requirements. It will fail if we didn't send a message to
+    # a registered required topic, etc.
+    def validate!
+      registered_topics = self.class.topics.map do |name, topic|
+        topic.to_h.merge!(
+          usage_count: messages_buffer[name]&.count || 0
+        )
+      end
+
+      used_topics = messages_buffer.map do |name, usage|
+        topic = self.class.topics[name] || Responders::Topic.new(name, registered: false)
+        topic.to_h.merge!(usage_count: usage.count)
+      end
+
+      result = Karafka::Schemas::ResponderUsage.call(
+        registered_topics: registered_topics,
+        used_topics: used_topics
+      )
+
+      return if result.success?
+
+      raise Karafka::Errors::InvalidResponderUsage, result.errors
+    end
+
+    # Takes all the messages from the buffer and delivers them one by one
+    # @note This method is executed after the validation, so we're sure that
+    #   what we send is legit and it will go to a proper topics
+    def deliver!
+      messages_buffer.each do |topic, data_elements|
+        # We map this topic name, so it will match namespaced/etc topic in Kafka
+        # @note By default will not change topic (if default mapper used)
+        mapped_topic = Karafka::App.config.topic_mapper.outgoing(topic)
+
+        data_elements.each do |data, options|
+          producer(options).call(
+            data,
+            options.merge(topic: mapped_topic)
+          )
+        end
+      end
+    end
+
+    # Method that needs to be implemented in a subclass. It should handle responding
+    #   on registered topics
+    # @raise [NotImplementedError] This method needs to be implemented in a subclass
+    def respond(*_data)
+      raise NotImplementedError, 'Implement this in a subclass'
+    end
+
+    # This method allow us to respond to a single topic with a given data. It can be used
+    # as many times as we need. Especially when we have 1:n flow
+    # @param topic [Symbol, String] topic to which we want to respond
+    # @param data [String, Object] string or object that we want to send
+    # @param options [Hash] options for waterdrop (e.g. partition_key)
+    # @note Respond to does not accept multiple data arguments.
+    def respond_to(topic, data, options = {})
+      Karafka.monitor.notice(self.class, topic: topic, data: data, options: options)
+
+      messages_buffer[topic.to_s] ||= []
+      messages_buffer[topic.to_s] << [@parser_class.generate(data), options]
+    end
+
+    # @param options [Hash] options for waterdrop
+    # @return [Class] WaterDrop producer (sync or async based on the settings)
+    def producer(options)
+      options[:async] ? WaterDrop::AsyncProducer : WaterDrop::SyncProducer
+    end
+  end
+end

data/lib/karafka/cli.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Karafka framework Cli
+  # If you want to add/modify command that belongs to CLI, please review all commands
+  # available in cli/ directory inside Karafka source code.
+  #
+  # @note Whole Cli is built using Thor
+  # @see https://github.com/erikhuda/thor
+  class Cli < Thor
+    package_name 'Karafka'
+
+    class << self
+      # Loads all Cli commands into Thor framework
+      # This method should be executed before we run Karafka::Cli.start, otherwise we won't
+      # have any Cli commands available
+      def prepare
+        cli_commands.each do |action|
+          action.bind_to(self)
+        end
+      end
+
+      private
+
+      # @return [Array<Class>] Array with Cli action classes that can be used as commands
+      def cli_commands
+        constants
+          .map! { |object| const_get(object) }
+          .keep_if do |object|
+            object.instance_of?(Class) && (object < Cli::Base)
+          end
+      end
+    end
+  end
+end
+
+# This is kinda trick - since we don't have a autoload and other magic stuff
+# like Rails does, so instead this method allows us to replace currently running
+# console with a new one via Kernel.exec. It will start console with new code loaded
+# Yes we know that it is not turbofast, however it is turbo convinient and small
+#
+# Also - the KARAFKA_CONSOLE is used to detect that we're executing the irb session
+# so this method is only available when the Karafka console is running
+#
+# We skip this because this should exist and be only valid in the console
+# :nocov:
+if ENV['KARAFKA_CONSOLE']
+  # Reloads Karafka irb console session
+  def reload!
+    puts "Reloading...\n"
+    Kernel.exec Karafka::Cli::Console.command
+  end
+end
+# :nocov:

data/lib/karafka/cli/base.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Karafka
+  class Cli < Thor
+    # Base class for all the command that we want to define
+    # This base class provides a nicer interface to Thor and allows to easier separate single
+    # independent commands
+    # In order to define a new command you need to:
+    #   - specify its desc
+    #   - implement call method
+    #
+    # @example Create a dummy command
+    #   class Dummy < Base
+    #     self.desc = 'Dummy command'
+    #
+    #     def call
+    #       puts 'I'm doing nothing!
+    #     end
+    #   end
+    class Base
+      include Thor::Shell
+
+      # We can use it to call other cli methods via this object
+      attr_reader :cli
+
+      # @param cli [Karafka::Cli] current Karafka Cli instance
+      def initialize(cli)
+        @cli = cli
+      end
+
+      # This method should implement proper cli action
+      def call
+        raise NotImplementedError, 'Implement this in a subclass'
+      end
+
+      class << self
+        # Allows to set options for Thor cli
+        # @see https://github.com/erikhuda/thor
+        # @param option Single option details
+        def option(*option)
+          @options ||= []
+          @options << option
+        end
+
+        # Allows to set description of a given cli command
+        # @param desc [String] Description of a given cli command
+        def desc(desc)
+          @desc ||= desc
+        end
+
+        # This method will bind a given Cli command into Karafka Cli
+        # This method is a wrapper to way Thor defines its commands
+        # @param cli_class [Karafka::Cli] Karafka cli_class
+        def bind_to(cli_class)
+          cli_class.desc name, @desc
+
+          (@options || []).each { |option| cli_class.option(*option) }
+
+          context = self
+
+          cli_class.send :define_method, name do |*args|
+            context.new(self).call(*args)
+          end
+        end
+
+        private
+
+        # @return [String] downcased current class name that we use to define name for
+        #   given Cli command
+        # @example for Karafka::Cli::Install
+        #   name #=> 'install'
+        def name
+          to_s.split('::').last.downcase
+        end
+      end
+    end
+  end
+end