karafka 1.1.0 → 1.3.0

data/.github/ISSUE_TEMPLATE.md

@@ -1,2 +0,0 @@
-<!-- Love karafka? Please consider supporting our collective:
-👉  https://opencollective.com/karafka/donate -->

data/lib/karafka/base_controller.rb

@@ -1,60 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka module namespace
-module Karafka
-  # Base controller from which all Karafka controllers should inherit
-  class BaseController
-    extend ActiveSupport::DescendantsTracker
-
-    class << self
-      attr_reader :topic
-
-      # Assigns a topic to a controller and build up proper controller functionalities, so it can
-      #   cooperate with the topic settings
-      # @param topic [Karafka::Routing::Topic]
-      # @return [Karafka::Routing::Topic] assigned topic
-      def topic=(topic)
-        @topic = topic
-        Controllers::Includer.call(self)
-      end
-    end
-
-    # @return [Karafka::Routing::Topic] topic to which a given controller is subscribed
-    def topic
-      self.class.topic
-    end
-
-    # Creates lazy loaded params batch object
-    # @note Until first params usage, it won't parse data at all
-    # @param messages [Array<Kafka::FetchedMessage>, Array<Hash>] messages with raw
-    #   content (from Kafka) or messages inside a hash (from backend, etc)
-    # @return [Karafka::Params::ParamsBatch] lazy loaded params batch
-    def params_batch=(messages)
-      @params_batch = Karafka::Params::ParamsBatch.new(messages, topic.parser)
-    end
-
-    # Executes the default controller flow.
-    def call
-      process
-    end
-
-    private
-
-    # We make it private as it should be accesible only from the inside of a controller
-    attr_reader :params_batch
-
-    # @return [Karafka::Connection::Consumer] messages consumer that can be used to
-    #    commit manually offset or pause / stop consumer based on the business logic
-    def consumer
-      Persistence::Consumer.read
-    end
-
-    # Method that will perform business logic and on data received from Kafka (it will consume
-    #   the data)
-    # @note This method needs bo be implemented in a subclass. We stub it here as a failover if
-    #   someone forgets about it or makes on with typo
-    def consume
-      raise NotImplementedError, 'Implement this in a subclass'
-    end
-  end
-end

data/lib/karafka/connection/consumer.rb

@@ -1,121 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Connection
-    # Class used as a wrapper around Ruby-Kafka to simplify additional
-    # features that we provide/might provide in future and to hide the internal implementation
-    class Consumer
-      # Creates a queue consumer that will pull the data from Kafka
-      # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group for which
-      #   we create a client
-      # @return [Karafka::Connection::Consumer] group consumer that can subscribe to
-      #   multiple topics
-      def initialize(consumer_group)
-        @consumer_group = consumer_group
-        Persistence::Consumer.write(self)
-      end
-
-      # Opens connection, gets messages and calls a block for each of the incoming messages
-      # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
-      # @note This will yield with raw messages - no preprocessing or reformatting.
-      def fetch_loop
-        send(
-          consumer_group.batch_fetching ? :consume_each_batch : :consume_each_message
-        ) { |messages| yield(messages) }
-      rescue Kafka::ProcessingError => e
-        # If there was an error during consumption, we have to log it, pause current partition
-        # and process other things
-        Karafka.monitor.notice_error(self.class, e.cause)
-        pause(e.topic, e.partition)
-        retry
-        # This is on purpose - see the notes for this method
-        # rubocop:disable RescueException
-      rescue Exception => e
-        # rubocop:enable RescueException
-        Karafka.monitor.notice_error(self.class, e)
-        retry
-      end
-
-      # Gracefuly stops topic consumption
-      # @note Stopping running consumers without a really important reason is not recommended
-      #   as until all the consumers are stopped, the server will keep running serving only
-      #   part of the messages
-      def stop
-        @kafka_consumer&.stop
-        @kafka_consumer = nil
-      end
-
-      # Pauses fetching and consumption of a given topic partition
-      # @param topic [String] topic that we want to pause
-      # @param partition [Integer] number partition that we want to pause
-      def pause(topic, partition)
-        settings = ConfigAdapter.pausing(consumer_group)
-        timeout = settings[:timeout]
-        raise(Errors::InvalidPauseTimeout, timeout) unless timeout.positive?
-        kafka_consumer.pause(topic, partition, settings)
-      end
-
-      # Marks a given message as consumed and commit the offsets
-      # @note In opposite to ruby-kafka, we commit the offset for each manual marking to be sure
-      #   that offset commit happen asap in case of a crash
-      # @param [Karafka::Params::Params] params message that we want to mark as processed
-      def mark_as_consumed(params)
-        kafka_consumer.mark_message_as_processed(params)
-        # Trigger an immediate, blocking offset commit in order to minimize the risk of crashing
-        # before the automatic triggers have kicked in.
-        kafka_consumer.commit_offsets
-      end
-
-      private
-
-      attr_reader :consumer_group
-
-      # Consumes messages from Kafka in batches
-      # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
-      def consume_each_batch
-        kafka_consumer.each_batch(
-          ConfigAdapter.consuming(consumer_group)
-        ) do |batch|
-          yield(batch.messages)
-        end
-      end
-
-      # Consumes messages from Kafka one by one
-      # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
-      def consume_each_message
-        kafka_consumer.each_message(
-          ConfigAdapter.consuming(consumer_group)
-        ) do |message|
-          #   always yield an array of messages, so we have consistent API (always a batch)
-          yield([message])
-        end
-      end
-
-      # @return [Kafka::Consumer] returns a ready to consume Kafka consumer
-      #   that is set up to consume from topics of a given consumer group
-      def kafka_consumer
-        @kafka_consumer ||= kafka.consumer(
-          ConfigAdapter.consumer(consumer_group)
-        ).tap do |consumer|
-          consumer_group.topics.each do |topic|
-            consumer.subscribe(*ConfigAdapter.subscription(topic))
-          end
-        end
-      rescue Kafka::ConnectionError
-        # If we would not wait it would totally spam log file with failed
-        # attempts if Kafka is down
-        sleep(consumer_group.reconnect_timeout)
-        # We don't log and just reraise - this will be logged
-        # down the road
-        raise
-      end
-
-      # @return [Kafka] returns a Kafka
-      # @note We don't cache it internally because we cache kafka_consumer that uses kafka
-      #   object instance
-      def kafka
-        Kafka.new(ConfigAdapter.client(consumer_group))
-      end
-    end
-  end
-end

data/lib/karafka/connection/processor.rb

@@ -1,61 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Connection
-    # Class that consumes messages for which we listen
-    module Processor
-      class << self
-        # Processes messages (does something with them)
-        # It will either schedule or run a proper controller action for messages
-        # @note This should be looped to obtain a constant listening
-        # @note We catch all the errors here, to make sure that none failures
-        #   for a given consumption will affect other consumed messages
-        #   If we wouldn't catch it, it would propagate up until killing the thread
-        # @param group_id [String] group_id of a group from which a given message came
-        # @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages fetched from kafka
-        def process(group_id, kafka_messages)
-          # @note We always get messages by topic and partition so we can take topic from the
-          # first one and it will be valid for all the messages
-          # We map from incoming topic name, as it might be namespaced, etc.
-          # @see topic_mapper internal docs
-          mapped_topic_name = Karafka::App.config.topic_mapper.incoming(kafka_messages[0].topic)
-          topic = Routing::Router.find("#{group_id}_#{mapped_topic_name}")
-          controller = Persistence::Controller.fetch(topic, kafka_messages[0].partition) do
-            topic.controller.new
-          end
-
-          # Depending on a case (persisted or not) we might use new controller instance per each
-          # batch, or use the same instance for all of them (for implementing buffering, etc)
-          send(
-            topic.batch_consuming ? :process_batch : :process_each,
-            controller,
-            kafka_messages
-          )
-        end
-
-        private
-
-        # Processes whole batch in one request (all at once)
-        # @param controller [Karafka::BaseController] base controller descendant
-        # @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages from kafka
-        def process_batch(controller, kafka_messages)
-          controller.params_batch = kafka_messages
-          Karafka.monitor.notice(self, kafka_messages)
-          controller.call
-        end
-
-        # Processes messages one by one (like with std http requests)
-        # @param controller [Karafka::BaseController] base controller descendant
-        # @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages from kafka
-        def process_each(controller, kafka_messages)
-          kafka_messages.each do |kafka_message|
-            # @note This is a simple trick - we just process one after another, but in order
-            # not to handle everywhere both cases (single vs batch), we just "fake" batching with
-            # a single message for each
-            process_batch(controller, [kafka_message])
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/controllers/callbacks.rb

@@ -1,54 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Controllers
-    # Additional callbacks that can be used to trigger some actions on certain moments like
-    # manual offset management, committing or anything else outside of a standard messages flow
-    # They are not included by default, as we don't want to provide functionalities that are
-    # not required by users by default
-    # Please refer to the wiki callbacks page for more details on how to use them
-    module Callbacks
-      # Types of events on which we run callbacks
-      TYPES = %i[
-        after_fetched
-        after_poll
-        before_poll
-        before_stop
-      ].freeze
-
-      # Class methods needed to make callbacks run
-      module ClassMethods
-        TYPES.each do |type|
-          # A Creates a callback wrapper
-          # @param method_name [Symbol, String] method name or nil if we plan to provide a block
-          # @yield A block with a code that should be executed before scheduling
-          define_method type do |method_name = nil, &block|
-            set_callback type, :before, method_name ? method_name : block
-          end
-        end
-      end
-
-      # @param controller_class [Class] controller class that we extend with callbacks
-      def self.included(controller_class)
-        controller_class.class_eval do
-          extend ClassMethods
-          include ActiveSupport::Callbacks
-
-          # The call method is wrapped with a set of callbacks
-          # We won't run process if any of the callbacks throw abort
-          # @see http://api.rubyonrails.org/classes/ActiveSupport/Callbacks/ClassMethods.html#method-i-get_callbacks
-          TYPES.each { |type| define_callbacks type }
-        end
-      end
-
-      # Executes the default controller flow, runs callbacks and if not halted will call process
-      # method of a proper backend. This is here because it interacts with the default Karafka
-      # call flow and needs to be overwritten in order to support callbacks
-      def call
-        run_callbacks :after_fetched do
-          process
-        end
-      end
-    end
-  end
-end

data/lib/karafka/controllers/includer.rb

@@ -1,51 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Additional functionalities for controllers
-  module Controllers
-    # Module used to inject functionalities into a given controller class, based on the controller
-    # topic and its settings
-    # We don't need all the behaviors in all the cases, so it is totally not worth having
-    # everything in all the cases all the time
-    module Includer
-      class << self
-        # @param controller_class [Class] controller class, that will get some functionalities
-        #   based on the topic under which it operates
-        def call(controller_class)
-          topic = controller_class.topic
-
-          bind_backend(controller_class, topic)
-          bind_params(controller_class, topic)
-          bind_responders(controller_class, topic)
-        end
-
-        private
-
-        # Figures out backend for a given controller class, based on the topic backend and
-        #   includes it into the controller class
-        # @param controller_class [Class] controller class
-        # @param topic [Karafka::Routing::Topic] topic of a controller class
-        def bind_backend(controller_class, topic)
-          backend = Kernel.const_get("::Karafka::Backends::#{topic.backend.to_s.capitalize}")
-          controller_class.include backend
-        end
-
-        # Adds a single #params support for non batch processed topics
-        # @param controller_class [Class] controller class
-        # @param topic [Karafka::Routing::Topic] topic of a controller class
-        def bind_params(controller_class, topic)
-          return if topic.batch_consuming
-          controller_class.include SingleParams
-        end
-
-        # Adds responders support for topics and controllers with responders defined for them
-        # @param controller_class [Class] controller class
-        # @param topic [Karafka::Routing::Topic] topic of a controller class
-        def bind_responders(controller_class, topic)
-          return unless topic.responder
-          controller_class.include Responders
-        end
-      end
-    end
-  end
-end

data/lib/karafka/controllers/responders.rb

@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Controllers
-    # Feature that allows us to use responders flow in controller
-    module Responders
-      # Responds with given data using given responder. This allows us to have a similar way of
-      # defining flows like synchronous protocols
-      # @param data Anything we want to pass to responder based on which we want to trigger further
-      #   Kafka responding
-      def respond_with(*data)
-        Karafka.monitor.notice(self.class, data: data)
-        # @note we build a new instance of responder each time, as a long running (persisted)
-        #   controllers can respond multiple times during the lifecycle
-        topic.responder.new(topic.parser).call(*data)
-      end
-    end
-  end
-end

data/lib/karafka/loader.rb

@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Loader for requiring all the files in a proper order
-  module Loader
-    # Order in which we want to load app files
-    DIRS = %w[
-      config/initializers
-      lib
-      app
-    ].freeze
-
-    # Will load files in a proper order (based on DIRS)
-    # @param [String] root path from which we want to start
-    def self.load(root)
-      DIRS.each do |dir|
-        path = File.join(root, dir)
-        next unless File.exist?(path)
-        load!(path)
-      end
-    end
-
-    # Requires all the ruby files from one path in a proper order
-    # @param path [String] path (dir) from which we want to load ruby files in a proper order
-    def self.load!(path)
-      require_all(path)
-    end
-  end
-end

data/lib/karafka/logger.rb

@@ -1,53 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Default logger for Event Delegator
-  # @note It uses ::Logger features - providing basic logging
-  class Logger < ::Logger
-    include Singleton
-
-    # Map containing informations about log level for given environment
-    ENV_MAP = {
-      'production' => ::Logger::ERROR,
-      'test' => ::Logger::ERROR,
-      'development' => ::Logger::INFO,
-      'debug' => ::Logger::DEBUG,
-      default: ::Logger::INFO
-    }.freeze
-
-    # Creates a new instance of logger ensuring that it has a place to write to
-    def initialize(*_args)
-      ensure_dir_exists
-      super(target)
-      self.level = ENV_MAP[Karafka.env] || ENV_MAP[:default]
-    end
-
-    private
-
-    # @return [Karafka::Helpers::MultiDelegator] multi delegator instance
-    #   to which we will be writtng logs
-    # We use this approach to log stuff to file and to the STDOUT at the same time
-    def target
-      Karafka::Helpers::MultiDelegator
-        .delegate(:write, :close)
-        .to(STDOUT, file)
-    end
-
-    # Makes sure the log directory exists
-    def ensure_dir_exists
-      dir = File.dirname(log_path)
-      FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
-    end
-
-    # @return [Pathname] Path to a file to which we should log
-    def log_path
-      @log_path ||= Karafka::App.root.join("log/#{Karafka.env}.log")
-    end
-
-    # @return [File] file to which we want to write our logs
-    # @note File is being opened in append mode ('a')
-    def file
-      @file ||= File.open(log_path, 'a')
-    end
-  end
-end