karafka 1.0.0 → 1.2.0

data/.github/ISSUE_TEMPLATE.md

@@ -1,2 +0,0 @@
-<!-- Love karafka? Please consider supporting our collective:
-👉  https://opencollective.com/karafka/donate -->

data/Rakefile

@@ -1,7 +0,0 @@
-# frozen_string_literal: true
-
-require 'rspec'
-require 'rspec/core/rake_task'
-
-RSpec::Core::RakeTask.new(:spec)
-task default: :spec

data/lib/karafka/base_controller.rb

@@ -1,118 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka module namespace
-module Karafka
-  # Base controller from which all Karafka controllers should inherit
-  # Similar to Rails controllers we can define after_received callbacks
-  # that will be executed
-  #
-  # Note that if after_received return false, the chain will be stopped and
-  #   the perform method won't be executed
-  #
-  # @example Create simple controller
-  #   class ExamplesController < Karafka::BaseController
-  #     def perform
-  #       # some logic here
-  #     end
-  #   end
-  #
-  # @example Create a controller with a block after_received
-  #   class ExampleController < Karafka::BaseController
-  #     after_received do
-  #       # Here we should have some checking logic
-  #       # If false is returned, won't schedule a perform action
-  #     end
-  #
-  #     def perform
-  #       # some logic here
-  #     end
-  #   end
-  #
-  # @example Create a controller with a method after_received
-  #   class ExampleController < Karafka::BaseController
-  #     after_received :after_received_method
-  #
-  #     def perform
-  #       # some logic here
-  #     end
-  #
-  #     private
-  #
-  #     def after_received_method
-  #       # Here we should have some checking logic
-  #       # If false is returned, won't schedule a perform action
-  #     end
-  #   end
-  class BaseController
-    extend ActiveSupport::DescendantsTracker
-    include ActiveSupport::Callbacks
-
-    # The call method is wrapped with a set of callbacks
-    # We won't run perform at the backend if any of the callbacks
-    # returns false
-    # @see http://api.rubyonrails.org/classes/ActiveSupport/Callbacks/ClassMethods.html#method-i-get_callbacks
-    define_callbacks :after_received
-
-    attr_accessor :params_batch
-
-    class << self
-      attr_reader :topic
-
-      # Assigns a topic to a controller and build up proper controller functionalities, so it can
-      #   cooperate with the topic settings
-      # @param topic [Karafka::Routing::Topic]
-      # @return [Karafka::Routing::Topic] assigned topic
-      def topic=(topic)
-        @topic = topic
-        Controllers::Includer.call(self)
-        @topic
-      end
-
-      # Creates a callback that will be executed after receiving message but before executing the
-      #   backend for processing
-      # @param method_name [Symbol, String] method name or nil if we plan to provide a block
-      # @yield A block with a code that should be executed before scheduling
-      # @example Define a block after_received callback
-      #   after_received do
-      #     # logic here
-      #   end
-      #
-      # @example Define a class name after_received callback
-      #   after_received :method_name
-      def after_received(method_name = nil, &block)
-        set_callback :after_received, :before, method_name ? method_name : block
-      end
-    end
-
-    # @return [Karafka::Routing::Topic] topic to which a given controller is subscribed
-    def topic
-      self.class.topic
-    end
-
-    # Creates lazy loaded params batch object
-    # @note Until first params usage, it won't parse data at all
-    # @param messages [Array<Kafka::FetchedMessage>, Array<Hash>] messages with raw
-    #   content (from Kafka) or messages inside a hash (from backend, etc)
-    # @return [Karafka::Params::ParamsBatch] lazy loaded params batch
-    def params_batch=(messages)
-      @params_batch = Karafka::Params::ParamsBatch.new(messages, topic.parser)
-    end
-
-    # Executes the default controller flow, runs callbacks and if not halted
-    # will call process method of a proper backend
-    def call
-      run_callbacks :after_received do
-        process
-      end
-    end
-
-    private
-
-    # Method that will perform business logic on data received from Kafka
-    # @note This method needs bo be implemented in a subclass. We stub it here as a failover if
-    #   someone forgets about it or makes on with typo
-    def perform
-      raise NotImplementedError, 'Implement this in a subclass'
-    end
-  end
-end

data/lib/karafka/connection/messages_consumer.rb

@@ -1,106 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Connection
-    # Class used as a wrapper around Ruby-Kafka to simplify additional
-    # features that we provide/might provide in future
-    class MessagesConsumer
-      # Creates a queue consumer that will pull the data from Kafka
-      # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group for which
-      #   we create a client
-      # @return [Karafka::Connection::MessagesConsumer] group consumer that can subscribe to
-      #   multiple topics
-      def initialize(consumer_group)
-        @consumer_group = consumer_group
-      end
-
-      # Opens connection, gets messages and calls a block for each of the incoming messages
-      # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
-      # @note This will yield with raw messages - no preprocessing or reformatting.
-      def fetch_loop
-        send(
-          consumer_group.batch_consuming ? :consume_each_batch : :consume_each_message
-        ) { |messages| yield(messages) }
-      rescue Kafka::ProcessingError => e
-        # If there was an error during processing, we have to log it, pause current partition
-        # and process other things
-        Karafka.monitor.notice_error(self.class, e.cause)
-        pause(e.topic, e.partition)
-        retry
-        # This is on purpose - see the notes for this method
-        # rubocop:disable RescueException
-      rescue Exception => e
-        # rubocop:enable RescueException
-        Karafka.monitor.notice_error(self.class, e)
-        retry
-      end
-
-      # Gracefuly stops topic consumption
-      def stop
-        @kafka_consumer&.stop
-        @kafka_consumer = nil
-      end
-
-      private
-
-      attr_reader :consumer_group
-
-      # Pauses processing of a given topic partition
-      # @param topic [String] topic that we want to pause
-      # @param partition [Integer] number partition that we want to pause
-      def pause(topic, partition)
-        settings = ConfigAdapter.pausing(consumer_group)
-        return false unless settings[:timeout].positive?
-        kafka_consumer.pause(topic, partition, settings)
-        true
-      end
-
-      # Consumes messages from Kafka in batches
-      # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
-      def consume_each_batch
-        kafka_consumer.each_batch(
-          ConfigAdapter.consuming(consumer_group)
-        ) do |batch|
-          yield(batch.messages)
-        end
-      end
-
-      # Consumes messages from Kafka one by one
-      # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
-      def consume_each_message
-        kafka_consumer.each_message(
-          ConfigAdapter.consuming(consumer_group)
-        ) do |message|
-          #   always yield an array of messages, so we have consistent API (always a batch)
-          yield([message])
-        end
-      end
-
-      # @return [Kafka::Consumer] returns a ready to consume Kafka consumer
-      #   that is set up to consume from topics of a given consumer group
-      def kafka_consumer
-        @kafka_consumer ||= kafka.consumer(
-          ConfigAdapter.consumer(consumer_group)
-        ).tap do |consumer|
-          consumer_group.topics.each do |topic|
-            consumer.subscribe(*ConfigAdapter.subscription(topic))
-          end
-        end
-      rescue Kafka::ConnectionError
-        # If we would not wait it would totally spam log file with failed
-        # attempts if Kafka is down
-        sleep(consumer_group.reconnect_timeout)
-        # We don't log and just reraise - this will be logged
-        # down the road
-        raise
-      end
-
-      # @return [Kafka] returns a Kafka
-      # @note We don't cache it internally because we cache kafka_consumer that uses kafka
-      #   object instance
-      def kafka
-        Kafka.new(ConfigAdapter.client(consumer_group))
-      end
-    end
-  end
-end

data/lib/karafka/connection/messages_processor.rb

@@ -1,59 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Connection
-    # Class that consumes messages for which we listen
-    module MessagesProcessor
-      class << self
-        # Processes messages (does something with them)
-        # It will either schedule or run a proper controller action for messages
-        # @note This should be looped to obtain a constant listening
-        # @note We catch all the errors here, to make sure that none failures
-        #   for a given consumption will affect other consumed messages
-        #   If we wouldn't catch it, it would propagate up until killing the Celluloid actor
-        # @param group_id [String] group_id of a group from which a given message came
-        # @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages fetched from kafka
-        def process(group_id, kafka_messages)
-          # @note We always get messages by topic and partition so we can take topic from the
-          # first one and it will be valid for all the messages
-          # We map from incoming topic name, as it might be namespaced, etc.
-          # @see topic_mapper internal docs
-          mapped_topic_name = Karafka::App.config.topic_mapper.incoming(kafka_messages[0].topic)
-          topic = Routing::Router.find("#{group_id}_#{mapped_topic_name}")
-
-          # Depending on a case (persisted or not) we might use new controller instance per each
-          # batch, or use the same instance for all of them (for implementing buffering, etc)
-          controller = Persistence.fetch(topic, kafka_messages[0].partition, :controller) do
-            topic.controller.new
-          end
-
-          handler = topic.batch_processing ? :process_batch : :process_each
-          send(handler, controller, kafka_messages)
-        end
-
-        private
-
-        # Processes whole batch in one request (all at once)
-        # @param controller [Karafka::BaseController] base controller descendant
-        # @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages from kafka
-        def process_batch(controller, kafka_messages)
-          controller.params_batch = kafka_messages
-          Karafka.monitor.notice(self, kafka_messages)
-          controller.call
-        end
-
-        # Processes messages one by one (like with std http requests)
-        # @param controller [Karafka::BaseController] base controller descendant
-        # @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages from kafka
-        def process_each(controller, kafka_messages)
-          kafka_messages.each do |kafka_message|
-            # @note This is a simple trick - we just process one after another, but in order
-            # not to handle everywhere both cases (single vs batch), we just "fake" batching with
-            # a single message for each
-            process_batch(controller, [kafka_message])
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/controllers/includer.rb

@@ -1,51 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Additional functionalities for controllers
-  module Controllers
-    # Module used to inject functionalities into a given controller class, based on the controller
-    # topic and its settings
-    # We don't need all the behaviors in all the cases, so it is totally not worth having
-    # everything in all the cases all the time
-    module Includer
-      class << self
-        # @param controller_class [Class] controller class, that will get some functionalities
-        #   based on the topic under which it operates
-        def call(controller_class)
-          topic = controller_class.topic
-
-          bind_backend(controller_class, topic)
-          bind_params(controller_class, topic)
-          bind_responders(controller_class, topic)
-        end
-
-        private
-
-        # Figures out backend for a given controller class, based on the topic backend and
-        #   includes it into the controller class
-        # @param controller_class [Class] controller class
-        # @param topic [Karafka::Routing::Topic] topic of a controller class
-        def bind_backend(controller_class, topic)
-          backend = Kernel.const_get("::Karafka::Backends::#{topic.backend.to_s.capitalize}")
-          controller_class.include backend
-        end
-
-        # Adds a single #params support for non batch processed topics
-        # @param controller_class [Class] controller class
-        # @param topic [Karafka::Routing::Topic] topic of a controller class
-        def bind_params(controller_class, topic)
-          return if topic.batch_processing
-          controller_class.include SingleParams
-        end
-
-        # Adds responders support for topics and controllers with responders defined for them
-        # @param controller_class [Class] controller class
-        # @param topic [Karafka::Routing::Topic] topic of a controller class
-        def bind_responders(controller_class, topic)
-          return unless topic.responder
-          controller_class.include Responders
-        end
-      end
-    end
-  end
-end

data/lib/karafka/controllers/responders.rb

@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Controllers
-    # Feature that allows us to use responders flow in controller
-    module Responders
-      # Responds with given data using given responder. This allows us to have a similar way of
-      # defining flows like synchronous protocols
-      # @param data Anything we want to pass to responder based on which we want to trigger further
-      #   Kafka responding
-      def respond_with(*data)
-        Karafka.monitor.notice(self.class, data: data)
-        # @note we build a new instance of responder each time, as a long running (persisted)
-        #   controllers can respond multiple times during the lifecycle
-        topic.responder.new(topic.parser).call(*data)
-      end
-    end
-  end
-end

data/lib/karafka/logger.rb

@@ -1,53 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Default logger for Event Delegator
-  # @note It uses ::Logger features - providing basic logging
-  class Logger < ::Logger
-    include Singleton
-
-    # Map containing informations about log level for given environment
-    ENV_MAP = {
-      'production' => ::Logger::ERROR,
-      'test' => ::Logger::ERROR,
-      'development' => ::Logger::INFO,
-      'debug' => ::Logger::DEBUG,
-      default: ::Logger::INFO
-    }.freeze
-
-    # Creates a new instance of logger ensuring that it has a place to write to
-    def initialize(*_args)
-      ensure_dir_exists
-      super(target)
-      self.level = ENV_MAP[Karafka.env] || ENV_MAP[:default]
-    end
-
-    private
-
-    # @return [Karafka::Helpers::MultiDelegator] multi delegator instance
-    #   to which we will be writtng logs
-    # We use this approach to log stuff to file and to the STDOUT at the same time
-    def target
-      Karafka::Helpers::MultiDelegator
-        .delegate(:write, :close)
-        .to(STDOUT, file)
-    end
-
-    # Makes sure the log directory exists
-    def ensure_dir_exists
-      dir = File.dirname(log_path)
-      FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
-    end
-
-    # @return [Pathname] Path to a file to which we should log
-    def log_path
-      @log_path ||= Karafka::App.root.join("log/#{Karafka.env}.log")
-    end
-
-    # @return [File] file to which we want to write our logs
-    # @note File is being opened in append mode ('a')
-    def file
-      @file ||= File.open(log_path, 'a')
-    end
-  end
-end

data/lib/karafka/monitor.rb

@@ -1,98 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Monitor is used to hookup external monitoring services to monitor how Karafka works
-  # It provides a standarized API for checking incoming messages/enqueueing etc
-  # By default it implements logging functionalities but can be replaced with any more
-  #   sophisticated logging/monitoring system like Errbit, Airbrake, NewRelic
-  # @note This class acts as a singleton because we are only permitted to have single monitor
-  #   per running process (just as logger)
-  # Keep in mind, that if you create your own monitor object, you will have to implement also
-  # logging functionality (or just inherit, super and do whatever you want)
-  class Monitor
-    include Singleton
-
-    # This method is executed in many important places in the code (during data flow), like
-    # the moment before #perform_async, etc. For full list just grep for 'monitor.notice'
-    # @param caller_class [Class] class of object that executed this call
-    # @param options [Hash] hash with options that we passed to notice. It differs based
-    #   on of who and when is calling
-    # @note We don't provide a name of method in which this was called, because we can take
-    #   it directly from Ruby (see #caller_label method of this class for more details)
-    # @example Notice about consuming with controller_class
-    #   Karafka.monitor.notice(self.class, controller_class: controller_class)
-    # @example Notice about terminating with a signal
-    #   Karafka.monitor.notice(self.class, signal: signal)
-    def notice(caller_class, options = {})
-      logger.info("#{caller_class}##{caller_label} with #{options}")
-    end
-
-    # This method is executed when we want to notify about an error that happened somewhere
-    # in the system
-    # @param caller_class [Class] class of object that executed this call
-    # @param e [Exception] exception that was raised
-    # @note We don't provide a name of method in which this was called, because we can take
-    #   it directly from Ruby (see #caller_label method of this class for more details)
-    # @example Notify about error
-    #   Karafka.monitor.notice(self.class, e)
-    def notice_error(caller_class, e)
-      caller_exceptions_map.each do |level, types|
-        next unless types.include?(caller_class)
-
-        return logger.public_send(level, e)
-      end
-
-      logger.info(e)
-    end
-
-    private
-
-    # @return [Hash] Hash containing informations on which level of notification should
-    #   we use for exceptions that happen in certain parts of Karafka
-    # @note Keep in mind that any not handled here class should be logged with info
-    # @note Those are not maps of exceptions classes but of classes that were callers of this
-    #   particular exception
-    def caller_exceptions_map
-      @caller_exceptions_map ||= {
-        error: [
-          Karafka::Connection::MessagesConsumer,
-          Karafka::Connection::Listener,
-          Karafka::Params::Params
-        ],
-        fatal: [
-          Karafka::Fetcher
-        ]
-      }
-    end
-
-    # @return [String] label of method that invoked #notice or #notice_error
-    # @example Check label of method that invoked #notice
-    #   caller_label #=> 'fetch'
-    # @example Check label of method that invoked #notice in a block
-    #   caller_label #=> 'block in fetch'
-    # @example Check label of method that invoked #notice_error
-    #   caller_label #=> 'rescue in target'
-    def caller_label
-      # We need to calculate ancestors because if someone inherits
-      # from this  class, caller chains is longer
-      index = self.class.ancestors.index(Karafka::Monitor)
-      # caller_locations has a differs in result whether it is a subclass of
-      # Karafka::Monitor, the basic Karafka::Monitor itself or a super for a subclass.
-      # So to cover all the cases we need to differentiate.
-      # @see https://github.com/karafka/karafka/issues/128
-      # @note It won't work if the monitor caller_label caller class is defined using
-      #   define method
-      super_execution = caller_locations(1, 2)[0].label == caller_locations(1, 2)[1].label
-
-      scope = super_execution ? 1 : nil
-      scope ||= index.positive? ? 0 : 1
-
-      caller_locations(index + 1, 2)[scope].label
-    end
-
-    # @return [Logger] logger instance
-    def logger
-      Karafka.logger
-    end
-  end
-end