karafka 1.1.2 → 1.2.0.beta1

data/lib/karafka/connection/processor.rb

@@ -1,61 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Connection
-    # Class that consumes messages for which we listen
-    module Processor
-      class << self
-        # Processes messages (does something with them)
-        # It will either schedule or run a proper controller action for messages
-        # @note This should be looped to obtain a constant listening
-        # @note We catch all the errors here, to make sure that none failures
-        #   for a given consumption will affect other consumed messages
-        #   If we wouldn't catch it, it would propagate up until killing the thread
-        # @param group_id [String] group_id of a group from which a given message came
-        # @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages fetched from kafka
-        def process(group_id, kafka_messages)
-          # @note We always get messages by topic and partition so we can take topic from the
-          # first one and it will be valid for all the messages
-          # We map from incoming topic name, as it might be namespaced, etc.
-          # @see topic_mapper internal docs
-          mapped_topic_name = Karafka::App.config.topic_mapper.incoming(kafka_messages[0].topic)
-          topic = Routing::Router.find("#{group_id}_#{mapped_topic_name}")
-          controller = Persistence::Controller.fetch(topic, kafka_messages[0].partition) do
-            topic.controller.new
-          end
-
-          # Depending on a case (persisted or not) we might use new controller instance per each
-          # batch, or use the same instance for all of them (for implementing buffering, etc)
-          send(
-            topic.batch_consuming ? :process_batch : :process_each,
-            controller,
-            kafka_messages
-          )
-        end
-
-        private
-
-        # Processes whole batch in one request (all at once)
-        # @param controller [Karafka::BaseController] base controller descendant
-        # @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages from kafka
-        def process_batch(controller, kafka_messages)
-          controller.params_batch = kafka_messages
-          Karafka.monitor.notice(self, kafka_messages)
-          controller.call
-        end
-
-        # Processes messages one by one (like with std http requests)
-        # @param controller [Karafka::BaseController] base controller descendant
-        # @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages from kafka
-        def process_each(controller, kafka_messages)
-          kafka_messages.each do |kafka_message|
-            # @note This is a simple trick - we just process one after another, but in order
-            # not to handle everywhere both cases (single vs batch), we just "fake" batching with
-            # a single message for each
-            process_batch(controller, [kafka_message])
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/controllers/includer.rb

@@ -1,51 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Additional functionalities for controllers
-  module Controllers
-    # Module used to inject functionalities into a given controller class, based on the controller
-    # topic and its settings
-    # We don't need all the behaviors in all the cases, so it is totally not worth having
-    # everything in all the cases all the time
-    module Includer
-      class << self
-        # @param controller_class [Class] controller class, that will get some functionalities
-        #   based on the topic under which it operates
-        def call(controller_class)
-          topic = controller_class.topic
-
-          bind_backend(controller_class, topic)
-          bind_params(controller_class, topic)
-          bind_responders(controller_class, topic)
-        end
-
-        private
-
-        # Figures out backend for a given controller class, based on the topic backend and
-        #   includes it into the controller class
-        # @param controller_class [Class] controller class
-        # @param topic [Karafka::Routing::Topic] topic of a controller class
-        def bind_backend(controller_class, topic)
-          backend = Kernel.const_get("::Karafka::Backends::#{topic.backend.to_s.capitalize}")
-          controller_class.include backend
-        end
-
-        # Adds a single #params support for non batch processed topics
-        # @param controller_class [Class] controller class
-        # @param topic [Karafka::Routing::Topic] topic of a controller class
-        def bind_params(controller_class, topic)
-          return if topic.batch_consuming
-          controller_class.include SingleParams
-        end
-
-        # Adds responders support for topics and controllers with responders defined for them
-        # @param controller_class [Class] controller class
-        # @param topic [Karafka::Routing::Topic] topic of a controller class
-        def bind_responders(controller_class, topic)
-          return unless topic.responder
-          controller_class.include Responders
-        end
-      end
-    end
-  end
-end

data/lib/karafka/controllers/responders.rb

@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Controllers
-    # Feature that allows us to use responders flow in controller
-    module Responders
-      # Responds with given data using given responder. This allows us to have a similar way of
-      # defining flows like synchronous protocols
-      # @param data Anything we want to pass to responder based on which we want to trigger further
-      #   Kafka responding
-      def respond_with(*data)
-        Karafka.monitor.notice(self.class, data: data)
-        # @note we build a new instance of responder each time, as a long running (persisted)
-        #   controllers can respond multiple times during the lifecycle
-        topic.responder.new(topic.parser).call(*data)
-      end
-    end
-  end
-end

data/lib/karafka/logger.rb

@@ -1,53 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Default logger for Event Delegator
-  # @note It uses ::Logger features - providing basic logging
-  class Logger < ::Logger
-    include Singleton
-
-    # Map containing informations about log level for given environment
-    ENV_MAP = {
-      'production' => ::Logger::ERROR,
-      'test' => ::Logger::ERROR,
-      'development' => ::Logger::INFO,
-      'debug' => ::Logger::DEBUG,
-      default: ::Logger::INFO
-    }.freeze
-
-    # Creates a new instance of logger ensuring that it has a place to write to
-    def initialize(*_args)
-      ensure_dir_exists
-      super(target)
-      self.level = ENV_MAP[Karafka.env] || ENV_MAP[:default]
-    end
-
-    private
-
-    # @return [Karafka::Helpers::MultiDelegator] multi delegator instance
-    #   to which we will be writtng logs
-    # We use this approach to log stuff to file and to the STDOUT at the same time
-    def target
-      Karafka::Helpers::MultiDelegator
-        .delegate(:write, :close)
-        .to(STDOUT, file)
-    end
-
-    # Makes sure the log directory exists
-    def ensure_dir_exists
-      dir = File.dirname(log_path)
-      FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
-    end
-
-    # @return [Pathname] Path to a file to which we should log
-    def log_path
-      @log_path ||= Karafka::App.root.join("log/#{Karafka.env}.log")
-    end
-
-    # @return [File] file to which we want to write our logs
-    # @note File is being opened in append mode ('a')
-    def file
-      @file ||= File.open(log_path, 'a')
-    end
-  end
-end

data/lib/karafka/monitor.rb

@@ -1,98 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Monitor is used to hookup external monitoring services to monitor how Karafka works
-  # It provides a standarized API for checking incoming messages/enqueueing etc
-  # By default it implements logging functionalities but can be replaced with any more
-  #   sophisticated logging/monitoring system like Errbit, Airbrake, NewRelic
-  # @note This class acts as a singleton because we are only permitted to have single monitor
-  #   per running process (just as logger)
-  # Keep in mind, that if you create your own monitor object, you will have to implement also
-  # logging functionality (or just inherit, super and do whatever you want)
-  class Monitor
-    include Singleton
-
-    # This method is executed in many important places in the code (during data flow), like
-    # the moment before #consume_async, etc. For full list just grep for 'monitor.notice'
-    # @param caller_class [Class] class of object that executed this call
-    # @param options [Hash] hash with options that we passed to notice. It differs based
-    #   on of who and when is calling
-    # @note We don't provide a name of method in which this was called, because we can take
-    #   it directly from Ruby (see #caller_label method of this class for more details)
-    # @example Notice about consuming with controller_class
-    #   Karafka.monitor.notice(self.class, controller_class: controller_class)
-    # @example Notice about terminating with a signal
-    #   Karafka.monitor.notice(self.class, signal: signal)
-    def notice(caller_class, options = {})
-      logger.info("#{caller_class}##{caller_label} with #{options}")
-    end
-
-    # This method is executed when we want to notify about an error that happened somewhere
-    # in the system
-    # @param caller_class [Class] class of object that executed this call
-    # @param e [Exception] exception that was raised
-    # @note We don't provide a name of method in which this was called, because we can take
-    #   it directly from Ruby (see #caller_label method of this class for more details)
-    # @example Notify about error
-    #   Karafka.monitor.notice(self.class, e)
-    def notice_error(caller_class, e)
-      caller_exceptions_map.each do |level, types|
-        next unless types.include?(caller_class)
-
-        return logger.public_send(level, e)
-      end
-
-      logger.info(e)
-    end
-
-    private
-
-    # @return [Hash] Hash containing informations on which level of notification should
-    #   we use for exceptions that happen in certain parts of Karafka
-    # @note Keep in mind that any not handled here class should be logged with info
-    # @note Those are not maps of exceptions classes but of classes that were callers of this
-    #   particular exception
-    def caller_exceptions_map
-      @caller_exceptions_map ||= {
-        error: [
-          Karafka::Connection::Consumer,
-          Karafka::Connection::Listener,
-          Karafka::Params::Params
-        ],
-        fatal: [
-          Karafka::Fetcher
-        ]
-      }
-    end
-
-    # @return [String] label of method that invoked #notice or #notice_error
-    # @example Check label of method that invoked #notice
-    #   caller_label #=> 'fetch'
-    # @example Check label of method that invoked #notice in a block
-    #   caller_label #=> 'block in fetch'
-    # @example Check label of method that invoked #notice_error
-    #   caller_label #=> 'rescue in target'
-    def caller_label
-      # We need to calculate ancestors because if someone inherits
-      # from this  class, caller chains is longer
-      index = self.class.ancestors.index(Karafka::Monitor)
-      # caller_locations has a differs in result whether it is a subclass of
-      # Karafka::Monitor, the basic Karafka::Monitor itself or a super for a subclass.
-      # So to cover all the cases we need to differentiate.
-      # @see https://github.com/karafka/karafka/issues/128
-      # @note It won't work if the monitor caller_label caller class is defined using
-      #   define method
-      super_execution = caller_locations(1, 2)[0].label == caller_locations(1, 2)[1].label
-
-      scope = super_execution ? 1 : nil
-      scope ||= index.positive? ? 0 : 1
-
-      caller_locations(index + 1, 2)[scope].label
-    end
-
-    # @return [Logger] logger instance
-    def logger
-      Karafka.logger
-    end
-  end
-end

data/lib/karafka/persistence/controller.rb

@@ -1,38 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Module used to provide a persistent cache layer for Karafka components that need to be
-  # shared inside of a same thread
-  module Persistence
-    # Module used to provide a persistent cache across batch requests for a given
-    # topic and partition to store some additional details when the persistent mode
-    # for a given topic is turned on
-    class Controller
-      # Thread.current scope under which we store controllers data
-      PERSISTENCE_SCOPE = :controllers
-
-      class << self
-        # @return [Hash] current thread persistence scope hash with all the controllers
-        def all
-          Thread.current[PERSISTENCE_SCOPE] ||= {}
-        end
-
-        # Used to build (if block given) and/or fetch a current controller instance that will be
-        #   used to process messages from a given topic and partition
-        # @return [Karafka::BaseController] base controller descendant
-        # @param topic [Karafka::Routing::Topic] topic instance for which we might cache
-        # @param partition [Integer] number of partition for which we want to cache
-        def fetch(topic, partition)
-          all[topic.id] ||= {}
-
-          # We always store a current instance
-          if topic.persistent
-            all[topic.id][partition] ||= yield
-          else
-            all[topic.id][partition] = yield
-          end
-        end
-      end
-    end
-  end
-end