karafka 0.5.0

data/lib/karafka/logger.rb

@@ -0,0 +1,52 @@
+module Karafka
+  # Default logger for Event Delegator
+  # @note It uses ::Logger features - providing basic logging
+  class Logger < ::Logger
+    # Map containing informations about log level for given environment
+    ENV_MAP = {
+      'production' => ::Logger::ERROR,
+      'test' => ::Logger::ERROR,
+      'development' => ::Logger::INFO,
+      'debug' => ::Logger::DEBUG,
+      default: ::Logger::INFO
+    }.freeze
+
+    class << self
+      # Returns a logger instance with appropriate settings, log level and environment
+      def instance
+        ensure_dir_exists
+        instance = new(target)
+        instance.level = ENV_MAP[Karafka.env] || ENV_MAP[:default]
+        instance
+      end
+
+      private
+
+      # @return [Karafka::Helpers::MultiDelegator] multi delegator instance
+      #   to which we will be writtng logs
+      # We use this approach to log stuff to file and to the STDOUT at the same time
+      def target
+        Karafka::Helpers::MultiDelegator
+          .delegate(:write, :close)
+          .to(STDOUT, file)
+      end
+
+      # Makes sure the log directory exists
+      def ensure_dir_exists
+        dir = File.dirname(log_path)
+        FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
+      end
+
+      # @return [Pathname] Path to a file to which we should log
+      def log_path
+        Karafka::App.root.join("log/#{Karafka.env}.log")
+      end
+
+      # @return [File] file to which we want to write our logs
+      # @note File is being opened in append mode ('a')
+      def file
+        File.open(log_path, 'a')
+      end
+    end
+  end
+end

data/lib/karafka/monitor.rb

@@ -0,0 +1,82 @@
+module Karafka
+  # Monitor is used to hookup external monitoring services to monitor how Karafka works
+  # It provides a standarized API for checking incoming messages/enqueueing etc
+  # By default it implements logging functionalities but can be replaced with any more
+  #   sophisticated logging/monitoring system like Errbit, Airbrake, NewRelic
+  # @note This class acts as a singleton because we are only permitted to have single monitor
+  #   per running process (just as logger)
+  # Keep in mind, that if you create your own monitor object, you will have to implement also
+  # logging functionality (or just inherit, super and do whatever you want)
+  class Monitor
+    include Singleton
+
+    # This method is executed in many important places in the code (during data flow), like
+    # the moment before #perform_async, etc. For full list just grep for 'monitor.notice'
+    # @param caller_class [Class] class of object that executed this call
+    # @param options [Hash] hash with options that we passed to notice. It differs based
+    #   on of who and when is calling
+    # @note We don't provide a name of method in which this was called, because we can take
+    #   it directly from Ruby (see #caller_label method of this class for more details)
+    # @example Notice about consuming with controller_class
+    #   Karafka.monitor.notice(self.class, controller_class: controller_class)
+    # @example Notice about terminating with a signal
+    #   Karafka.monitor.notice(self.class, signal: signal)
+    def notice(caller_class, options = {})
+      logger.info("#{caller_class}##{caller_label} with #{options}")
+    end
+
+    # This method is executed when we want to notify about an error that happened somewhere
+    # in the system
+    # @param caller_class [Class] class of object that executed this call
+    # @param e [Exception] exception that was raised
+    # @note We don't provide a name of method in which this was called, because we can take
+    #   it directly from Ruby (see #caller_label method of this class for more details)
+    # @example Notify about error
+    #   Karafka.monitor.notice(self.class, e)
+    def notice_error(caller_class, e)
+      caller_exceptions_map.each do |level, types|
+        next unless types.include?(caller_class)
+
+        return logger.public_send(level, e)
+      end
+
+      logger.info(e)
+    end
+
+    private
+
+    # @return [Hash] Hash containing informations on which level of notification should
+    #   we use for exceptions that happen in certain parts of Karafka
+    # @note Keep in mind that any not handled here class should be logged with info
+    # @note Those are not maps of exceptions classes but of classes that were callers of this
+    #   particular exception
+    def caller_exceptions_map
+      @caller_exceptions_map ||= {
+        error: [
+          Karafka::Connection::Consumer,
+          Karafka::Connection::Listener,
+          Karafka::Params::Params
+        ],
+        fatal: [
+          Karafka::Fetcher
+        ]
+      }
+    end
+
+    # @return [String] label of method that invoked #notice or #notice_error
+    # @example Check label of method that invoked #notice
+    #   caller_label #=> 'fetch'
+    # @example Check label of method that invoked #notice in a block
+    #   caller_label #=> 'block in fetch'
+    # @example Check label of method that invoked #notice_error
+    #   caller_label #=> 'rescue in target'
+    def caller_label
+      caller_locations(1, 2)[1].label
+    end
+
+    # @return [Logger] logger instance
+    def logger
+      Karafka.logger
+    end
+  end
+end

data/lib/karafka/params/interchanger.rb

@@ -0,0 +1,33 @@
+module Karafka
+  module Params
+    # Interchangers allow us to format/encode/pack data that is being send to perform_async
+    # This is meant to target mostly issues with data encoding like this one:
+    # https://github.com/mperham/sidekiq/issues/197
+    # Each custom interchanger should implement following methods:
+    #   - load - it is meant to encode params before they get stored inside Redis
+    #   - parse - decoded params back to a hash format that we can use
+    class Interchanger
+      class << self
+        # @param params [Karafka::Params::Params] Karafka params object
+        # @note Params might not be parsed because of lazy loading feature. If you implement your
+        #   own interchanger logic, this method needs to return data that can be converted to
+        #   json with default Sidekiqs logic
+        # @return [Karafka::Params::Params] same as input. We assume that our incoming data is
+        #   jsonable-safe and we can rely on a direct Sidekiq encoding logic
+        def load(params)
+          params
+        end
+
+        # @param params [Hash] Sidekiqs params that are now a Hash (after they were JSON#parse)
+        # @note Hash is what we need to build Karafka::Params::Params, so we do nothing
+        #   with it. If you implement your own interchanger logic, this method needs to return
+        #   a hash with appropriate data that will be used to build Karafka::Params::Params
+        # @return [Hash] We return exactly what we received. We rely on sidekiqs default
+        #   interchanging format
+        def parse(params)
+          params
+        end
+      end
+    end
+  end
+end

data/lib/karafka/params/params.rb

@@ -0,0 +1,102 @@
+module Karafka
+  # Params namespace encapsulating all the logic that is directly related to params handling
+  module Params
+    # Class-wrapper for hash with indifferent access with additional lazy loading feature
+    # It provides lazy loading not only until the first usage, but also allows us to skip
+    # using parser until we execute our logic inside worker. That way we can operate with
+    # heavy-parsing data without slowing down the whole application. If we won't use
+    # params in before_enqueue (or if we don't us before_enqueue at all), it will make
+    # Karafka faster, because it will pass data as it is directly to Sidekiq
+    class Params < HashWithIndifferentAccess
+      class << self
+        # We allow building instances only via the #build method
+        private_class_method :new
+
+        # @param message [Karafka::Connection::Message, Hash] message that we get out of Kafka
+        #   in case of building params inside main Karafka process in
+        #   Karafka::Connection::Consumer, or a hash when we retrieve data from Sidekiq
+        # @param controller [Karafka::BaseController] Karafka's base controllers descendant
+        #   instance that wants to use params
+        # @return [Karafka::Params::Params] Karafka params object not yet used parser for
+        #   retrieving data that we've got from Kafka
+        # @example Build params instance from a hash
+        #   Karafka::Params::Params.build({ key: 'value' }, DataController.new) #=> params object
+        # @example Build params instance from a Karafka::Connection::Message object
+        #   Karafka::Params::Params.build(message, IncomingController.new) #=> params object
+        def build(message, controller)
+          # Hash case happens inside workers
+          if message.is_a?(Hash)
+            defaults(controller).merge!(message)
+          else
+            # This happens inside Karafka::Connection::Consumer
+            defaults(controller).merge!(
+              parsed: false,
+              received_at: Time.now,
+              content: message.content
+            )
+          end
+        end
+
+        private
+
+        # @param controller [Karafka::BaseController] Karafka's base controllers descendant
+        #   instance that wants to use params
+        # @return [Karafka::Params::Params] freshly initialized only with default values object
+        #   that can be populated with incoming data
+        def defaults(controller)
+          # We initialize some default values that will be used both in Karafka main process and
+          # inside workers
+          new(
+            controller: controller.class,
+            worker: controller.worker,
+            parser: controller.parser,
+            topic: controller.topic
+          )
+        end
+      end
+
+      # @return [Karafka::Params::Params] this will trigger parser execution. If we decide to
+      #   retrieve data, parser will be executed to parse data. Output of parsing will be merged
+      #   to the current object. This object will be also marked as already parsed, so we won't
+      #   parse it again.
+      def retrieve
+        return self if self[:parsed]
+
+        merge!(parse(delete(:content)))
+      end
+
+      # Overwritten merge! method - it behaves differently for keys that are the same in our hash
+      #  and in a other_hash - it will not replace keys that are the same in our hash
+      #  and in the other one
+      # @param other_hash [Hash, HashWithIndifferentAccess] hash that we want to merge into current
+      # @return [Karafka::Params::Params] our parameters hash with merged values
+      # @example Merge with hash without same keys
+      #   new(a: 1, b: 2).merge!(c: 3) #=> { a: 1, b: 2, c: 3 }
+      # @example Merge with hash with same keys (symbol based)
+      #   new(a: 1).merge!(a: 2) #=> { a: 1 }
+      # @example Merge with hash with same keys (string based)
+      #   new(a: 1).merge!('a' => 2) #=> { a: 1 }
+      # @example Merge with hash with same keys (current string based)
+      #   new('a' => 1).merge!(a: 2) #=> { a: 1 }
+      def merge!(other_hash)
+        super(other_hash) { |_key, base_value, _new_value| base_value }
+      end
+
+      private
+
+      # @param content [String] Raw data that we want to parse using controller's parser
+      # @note If something goes wrong, it will return raw data in a hash with a message key
+      # @return [Hash] parsed data or a hash with message key containing raw data if something
+      #   went wrong during parsing
+      def parse(content)
+        self[:parser].parse(content)
+        # We catch both of them, because for default JSON - we use JSON parser directly
+      rescue ::Karafka::Errors::ParserError, JSON::ParserError => e
+        Karafka.monitor.notice_error(self.class, e)
+        return { message: content }
+      ensure
+        self[:parsed] = true
+      end
+    end
+  end
+end

data/lib/karafka/patches/dry/configurable/config.rb

@@ -0,0 +1,37 @@
+# Patch that will allow to use proc based lazy evaluated settings with Dry Configurable
+# @see https://github.com/dry-rb/dry-configurable/blob/master/lib/dry/configurable.rb
+module Dry
+  # Configurable module for Dry-Configurable
+  module Configurable
+    # Config node instance struct
+    class Config
+      # @param args [Array] All arguments that a Struct accepts
+      def initialize(*args)
+        super
+        setup_dynamics
+      end
+
+      private
+
+      # Method that sets up all the proc based lazy evaluated dynamic config values
+      def setup_dynamics
+        each_pair do |key, value|
+          next unless value.is_a?(Proc)
+
+          rebuild(key)
+        end
+      end
+
+      # Method that rebuilds a given accessor, so when it consists a proc value, it will
+      # evaluate it upon return
+      # @param method_name [Symbol] name of an accessor that we want to rebuild
+      def rebuild(method_name)
+        metaclass = class << self; self; end
+
+        metaclass.send(:define_method, method_name) do
+          super().is_a?(Proc) ? super().call : super()
+        end
+      end
+    end
+  end
+end

data/lib/karafka/process.rb

@@ -0,0 +1,61 @@
+module Karafka
+  # Class used to catch signals from ruby Signal class in order to manage Karafka shutdown
+  # @note There might be only one process - this class is a singleton
+  class Process
+    include Singleton
+
+    # Signal types that we handle
+    HANDLED_SIGNALS = %i(
+      SIGINT SIGQUIT
+    ).freeze
+
+    HANDLED_SIGNALS.each do |signal|
+      # Assigns a callback that will happen when certain signal will be send
+      # to Karafka server instance
+      # @note It does not define the callback itself -it needs to be passed in a block
+      # @example Define an action that should be taken on_sigint
+      #   process.on_sigint do
+      #     Karafka.logger.info('Log something here')
+      #     exit
+      #   end
+      define_method :"on_#{signal.to_s.downcase}" do |&block|
+        @callbacks[signal] << block
+      end
+    end
+
+    # Creates an instance of process and creates empty hash for callbacks
+    def initialize
+      @callbacks = {}
+      HANDLED_SIGNALS.each { |signal| @callbacks[signal] = [] }
+    end
+
+    # Method catches all HANDLED_SIGNALS and performs appropriate callbacks (if defined)
+    # @note If there are no callbacks, this method will just ignore a given signal that was sent
+    # @yield [Block] block of code that we want to execute and supervise
+    def supervise
+      HANDLED_SIGNALS.each { |signal| trap_signal(signal) }
+      yield
+    end
+
+    private
+
+    # Traps a single signal and performs callbacks (if any) or just ignores this signal
+    # @param [Symbol] signal type that we want to catch
+    def trap_signal(signal)
+      trap(signal) do
+        notice_signal(signal)
+        (@callbacks[signal] || []).each(&:call)
+      end
+    end
+
+    # Informs monitoring about trapped signal
+    # @param [Symbol] signal type that we received
+    # @note We cannot perform logging from trap context, that's why
+    #   we have to spin up a new thread to do this
+    def notice_signal(signal)
+      Thread.new do
+        Karafka.monitor.notice(self.class, signal: signal)
+      end
+    end
+  end
+end

data/lib/karafka/responders/builder.rb

@@ -0,0 +1,33 @@
+module Karafka
+  # Responders namespace encapsulates all the internal responder implementation parts
+  module Responders
+    # Responders builder is used to find (based on the controller class name) a responder that
+    # match the controller. This is used when user does not provide a responder inside routing
+    # but he still names responder with the same convention (and namespaces) as controller
+    # @example Matching responder exists
+    #   Karafka::Responder::Builder(NewEventsController).build #=> NewEventsResponder
+    # @example Matching responder does not exist
+    #   Karafka::Responder::Builder(NewBuildsController).build #=> nil
+    class Builder
+      # @param controller_class [Karafka::BaseController, nil] descendant of
+      #   Karafka::BaseController
+      # @example Tries to find a responder that matches a given controller. If nothing found,
+      #   will return nil (nil is accepted, because it means that a given controller don't
+      #   pipe stuff further on)
+      def initialize(controller_class)
+        @controller_class = controller_class
+      end
+
+      # Tries to figure out a responder based on a controller class name
+      # @return [Class] Responder class (not an instance)
+      # @return [nil] or nil if there's no matching responding class
+      def build
+        Helpers::ClassMatcher.new(
+          @controller_class,
+          from: 'Controller',
+          to: 'Responder'
+        ).match
+      end
+    end
+  end
+end

data/lib/karafka/responders/topic.rb

@@ -0,0 +1,43 @@
+module Karafka
+  module Responders
+    # Topic describes a single topic on which we want to respond with responding requirements
+    # @example Define topic (required by default)
+    #   Karafka::Responders::Topic.new(:topic_name, {}) #=> #<Karafka::Responders::Topic...
+    # @example Define optional topic
+    #   Karafka::Responders::Topic.new(:topic_name, optional: true)
+    # @example Define topic that on which we want to respond multiple times
+    #   Karafka::Responders::Topic.new(:topic_name, multiple_usage: true)
+    class Topic
+      # Name of the topic on which we want to respond
+      attr_reader :name
+
+      # @param name [Symbol, String] name of a topic on which we want to respond
+      # @param options [Hash] non-default options for this topic
+      # @return [Karafka::Responders::Topic] topic description object
+      def initialize(name, options)
+        @name = name.to_s
+        @options = options
+        validate!
+      end
+
+      # @return [Boolean] is this a required topic (if not, it is optional)
+      def required?
+        return false if @options[:optional]
+        @options[:required] || true
+      end
+
+      # @return [Boolean] do we expect to use it multiple times in a single respond flow
+      def multiple_usage?
+        @options[:multiple_usage] || false
+      end
+
+      private
+
+      # Checks topic name with the same regexp as routing
+      # @raise [Karafka::Errors::InvalidTopicName] raised when topic name is invalid
+      def validate!
+        raise Errors::InvalidTopicName, name if Routing::Route::NAME_FORMAT !~ name
+      end
+    end
+  end
+end