karafka 1.1.0

data/lib/karafka/helpers/multi_delegator.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Module containing classes and methods that provide some additional functionalities
+  module Helpers
+    # @note Taken from http://stackoverflow.com/questions/6407141
+    # Multidelegator is used to delegate calls to multiple targets
+    class MultiDelegator
+      # @param targets to which we want to delegate methods
+      #
+      def initialize(*targets)
+        @targets = targets
+      end
+
+      class << self
+        # @param methods names that should be delegated to
+        # @example Delegate write and close to STDOUT and file
+        #   Logger.new MultiDelegator.delegate(:write, :close).to(STDOUT, log_file)
+        def delegate(*methods)
+          methods.each do |m|
+            define_method(m) do |*args|
+              @targets.map { |t| t.send(m, *args) }
+            end
+          end
+
+          self
+        end
+
+        alias to new
+      end
+    end
+  end
+end

data/lib/karafka/loader.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Loader for requiring all the files in a proper order
+  module Loader
+    # Order in which we want to load app files
+    DIRS = %w[
+      config/initializers
+      lib
+      app
+    ].freeze
+
+    # Will load files in a proper order (based on DIRS)
+    # @param [String] root path from which we want to start
+    def self.load(root)
+      DIRS.each do |dir|
+        path = File.join(root, dir)
+        next unless File.exist?(path)
+        load!(path)
+      end
+    end
+
+    # Requires all the ruby files from one path in a proper order
+    # @param path [String] path (dir) from which we want to load ruby files in a proper order
+    def self.load!(path)
+      require_all(path)
+    end
+  end
+end

data/lib/karafka/logger.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Default logger for Event Delegator
+  # @note It uses ::Logger features - providing basic logging
+  class Logger < ::Logger
+    include Singleton
+
+    # Map containing informations about log level for given environment
+    ENV_MAP = {
+      'production' => ::Logger::ERROR,
+      'test' => ::Logger::ERROR,
+      'development' => ::Logger::INFO,
+      'debug' => ::Logger::DEBUG,
+      default: ::Logger::INFO
+    }.freeze
+
+    # Creates a new instance of logger ensuring that it has a place to write to
+    def initialize(*_args)
+      ensure_dir_exists
+      super(target)
+      self.level = ENV_MAP[Karafka.env] || ENV_MAP[:default]
+    end
+
+    private
+
+    # @return [Karafka::Helpers::MultiDelegator] multi delegator instance
+    #   to which we will be writtng logs
+    # We use this approach to log stuff to file and to the STDOUT at the same time
+    def target
+      Karafka::Helpers::MultiDelegator
+        .delegate(:write, :close)
+        .to(STDOUT, file)
+    end
+
+    # Makes sure the log directory exists
+    def ensure_dir_exists
+      dir = File.dirname(log_path)
+      FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
+    end
+
+    # @return [Pathname] Path to a file to which we should log
+    def log_path
+      @log_path ||= Karafka::App.root.join("log/#{Karafka.env}.log")
+    end
+
+    # @return [File] file to which we want to write our logs
+    # @note File is being opened in append mode ('a')
+    def file
+      @file ||= File.open(log_path, 'a')
+    end
+  end
+end

data/lib/karafka/monitor.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Monitor is used to hookup external monitoring services to monitor how Karafka works
+  # It provides a standarized API for checking incoming messages/enqueueing etc
+  # By default it implements logging functionalities but can be replaced with any more
+  #   sophisticated logging/monitoring system like Errbit, Airbrake, NewRelic
+  # @note This class acts as a singleton because we are only permitted to have single monitor
+  #   per running process (just as logger)
+  # Keep in mind, that if you create your own monitor object, you will have to implement also
+  # logging functionality (or just inherit, super and do whatever you want)
+  class Monitor
+    include Singleton
+
+    # This method is executed in many important places in the code (during data flow), like
+    # the moment before #consume_async, etc. For full list just grep for 'monitor.notice'
+    # @param caller_class [Class] class of object that executed this call
+    # @param options [Hash] hash with options that we passed to notice. It differs based
+    #   on of who and when is calling
+    # @note We don't provide a name of method in which this was called, because we can take
+    #   it directly from Ruby (see #caller_label method of this class for more details)
+    # @example Notice about consuming with controller_class
+    #   Karafka.monitor.notice(self.class, controller_class: controller_class)
+    # @example Notice about terminating with a signal
+    #   Karafka.monitor.notice(self.class, signal: signal)
+    def notice(caller_class, options = {})
+      logger.info("#{caller_class}##{caller_label} with #{options}")
+    end
+
+    # This method is executed when we want to notify about an error that happened somewhere
+    # in the system
+    # @param caller_class [Class] class of object that executed this call
+    # @param e [Exception] exception that was raised
+    # @note We don't provide a name of method in which this was called, because we can take
+    #   it directly from Ruby (see #caller_label method of this class for more details)
+    # @example Notify about error
+    #   Karafka.monitor.notice(self.class, e)
+    def notice_error(caller_class, e)
+      caller_exceptions_map.each do |level, types|
+        next unless types.include?(caller_class)
+
+        return logger.public_send(level, e)
+      end
+
+      logger.info(e)
+    end
+
+    private
+
+    # @return [Hash] Hash containing informations on which level of notification should
+    #   we use for exceptions that happen in certain parts of Karafka
+    # @note Keep in mind that any not handled here class should be logged with info
+    # @note Those are not maps of exceptions classes but of classes that were callers of this
+    #   particular exception
+    def caller_exceptions_map
+      @caller_exceptions_map ||= {
+        error: [
+          Karafka::Connection::Consumer,
+          Karafka::Connection::Listener,
+          Karafka::Params::Params
+        ],
+        fatal: [
+          Karafka::Fetcher
+        ]
+      }
+    end
+
+    # @return [String] label of method that invoked #notice or #notice_error
+    # @example Check label of method that invoked #notice
+    #   caller_label #=> 'fetch'
+    # @example Check label of method that invoked #notice in a block
+    #   caller_label #=> 'block in fetch'
+    # @example Check label of method that invoked #notice_error
+    #   caller_label #=> 'rescue in target'
+    def caller_label
+      # We need to calculate ancestors because if someone inherits
+      # from this  class, caller chains is longer
+      index = self.class.ancestors.index(Karafka::Monitor)
+      # caller_locations has a differs in result whether it is a subclass of
+      # Karafka::Monitor, the basic Karafka::Monitor itself or a super for a subclass.
+      # So to cover all the cases we need to differentiate.
+      # @see https://github.com/karafka/karafka/issues/128
+      # @note It won't work if the monitor caller_label caller class is defined using
+      #   define method
+      super_execution = caller_locations(1, 2)[0].label == caller_locations(1, 2)[1].label
+
+      scope = super_execution ? 1 : nil
+      scope ||= index.positive? ? 0 : 1
+
+      caller_locations(index + 1, 2)[scope].label
+    end
+
+    # @return [Logger] logger instance
+    def logger
+      Karafka.logger
+    end
+  end
+end

data/lib/karafka/params/params.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Params namespace encapsulating all the logic that is directly related to params handling
+  module Params
+    # Class-wrapper for hash with indifferent access with additional lazy loading feature
+    # It provides lazy loading not only until the first usage, but also allows us to skip
+    # using parser until we execute our logic. That way we can operate with
+    # heavy-parsing data without slowing down the whole application.
+    class Params < HashWithIndifferentAccess
+      # Kafka::FetchedMessage attributes that we want to use inside of params
+      KAFKA_MESSAGE_ATTRIBUTES = %i[
+        value
+        partition
+        offset
+        key
+        create_time
+      ].freeze
+
+      # Params attributes that should be available via a method call invocation for Kafka
+      # client compatibility.
+      # Kafka passes internally Kafka::FetchedMessage object and the ruby-kafka consumer
+      # uses those fields via method calls, so in order to be able to pass there our params
+      # objects, have to have same api.
+      PARAMS_METHOD_ATTRIBUTES = %i[
+        topic
+        partition
+        offset
+        key
+        create_time
+      ].freeze
+
+      class << self
+        # We allow building instances only via the #build method
+
+        # @param message [Kafka::FetchedMessage, Hash] message that we get out of Kafka
+        #   in case of building params inside main Karafka process in
+        #   Karafka::Connection::Consumer, or a hash when we retrieve data that is already parsed
+        # @param parser [Class] parser class that we will use to unparse data
+        # @return [Karafka::Params::Params] Karafka params object not yet used parser for
+        #   retrieving data that we've got from Kafka
+        # @example Build params instance from a hash
+        #   Karafka::Params::Params.build({ key: 'value' }) #=> params object
+        # @example Build params instance from a Kafka::FetchedMessage object
+        #   Karafka::Params::Params.build(message) #=> params object
+        def build(message, parser)
+          # Hash case happens inside backends that interchange data
+          if message.is_a?(Hash)
+            new(parser: parser).send(:merge!, message)
+          else
+            # This happens inside Kafka::FetchedProcessor
+            new(
+              parser: parser,
+              parsed: false,
+              received_at: Time.now
+            ).tap do |instance|
+              KAFKA_MESSAGE_ATTRIBUTES.each do |attribute|
+                instance[attribute] = message.send(attribute)
+              end
+
+              # When we get raw messages, they might have a topic, that was modified by a
+              # topic mapper. We need to "reverse" this change and map back to the non-modified
+              # format, so our internal flow is not corrupted with the mapping
+              instance[:topic] = Karafka::App.config.topic_mapper.incoming(message.topic)
+            end
+          end
+        end
+
+        # Defines a method call accessor to a particular hash field.
+        # @note Won't work for complex key names that contain spaces, etc
+        # @param key [Symbol] name of a field that we want to retrieve with a method call
+        # @example
+        #   key_attr_reader :example
+        #   params.example #=> 'my example value'
+        def key_attr_reader(key)
+          define_method key do
+            self[key]
+          end
+        end
+      end
+
+      # @return [Karafka::Params::Params] this will trigger parser execution. If we decide to
+      #   retrieve data, parser will be executed to parse data. Output of parsing will be merged
+      #   to the current object. This object will be also marked as already parsed, so we won't
+      #   parse it again.
+      def retrieve!
+        return self if self[:parsed]
+
+        merge!(parse(delete(:value)))
+      end
+
+      PARAMS_METHOD_ATTRIBUTES.each(&method(:key_attr_reader))
+
+      private
+
+      # Overwritten merge! method - it behaves differently for keys that are the same in our hash
+      #  and in a other_hash - it will not replace keys that are the same in our hash
+      #  and in the other one
+      # @param other_hash [Hash, HashWithIndifferentAccess] hash that we want to merge into current
+      # @return [Karafka::Params::Params] our parameters hash with merged values
+      # @example Merge with hash without same keys
+      #   new(a: 1, b: 2).merge!(c: 3) #=> { a: 1, b: 2, c: 3 }
+      # @example Merge with hash with same keys (symbol based)
+      #   new(a: 1).merge!(a: 2) #=> { a: 1 }
+      # @example Merge with hash with same keys (string based)
+      #   new(a: 1).merge!('a' => 2) #=> { a: 1 }
+      # @example Merge with hash with same keys (current string based)
+      #   new('a' => 1).merge!(a: 2) #=> { a: 1 }
+      def merge!(other_hash)
+        super(other_hash) { |_key, base_value, _new_value| base_value }
+      end
+
+      # @param value [String] Raw data that we want to parse using controller's parser
+      # @note If something goes wrong, it will return raw data in a hash with a message key
+      # @return [Hash] parsed data or a hash with message key containing raw data if something
+      #   went wrong during parsing
+      def parse(value)
+        self[:parser].parse(value)
+        # We catch both of them, because for default JSON - we use JSON parser directly
+      rescue ::Karafka::Errors::ParserError => e
+        Karafka.monitor.notice_error(self.class, e)
+        raise e
+      ensure
+        self[:parsed] = true
+      end
+    end
+  end
+end

data/lib/karafka/params/params_batch.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Params
+    # Params batch represents a set of messages received from Kafka.
+    # @note Params internally are lazy loaded before first use. That way we can skip parsing
+    #   process if we have after_fetched that rejects some incoming messages without using params
+    #   It can be also used when handling really heavy data (in terms of parsing).
+    class ParamsBatch
+      include Enumerable
+
+      # Builds up a params batch based on raw kafka messages
+      # @param messages_batch [Array<Kafka::FetchedMessage>] messages batch
+      # @param topic_parser [Class] topic parser for unparsing messages values
+      def initialize(messages_batch, topic_parser)
+        @params_batch = messages_batch.map do |message|
+          Karafka::Params::Params.build(message, topic_parser)
+        end
+      end
+
+      # @yieldparam [Karafka::Params::Params] each parsed and loaded params instance
+      # @note Invocation of this method will cause loading and parsing each param after another.
+      #   If you want to get access without parsing, please access params_batch directly
+      def each
+        @params_batch.each { |param| yield(param.retrieve!) }
+      end
+
+      # @return [Array<Karafka::Params::Params>] returns all the params in a loaded state, so they
+      #   can be used for batch insert, etc. Without invoking all, up until first use, they won't
+      #   be parsed
+      def parsed
+        each(&:itself)
+      end
+
+      # @return [Array<Karafka::Params::Params>] pure array with params (not parsed)
+      def to_a
+        @params_batch
+      end
+    end
+  end
+end

data/lib/karafka/parsers/json.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Module for all supported by default parsers for incoming/outgoing data
+  module Parsers
+    # Default Karafka Json parser for serializing and deserializing data
+    class Json
+      # @param content [String] content based on which we want to get our hash
+      # @return [Hash] hash with parsed JSON data
+      # @example
+      #   Json.parse("{\"a\":1}") #=> { 'a' => 1 }
+      def self.parse(content)
+        ::MultiJson.load(content)
+      rescue ::MultiJson::ParseError => e
+        raise ::Karafka::Errors::ParserError, e
+      end
+
+      # @param content [Object] any object that we want to convert to a json string
+      # @return [String] Valid JSON string containing serialized data
+      # @raise [Karafka::Errors::ParserError] raised when we don't have a way to parse
+      #   given content to a json string format
+      # @note When string is passed to this method, we assume that it is already a json
+      #   string and we don't serialize it again. This allows us to serialize data before
+      #   it is being forwarded to a parser if we want to have a custom (not that simple)
+      #   json serialization
+      #
+      # @example From an ActiveRecord object
+      #   Json.generate(Repository.first) #=> "{\"repository\":{\"id\":\"04b504e0\"}}"
+      # @example From a string (no changes)
+      #   Json.generate("{\"a\":1}") #=> "{\"a\":1}"
+      def self.generate(content)
+        return content if content.is_a?(String)
+        return content.to_json if content.respond_to?(:to_json)
+        raise Karafka::Errors::ParserError, content
+      end
+    end
+  end
+end

data/lib/karafka/patches/dry_configurable.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace for patches of external gems/libraries
+  module Patches
+    # Patch that will allow to use proc based lazy evaluated settings with Dry Configurable
+    # @see https://github.com/dry-rb/dry-configurable/blob/master/lib/dry/configurable.rb
+    module DryConfigurable
+      # We overwrite ::Dry::Configurable::Config to change on proc behaviour
+      # Unfortunately it does not provide an on call proc evaluation, so
+      # this feature had to be added here on demand/
+      # @param args Any arguments that DryConfigurable::Config accepts
+      def initialize(*args)
+        super
+
+        @config.each_key(&method(:rebuild))
+      end
+
+      private
+
+      # Method that rebuilds a given accessor, so when it consists a proc value, it will
+      # evaluate it upon return
+      # @param method_name [Symbol] name of an accessor that we want to rebuild
+      def rebuild(method_name)
+        define_singleton_method method_name do
+          super().is_a?(Proc) ? super().call : super()
+        end
+      end
+    end
+  end
+end