karafka 1.2.0

data/lib/karafka/helpers/class_matcher.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Helpers
+    # Class used to autodetect corresponding classes that are internally inside Karafka framework
+    # It is used among others to match:
+    #   consumer => responder
+    class ClassMatcher
+      # Regexp used to remove any non classy like characters that might be in the consumer
+      # class name (if defined dynamically, etc)
+      CONSTANT_REGEXP = %r{[?!=+\-\*/\^\|&\[\]<>%~\#\:\s\(\)]}
+
+      # @param klass [Class] class to which we want to find a corresponding class
+      # @param from [String] what type of object is it (based on postfix name part)
+      # @param to [String] what are we looking for (based on a postfix name part)
+      # @example Consumer that has a corresponding responder
+      #   matcher = Karafka::Helpers::ClassMatcher.new(SuperConsumer, 'Consumer', 'Responder')
+      #   matcher.match #=> SuperResponder
+      # @example Consumer without a corresponding responder
+      #   matcher = Karafka::Helpers::ClassMatcher.new(Super2Consumer, 'Consumer', 'Responder')
+      #   matcher.match #=> nil
+      def initialize(klass, from:, to:)
+        @klass = klass
+        @from = from
+        @to = to
+      end
+
+      # @return [Class] matched class
+      # @return [nil] nil if we couldn't find matching class
+      def match
+        return nil if name.empty?
+        return nil unless scope.const_defined?(name)
+        matching = scope.const_get(name)
+        same_scope?(matching) ? matching : nil
+      end
+
+      # @return [String] name of a new class that we're looking for
+      # @note This method returns name of a class without a namespace
+      # @example From SuperConsumer matching responder
+      #   matcher.name #=> 'SuperResponder'
+      # @example From Namespaced::Super2Consumer matching responder
+      #   matcher.name #=> Super2Responder
+      def name
+        inflected = @klass.to_s.split('::').last.to_s
+        inflected.gsub!(@from, @to)
+        inflected.gsub!(CONSTANT_REGEXP, '')
+        inflected
+      end
+
+      # @return [Class, Module] class or module in which we're looking for a matching
+      def scope
+        scope_of(@klass)
+      end
+
+      private
+
+      # @param klass [Class] class for which we want to extract it's enclosing class/module
+      # @return [Class, Module] enclosing class/module
+      # @return [::Object] object if it was a root class
+      #
+      # @example Non-namespaced class
+      #   scope_of(SuperClass) #=> Object
+      # @example Namespaced class
+      #   scope_of(Abc::SuperClass) #=> Abc
+      def scope_of(klass)
+        enclosing = klass.to_s.split('::')[0...-1]
+        return ::Object if enclosing.empty?
+        ::Object.const_get(enclosing.join('::'))
+      end
+
+      # @param matching [Class] class of which scope we want to check
+      # @return [Boolean] true if the scope of class is the same as scope of matching
+      def same_scope?(matching)
+        scope == scope_of(matching)
+      end
+    end
+  end
+end

data/lib/karafka/helpers/config_retriever.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Helpers
+    # A helper method that allows us to build methods that try to get a given
+    # attribute from its instance value and if it fails, will fallback to
+    # the default config or config.kafka value for a given attribute.
+    # It is used to simplify the checkings.
+    # @note Worth noticing, that the value might be equal to false, so even
+    #   then we need to return it. That's why we check for nil?
+    # @example Define config retried attribute for start_from_beginning
+    # class Test
+    #   extend Karafka::Helpers::ConfigRetriever
+    #   config_retriever_for :start_from_beginning
+    # end
+    #
+    # Test.new.start_from_beginning #=> false
+    # test_instance = Test.new
+    # test_instance.start_from_beginning = true
+    # test_instance.start_from_beginning #=> true
+    module ConfigRetriever
+      # Builds proper methods for setting and retrieving (with fallback) given attribute value
+      # @param attribute [Symbol] attribute name based on which we will build
+      #   accessor with fallback
+      def config_retriever_for(attribute)
+        attr_writer attribute unless method_defined? :"#{attribute}="
+
+        # Don't redefine if we already have accessor for a given element
+        return if method_defined? attribute
+
+        define_method attribute do
+          current_value = instance_variable_get(:"@#{attribute}")
+          return current_value unless current_value.nil?
+
+          value = if Karafka::App.config.respond_to?(attribute)
+                    Karafka::App.config.send(attribute)
+                  else
+                    Karafka::App.config.kafka.send(attribute)
+                  end
+
+          instance_variable_set(:"@#{attribute}", value)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/helpers/multi_delegator.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Module containing classes and methods that provide some additional functionalities
+  module Helpers
+    # @note Taken from http://stackoverflow.com/questions/6407141
+    # Multidelegator is used to delegate calls to multiple targets
+    class MultiDelegator
+      # @param targets to which we want to delegate methods
+      #
+      def initialize(*targets)
+        @targets = targets
+      end
+
+      class << self
+        # @param methods names that should be delegated to
+        # @example Delegate write and close to STDOUT and file
+        #   Logger.new MultiDelegator.delegate(:write, :close).to(STDOUT, log_file)
+        def delegate(*methods)
+          methods.each do |m|
+            define_method(m) do |*args|
+              @targets.map { |t| t.send(m, *args) }
+            end
+          end
+
+          self
+        end
+
+        alias to new
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/listener.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    # Default listener that hooks up to our instrumentation and uses its events for logging
+    # It can be removed/replaced or anything without any harm to the Karafka app flow
+    module Listener
+      # Log levels that we use in this particular listener
+      USED_LOG_LEVELS = %i[
+        debug
+        info
+        error
+        fatal
+      ].freeze
+
+      # Injects WaterDrop listener logger actions
+      extend WaterDrop::Instrumentation::Listener
+
+      class << self
+        # Logs details about incoming messages and with which consumer we will consume them
+        # @param event [Dry::Events::Event] event details including payload
+        def on_connection_delegator_call(event)
+          consumer = event[:consumer]
+          topic = consumer.topic.name
+          kafka_messages = event[:kafka_messages]
+          info "#{kafka_messages.count} messages on #{topic} topic delegated to #{consumer.class}"
+        end
+
+        # Logs details about each received message value parsing
+        # @param event [Dry::Events::Event] event details including payload
+        def on_params_params_parse(event)
+          # Keep in mind, that a caller here is a param object not a controller,
+          # so it returns a topic as a string, not a routing topic
+          debug "Params parsing for #{event[:caller].topic} topic successful in #{event[:time]} ms"
+        end
+
+        # Logs unsuccessful parsing attempts of incoming data
+        # @param event [Dry::Events::Event] event details including payload
+        def on_params_params_parse_error(event)
+          error "Params parsing error for #{event[:caller].topic} topic: #{event[:error]}"
+        end
+
+        # Logs errors that occured in a listener fetch loop
+        # @param event [Dry::Events::Event] event details including payload
+        # @note It's an error as we can recover from it not a fatal
+        def on_connection_listener_fetch_loop_error(event)
+          error "Listener fetch loop error: #{event[:error]}"
+        end
+
+        # Logs errors that are related to the connection itself
+        # @note Karafka will attempt to reconnect, so an error not a fatal
+        # @param event [Dry::Events::Event] event details including payload
+        def on_connection_client_fetch_loop_error(event)
+          error "Client fetch loop error: #{event[:error]}"
+        end
+
+        # Logs info about crashed fetcher
+        # @note If this happens, Karafka will shutdown as it means a critical error
+        #   in one of the threads
+        # @param event [Dry::Events::Event] event details including payload
+        def on_fetcher_call_error(event)
+          fatal "Fetcher crash due to an error: #{event[:error]}"
+        end
+
+        # Logs info about processing of a certain dataset with an inline backend
+        # @param event [Dry::Events::Event] event details including payload
+        def on_backends_inline_process(event)
+          count = event[:caller].send(:params_batch).to_a.size
+          topic = event[:caller].topic.name
+          time = event[:time]
+          info "Inline processing of topic #{topic} with #{count} messages took #{time} ms"
+        end
+
+        # Logs info about system signals that Karafka received
+        # @param event [Dry::Events::Event] event details including payload
+        def on_process_notice_signal(event)
+          info "Received #{event[:signal]} system signal"
+        end
+
+        # Logs info about responder usage withing a controller flow
+        # @param event [Dry::Events::Event] event details including payload
+        def on_consumers_responders_respond_with(event)
+          calling = event[:caller].class
+          responder = calling.topic.responder
+          data = event[:data]
+          info "Responded from #{calling} using #{responder} with following data #{data}"
+        end
+
+        # Logs info that we're going to stop the Karafka server
+        # @param _event [Dry::Events::Event] event details including payload
+        def on_server_stop(_event)
+          # We use a separate thread as logging can't be called from trap context
+          Thread.new { info "Stopping Karafka server #{::Process.pid}" }
+        end
+
+        # Logs an error that Karafka was unable to stop the server gracefully and it had to do a
+        #   forced exit
+        # @param _event [Dry::Events::Event] event details including payload
+        def on_server_stop_error(_event)
+          # We use a separate thread as logging can't be called from trap context
+          Thread.new { error "Forceful Karafka server #{::Process.pid} stop" }
+        end
+
+        USED_LOG_LEVELS.each do |log_level|
+          define_method log_level do |*args|
+            Karafka.logger.send(log_level, *args)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/logger.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    # Default logger for Event Delegator
+    # @note It uses ::Logger features - providing basic logging
+    class Logger < ::Logger
+      include Singleton
+
+      # Map containing information about log level for given environment
+      ENV_MAP = {
+        'production' => ::Logger::ERROR,
+        'test' => ::Logger::ERROR,
+        'development' => ::Logger::INFO,
+        'debug' => ::Logger::DEBUG,
+        'default' => ::Logger::INFO
+      }.freeze
+
+      # Creates a new instance of logger ensuring that it has a place to write to
+      def initialize(*_args)
+        ensure_dir_exists
+        super(target)
+        self.level = ENV_MAP[Karafka.env] || ENV_MAP['default']
+      end
+
+      private
+
+      # @return [Karafka::Helpers::MultiDelegator] multi delegator instance
+      #   to which we will be writtng logs
+      # We use this approach to log stuff to file and to the STDOUT at the same time
+      def target
+        Karafka::Helpers::MultiDelegator
+          .delegate(:write, :close)
+          .to(STDOUT, file)
+      end
+
+      # Makes sure the log directory exists
+      def ensure_dir_exists
+        dir = File.dirname(log_path)
+        FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
+      end
+
+      # @return [Pathname] Path to a file to which we should log
+      def log_path
+        @log_path ||= Karafka::App.root.join("log/#{Karafka.env}.log")
+      end
+
+      # @return [File] file to which we want to write our logs
+      # @note File is being opened in append mode ('a')
+      def file
+        @file ||= File.open(log_path, 'a')
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/monitor.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace for all the things related with Karafka instrumentation process
+  module Instrumentation
+    # Monitor is used to hookup external monitoring services to monitor how Karafka works
+    # It provides a standardized API for checking incoming messages/enqueueing etc
+    # Since it is a pub-sub based on dry-monitor, you can use as many subscribers/loggers at the
+    # same time, which means that you might have for example file logging and newrelic at the same
+    # time
+    # @note This class acts as a singleton because we are only permitted to have single monitor
+    #   per running process (just as logger)
+    class Monitor < Dry::Monitor::Notifications
+      include Singleton
+
+      # List of events that we support in the system and to which a monitor client can hook up
+      # @note The non-error once support timestamp benchmarking
+      # @note Depending on Karafka extensions and additional engines, this might not be the
+      #   complete list of all the events. Please use the #available_events on fully loaded
+      #   Karafka system to determine all of the events you can use.
+      #   Last 4 events are from WaterDrop but for convenience we use the same monitor for the
+      #   whole karafka ecosystem
+      BASE_EVENTS = %w[
+        params.params.parse
+        params.params.parse.error
+        connection.listener.fetch_loop.error
+        connection.client.fetch_loop.error
+        connection.delegator.call
+        fetcher.call.error
+        backends.inline.process
+        process.notice_signal
+        consumers.responders.respond_with
+        async_producer.call.error
+        async_producer.call.retry
+        sync_producer.call.error
+        sync_producer.call.retry
+        server.stop
+        server.stop.error
+      ].freeze
+
+      private_constant :BASE_EVENTS
+
+      # @return [Karafka::Instrumentation::Monitor] monitor instance for system instrumentation
+      def initialize
+        super(:karafka)
+        BASE_EVENTS.each(&method(:register_event))
+      end
+
+      # Allows us to subscribe to events with a code that will be yielded upon events
+      # @param event_name_or_listener [String, Object] name of the event we want to subscribe to
+      #   or a listener if we decide to go with object listener
+      def subscribe(event_name_or_listener)
+        return super unless event_name_or_listener.is_a?(String)
+        return super if available_events.include?(event_name_or_listener)
+        raise Errors::UnregisteredMonitorEvent, event_name_or_listener
+      end
+
+      # @return [Array<String>] names of available events to which we can subscribe
+      def available_events
+        __bus__.events.keys
+      end
+    end
+  end
+end

data/lib/karafka/loader.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Loader for requiring all the files in a proper order
+  module Loader
+    # Order in which we want to load app files
+    DIRS = %w[
+      lib
+      app
+    ].freeze
+
+    # Will load files in a proper order (based on DIRS)
+    # @param [String] root path from which we want to start
+    def self.load(root)
+      DIRS.each do |dir|
+        path = File.join(root, dir)
+        next unless File.exist?(path)
+        load!(path)
+      end
+    end
+
+    # Requires all the ruby files from one path in a proper order
+    # @param path [String] path (dir) from which we want to load ruby files in a proper order
+    def self.load!(path)
+      require_all(path)
+    end
+  end
+end

data/lib/karafka/params/dsl.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Params namespace encapsulating all the logic that is directly related to params handling
+  module Params
+    # Dsl for Karafka params. We don't provide the params class here as we want to allow users to
+    # use either hash (default) or Rails hash with indifferent access as a base for their params
+    #
+    # We do that because both of them have their own advantages and we don't want to enforce users
+    # to handle things differently if they already use any of those
+    #
+    # It provides lazy loading not only until the first usage, but also allows us to skip
+    # using parser until we execute our logic. That way we can operate with
+    # heavy-parsing data without slowing down the whole application.
+    module Dsl
+      # Params keys that are "our" and internal. We use this list for additional backends
+      # that somehow operatae on those keys
+      SYSTEM_KEYS = %w[
+        parser
+        value
+        partition
+        offset
+        key
+        create_time
+        receive_time
+        topic
+        parsed
+      ].freeze
+
+      # Params attributes that should be available via a method call invocation for Kafka
+      # client compatibility.
+      # Kafka passes internally Kafka::FetchedMessage object and the ruby-kafka consumer
+      # uses those fields via method calls, so in order to be able to pass there our params
+      # objects, have to have same api.
+      METHOD_ATTRIBUTES = %w[
+        topic
+        partition
+        offset
+        key
+        create_time
+        receive_time
+      ].freeze
+
+      private_constant :METHOD_ATTRIBUTES
+
+      # Class methods required by params to work
+      module ClassMethods
+        # We allow building instances only via the #build method
+
+        # @param message [Kafka::FetchedMessage, Hash] message that we get out of Kafka
+        #   in case of building params inside main Karafka process in
+        #   Karafka::Connection::Consumer, or a hash when we retrieve data that is already parsed
+        # @param parser [Class] parser class that we will use to unparse data
+        # @return [Karafka::Params::Params] Karafka params object not yet used parser for
+        #   retrieving data that we've got from Kafka
+        # @example Build params instance from a hash
+        #   Karafka::Params::Params.build({ key: 'value' }) #=> params object
+        # @example Build params instance from a Kafka::FetchedMessage object
+        #   Karafka::Params::Params.build(message) #=> params object
+        def build(message, parser)
+          instance = new
+          instance['parser'] = parser
+
+          # Non kafka fetched message can happen when we interchange data with an
+          # additional backend
+          if message.is_a?(Kafka::FetchedMessage)
+            instance.send(
+              :merge!,
+              'value' => message.value,
+              'partition' => message.partition,
+              'offset' => message.offset,
+              'key' => message.key,
+              'create_time' => message.create_time,
+              'receive_time' => Time.now,
+              # When we get raw messages, they might have a topic, that was modified by a
+              # topic mapper. We need to "reverse" this change and map back to the non-modified
+              # format, so our internal flow is not corrupted with the mapping
+              'topic' => Karafka::App.config.topic_mapper.incoming(message.topic)
+            )
+          else
+            instance.send(:merge!, message)
+          end
+
+          instance
+        end
+      end
+
+      # @return [Karafka::Params::Params] this will trigger parser execution. If we decide to
+      #   retrieve data, parser will be executed to parse data. Output of parsing will be merged
+      #   to the current object. This object will be also marked as already parsed, so we won't
+      #   parse it again.
+      def retrieve!
+        return self if self['parsed']
+        self['parsed'] = true
+
+        merge!(parse(delete('value')))
+      end
+
+      # Includes and extends the base params klass with everything that is needed by Karafka to
+      #   fully work in any conditions.
+      # @param params_klass [Karafka::Params::Params] initialized params class that we will
+      #   use for a given Karafka process
+      def self.included(params_klass)
+        params_klass.extend(Dsl::ClassMethods)
+
+        METHOD_ATTRIBUTES.each do |attr|
+          # Defines a method call accessor to a particular hash field.
+          # @note Won't work for complex key names that contain spaces, etc
+          # @param key [Symbol] name of a field that we want to retrieve with a method call
+          # @example
+          #   key_attr_reader :example
+          #   params.example #=> 'my example value'
+          params_klass.send :define_method, attr do
+            self[attr]
+          end
+        end
+
+        params_klass.send :private, :merge!
+        params_klass.send :private, :parse
+      end
+
+      private
+
+      # Overwritten merge! method - it behaves differently for keys that are the same in our hash
+      #  and in a other_hash - it will not replace keys that are the same in our hash
+      #  and in the other one. This protects some important Karafka params keys that cannot be
+      #  replaced with custom values from incoming Kafka message
+      # @param other_hash [Hash] hash that we want to merge into current
+      # @return [Karafka::Params::Params] our parameters hash with merged values
+      # @example Merge with hash without same keys
+      #   new(a: 1, b: 2).merge!(c: 3) #=> { a: 1, b: 2, c: 3 }
+      # @example Merge with hash with same keys (symbol based)
+      #   new(a: 1).merge!(a: 2) #=> { a: 1 }
+      # @example Merge with hash with same keys (string based)
+      #   new(a: 1).merge!('a' => 2) #=> { a: 1 }
+      # @example Merge with hash with same keys (current string based)
+      #   new('a' => 1).merge!(a: 2) #=> { a: 1 }
+      def merge!(other_hash)
+        super(other_hash) { |_key, base_value, _new_value| base_value }
+      end
+
+      # @param value [String] Raw data that we want to parse using consumer parser
+      # @note If something goes wrong, it will return raw data in a hash with a message key
+      # @return [Hash] parsed data or a hash with message key containing raw data if something
+      #   went wrong during parsing
+      def parse(value)
+        Karafka.monitor.instrument('params.params.parse', caller: self) do
+          self['parser'].parse(value)
+        end
+      rescue ::Karafka::Errors::ParserError => e
+        Karafka.monitor.instrument('params.params.parse.error', caller: self, error: e)
+        raise e
+      end
+    end
+  end
+end