karafka 1.1.0 → 1.2.0

data/lib/karafka/instrumentation/logger.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    # Default logger for Event Delegator
+    # @note It uses ::Logger features - providing basic logging
+    class Logger < ::Logger
+      include Singleton
+
+      # Map containing information about log level for given environment
+      ENV_MAP = {
+        'production' => ::Logger::ERROR,
+        'test' => ::Logger::ERROR,
+        'development' => ::Logger::INFO,
+        'debug' => ::Logger::DEBUG,
+        'default' => ::Logger::INFO
+      }.freeze
+
+      # Creates a new instance of logger ensuring that it has a place to write to
+      def initialize(*_args)
+        ensure_dir_exists
+        super(target)
+        self.level = ENV_MAP[Karafka.env] || ENV_MAP['default']
+      end
+
+      private
+
+      # @return [Karafka::Helpers::MultiDelegator] multi delegator instance
+      #   to which we will be writtng logs
+      # We use this approach to log stuff to file and to the STDOUT at the same time
+      def target
+        Karafka::Helpers::MultiDelegator
+          .delegate(:write, :close)
+          .to(STDOUT, file)
+      end
+
+      # Makes sure the log directory exists
+      def ensure_dir_exists
+        dir = File.dirname(log_path)
+        FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
+      end
+
+      # @return [Pathname] Path to a file to which we should log
+      def log_path
+        @log_path ||= Karafka::App.root.join("log/#{Karafka.env}.log")
+      end
+
+      # @return [File] file to which we want to write our logs
+      # @note File is being opened in append mode ('a')
+      def file
+        @file ||= File.open(log_path, 'a')
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/monitor.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace for all the things related with Karafka instrumentation process
+  module Instrumentation
+    # Monitor is used to hookup external monitoring services to monitor how Karafka works
+    # It provides a standardized API for checking incoming messages/enqueueing etc
+    # Since it is a pub-sub based on dry-monitor, you can use as many subscribers/loggers at the
+    # same time, which means that you might have for example file logging and newrelic at the same
+    # time
+    # @note This class acts as a singleton because we are only permitted to have single monitor
+    #   per running process (just as logger)
+    class Monitor < Dry::Monitor::Notifications
+      include Singleton
+
+      # List of events that we support in the system and to which a monitor client can hook up
+      # @note The non-error once support timestamp benchmarking
+      # @note Depending on Karafka extensions and additional engines, this might not be the
+      #   complete list of all the events. Please use the #available_events on fully loaded
+      #   Karafka system to determine all of the events you can use.
+      #   Last 4 events are from WaterDrop but for convenience we use the same monitor for the
+      #   whole karafka ecosystem
+      BASE_EVENTS = %w[
+        params.params.parse
+        params.params.parse.error
+        connection.listener.fetch_loop.error
+        connection.client.fetch_loop.error
+        connection.delegator.call
+        fetcher.call.error
+        backends.inline.process
+        process.notice_signal
+        consumers.responders.respond_with
+        async_producer.call.error
+        async_producer.call.retry
+        sync_producer.call.error
+        sync_producer.call.retry
+        server.stop
+        server.stop.error
+      ].freeze
+
+      private_constant :BASE_EVENTS
+
+      # @return [Karafka::Instrumentation::Monitor] monitor instance for system instrumentation
+      def initialize
+        super(:karafka)
+        BASE_EVENTS.each(&method(:register_event))
+      end
+
+      # Allows us to subscribe to events with a code that will be yielded upon events
+      # @param event_name_or_listener [String, Object] name of the event we want to subscribe to
+      #   or a listener if we decide to go with object listener
+      def subscribe(event_name_or_listener)
+        return super unless event_name_or_listener.is_a?(String)
+        return super if available_events.include?(event_name_or_listener)
+        raise Errors::UnregisteredMonitorEvent, event_name_or_listener
+      end
+
+      # @return [Array<String>] names of available events to which we can subscribe
+      def available_events
+        __bus__.events.keys
+      end
+    end
+  end
+end

data/lib/karafka/loader.rb

@@ -5,7 +5,6 @@ module Karafka
   module Loader
     # Order in which we want to load app files
     DIRS = %w[
-      config/initializers
       lib
       app
     ].freeze

data/lib/karafka/params/{params.rb → dsl.rb}

@@ -3,18 +3,28 @@
 module Karafka
   # Params namespace encapsulating all the logic that is directly related to params handling
   module Params
-    # Class-wrapper for hash with indifferent access with additional lazy loading feature
+    # Dsl for Karafka params. We don't provide the params class here as we want to allow users to
+    # use either hash (default) or Rails hash with indifferent access as a base for their params
+    #
+    # We do that because both of them have their own advantages and we don't want to enforce users
+    # to handle things differently if they already use any of those
+    #
     # It provides lazy loading not only until the first usage, but also allows us to skip
     # using parser until we execute our logic. That way we can operate with
     # heavy-parsing data without slowing down the whole application.
-    class Params < HashWithIndifferentAccess
-      # Kafka::FetchedMessage attributes that we want to use inside of params
-      KAFKA_MESSAGE_ATTRIBUTES = %i[
+    module Dsl
+      # Params keys that are "our" and internal. We use this list for additional backends
+      # that somehow operatae on those keys
+      SYSTEM_KEYS = %w[
+        parser
         value
         partition
         offset
         key
         create_time
+        receive_time
+        topic
+        parsed
       ].freeze
 
       # Params attributes that should be available via a method call invocation for Kafka
@@ -22,15 +32,19 @@ module Karafka
       # Kafka passes internally Kafka::FetchedMessage object and the ruby-kafka consumer
       # uses those fields via method calls, so in order to be able to pass there our params
       # objects, have to have same api.
-      PARAMS_METHOD_ATTRIBUTES = %i[
+      METHOD_ATTRIBUTES = %w[
         topic
         partition
         offset
         key
         create_time
+        receive_time
       ].freeze
 
-      class << self
+      private_constant :METHOD_ATTRIBUTES
+
+      # Class methods required by params to work
+      module ClassMethods
         # We allow building instances only via the #build method
 
         # @param message [Kafka::FetchedMessage, Hash] message that we get out of Kafka
@@ -44,38 +58,30 @@ module Karafka
         # @example Build params instance from a Kafka::FetchedMessage object
         #   Karafka::Params::Params.build(message) #=> params object
         def build(message, parser)
-          # Hash case happens inside backends that interchange data
-          if message.is_a?(Hash)
-            new(parser: parser).send(:merge!, message)
-          else
-            # This happens inside Kafka::FetchedProcessor
-            new(
-              parser: parser,
-              parsed: false,
-              received_at: Time.now
-            ).tap do |instance|
-              KAFKA_MESSAGE_ATTRIBUTES.each do |attribute|
-                instance[attribute] = message.send(attribute)
-              end
+          instance = new
+          instance['parser'] = parser
 
+          # Non kafka fetched message can happen when we interchange data with an
+          # additional backend
+          if message.is_a?(Kafka::FetchedMessage)
+            instance.send(
+              :merge!,
+              'value' => message.value,
+              'partition' => message.partition,
+              'offset' => message.offset,
+              'key' => message.key,
+              'create_time' => message.create_time,
+              'receive_time' => Time.now,
               # When we get raw messages, they might have a topic, that was modified by a
               # topic mapper. We need to "reverse" this change and map back to the non-modified
               # format, so our internal flow is not corrupted with the mapping
-              instance[:topic] = Karafka::App.config.topic_mapper.incoming(message.topic)
-            end
+              'topic' => Karafka::App.config.topic_mapper.incoming(message.topic)
+            )
+          else
+            instance.send(:merge!, message)
           end
-        end
 
-        # Defines a method call accessor to a particular hash field.
-        # @note Won't work for complex key names that contain spaces, etc
-        # @param key [Symbol] name of a field that we want to retrieve with a method call
-        # @example
-        #   key_attr_reader :example
-        #   params.example #=> 'my example value'
-        def key_attr_reader(key)
-          define_method key do
-            self[key]
-          end
+          instance
         end
       end
 
@@ -84,19 +90,42 @@ module Karafka
       #   to the current object. This object will be also marked as already parsed, so we won't
       #   parse it again.
       def retrieve!
-        return self if self[:parsed]
+        return self if self['parsed']
+        self['parsed'] = true
 
-        merge!(parse(delete(:value)))
+        merge!(parse(delete('value')))
       end
 
-      PARAMS_METHOD_ATTRIBUTES.each(&method(:key_attr_reader))
+      # Includes and extends the base params klass with everything that is needed by Karafka to
+      #   fully work in any conditions.
+      # @param params_klass [Karafka::Params::Params] initialized params class that we will
+      #   use for a given Karafka process
+      def self.included(params_klass)
+        params_klass.extend(Dsl::ClassMethods)
+
+        METHOD_ATTRIBUTES.each do |attr|
+          # Defines a method call accessor to a particular hash field.
+          # @note Won't work for complex key names that contain spaces, etc
+          # @param key [Symbol] name of a field that we want to retrieve with a method call
+          # @example
+          #   key_attr_reader :example
+          #   params.example #=> 'my example value'
+          params_klass.send :define_method, attr do
+            self[attr]
+          end
+        end
+
+        params_klass.send :private, :merge!
+        params_klass.send :private, :parse
+      end
 
       private
 
       # Overwritten merge! method - it behaves differently for keys that are the same in our hash
       #  and in a other_hash - it will not replace keys that are the same in our hash
-      #  and in the other one
-      # @param other_hash [Hash, HashWithIndifferentAccess] hash that we want to merge into current
+      #  and in the other one. This protects some important Karafka params keys that cannot be
+      #  replaced with custom values from incoming Kafka message
+      # @param other_hash [Hash] hash that we want to merge into current
       # @return [Karafka::Params::Params] our parameters hash with merged values
       # @example Merge with hash without same keys
       #   new(a: 1, b: 2).merge!(c: 3) #=> { a: 1, b: 2, c: 3 }
@@ -110,18 +139,17 @@ module Karafka
         super(other_hash) { |_key, base_value, _new_value| base_value }
       end
 
-      # @param value [String] Raw data that we want to parse using controller's parser
+      # @param value [String] Raw data that we want to parse using consumer parser
       # @note If something goes wrong, it will return raw data in a hash with a message key
       # @return [Hash] parsed data or a hash with message key containing raw data if something
       #   went wrong during parsing
       def parse(value)
-        self[:parser].parse(value)
-        # We catch both of them, because for default JSON - we use JSON parser directly
+        Karafka.monitor.instrument('params.params.parse', caller: self) do
+          self['parser'].parse(value)
+        end
       rescue ::Karafka::Errors::ParserError => e
-        Karafka.monitor.notice_error(self.class, e)
+        Karafka.monitor.instrument('params.params.parse.error', caller: self, error: e)
         raise e
-      ensure
-        self[:parsed] = true
       end
     end
   end

data/lib/karafka/params/params_batch.rb

@@ -4,7 +4,7 @@ module Karafka
   module Params
     # Params batch represents a set of messages received from Kafka.
     # @note Params internally are lazy loaded before first use. That way we can skip parsing
-    #   process if we have after_fetched that rejects some incoming messages without using params
+    #   process if we have after_fetch that rejects some incoming messages without using params
     #   It can be also used when handling really heavy data (in terms of parsing).
     class ParamsBatch
       include Enumerable
@@ -13,7 +13,7 @@ module Karafka
       # @param messages_batch [Array<Kafka::FetchedMessage>] messages batch
       # @param topic_parser [Class] topic parser for unparsing messages values
       def initialize(messages_batch, topic_parser)
-        @params_batch = messages_batch.map do |message|
+        @params_batch = messages_batch.map! do |message|
           Karafka::Params::Params.build(message, topic_parser)
         end
       end
@@ -32,6 +32,11 @@ module Karafka
         each(&:itself)
       end
 
+      # @return [Karafka::Params::Params] last element after the unparsing process
+      def last
+        @params_batch.last.retrieve!
+      end
+
       # @return [Array<Karafka::Params::Params>] pure array with params (not parsed)
       def to_a
         @params_batch

data/lib/karafka/patches/dry_configurable.rb

@@ -19,11 +19,15 @@ module Karafka
       private
 
       # Method that rebuilds a given accessor, so when it consists a proc value, it will
-      # evaluate it upon return
+      # evaluate it upon return for blocks that don't require any arguments, otherwise
+      # it will return the block
       # @param method_name [Symbol] name of an accessor that we want to rebuild
       def rebuild(method_name)
         define_singleton_method method_name do
-          super().is_a?(Proc) ? super().call : super()
+          value = super()
+          return value unless value.is_a?(Proc)
+          return value unless value.parameters.empty?
+          value.call
         end
       end
     end

data/lib/karafka/patches/ruby_kafka.rb

@@ -2,7 +2,7 @@
 
 module Karafka
   module Patches
-    # Batches for Ruby Kafka gem
+    # Patches for Ruby Kafka gem
     module RubyKafka
       # This patch allows us to inject business logic in between fetches and before the consumer
       # stop, so we can perform stop commit or anything else that we need since
@@ -13,19 +13,19 @@ module Karafka
       # thread)
       def consumer_loop
         super do
-          controllers = Karafka::Persistence::Controller
-                        .all
-                        .values
-                        .flat_map(&:values)
-                        .select { |ctrl| ctrl.respond_to?(:run_callbacks) }
+          consumers = Karafka::Persistence::Consumer
+                      .all
+                      .values
+                      .flat_map(&:values)
+                      .select { |ctrl| ctrl.respond_to?(:run_callbacks) }
 
           if Karafka::App.stopped?
-            controllers.each { |ctrl| ctrl.run_callbacks :before_stop }
-            Karafka::Persistence::Consumer.read.stop
+            consumers.each { |ctrl| ctrl.run_callbacks :before_stop }
+            Karafka::Persistence::Client.read.stop
           else
-            controllers.each { |ctrl| ctrl.run_callbacks :before_poll }
+            consumers.each { |ctrl| ctrl.run_callbacks :before_poll }
             yield
-            controllers.each { |ctrl| ctrl.run_callbacks :after_poll }
+            consumers.each { |ctrl| ctrl.run_callbacks :after_poll }
           end
         end
       end

data/lib/karafka/persistence/client.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Persistence
+    # Persistence layer to store current thread messages consumer client for further use
+    class Client
+      # Thread.current key under which we store current thread messages consumer client
+      PERSISTENCE_SCOPE = :client
+
+      # @param client [Karafka::Connection::Client] messages consumer client of
+      #   a current thread
+      # @return [Karafka::Connection::Client] persisted messages consumer client
+      def self.write(client)
+        Thread.current[PERSISTENCE_SCOPE] = client
+      end
+
+      # @return [Karafka::Connection::Client] persisted messages consumer client
+      # @raise [Karafka::Errors::MissingConsumer] raised when no thread messages consumer
+      #   client but we try to use it anyway
+      def self.read
+        Thread.current[PERSISTENCE_SCOPE] || raise(Errors::MissingClient)
+      end
+    end
+  end
+end

data/lib/karafka/persistence/consumer.rb

@@ -1,24 +1,37 @@
 # frozen_string_literal: true
 
 module Karafka
+  # Module used to provide a persistent cache layer for Karafka components that need to be
+  # shared inside of a same thread
   module Persistence
-    # Persistence layer to store current thread messages consumer for further use
+    # Module used to provide a persistent cache across batch requests for a given
+    # topic and partition to store some additional details when the persistent mode
+    # for a given topic is turned on
     class Consumer
-      # Thread.current key under which we store current thread messages consumer
-      PERSISTENCE_SCOPE = :consumer
+      # Thread.current scope under which we store consumers data
+      PERSISTENCE_SCOPE = :consumers
 
-      # @param consumer [Karafka::Connection::Consumer] messages consumer of
-      #   a current thread
-      # @return [Karafka::Connection::Consumer] persisted messages consumer
-      def self.write(consumer)
-        Thread.current[PERSISTENCE_SCOPE] = consumer
-      end
+      class << self
+        # @return [Hash] current thread persistence scope hash with all the consumers
+        def all
+          # @note This does not need to be threadsafe (Hash) as it is always executed in a
+          # current thread context
+          Thread.current[PERSISTENCE_SCOPE] ||= Hash.new { |hash, key| hash[key] = {} }
+        end
 
-      # @return [Karafka::Connection::Consumer] persisted messages consumer
-      # @raise [Karafka::Errors::MissingConsumer] raised when no thread messages consumer
-      #   but we try to use it anyway
-      def self.read
-        Thread.current[PERSISTENCE_SCOPE] || raise(Errors::MissingConsumer)
+        # Used to build (if block given) and/or fetch a current consumer instance that will be
+        #   used to process messages from a given topic and partition
+        # @return [Karafka::BaseConsumer] base consumer descendant
+        # @param topic [Karafka::Routing::Topic] topic instance for which we might cache
+        # @param partition [Integer] number of partition for which we want to cache
+        def fetch(topic, partition)
+          # We always store a current instance for callback reasons
+          if topic.persistent
+            all[topic][partition] ||= topic.consumer.new
+          else
+            all[topic][partition] = topic.consumer.new
+          end
+        end
       end
     end
   end