karafka 1.2.13 → 1.3.0.rc1

data/lib/karafka/params/builders/metadata.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Params
+    module Builders
+      # Builder for creating metadata object based on the message or batch informations
+      # @note We have 2 ways of creating metadata based on the way ruby-kafka operates
+      module Metadata
+        class << self
+          # Creates metadata based on the kafka batch data
+          # @param kafka_batch [Kafka::FetchedBatch] kafka batch details
+          # @param topic [Karafka::Routing::Topic] topic for which we've fetched the batch
+          # @return [Karafka::Params::Metadata] metadata object
+          def from_kafka_batch(kafka_batch, topic)
+            Karafka::Params::Metadata
+              .new
+              .merge!(
+                'batch_size' => kafka_batch.messages.count,
+                'first_offset' => kafka_batch.first_offset,
+                'highwater_mark_offset' => kafka_batch.highwater_mark_offset,
+                'last_offset' => kafka_batch.last_offset,
+                'offset_lag' => kafka_batch.offset_lag,
+                'deserializer' => topic.deserializer,
+                'partition' => kafka_batch.partition,
+                'topic' => kafka_batch.topic,
+                'unknown_last_offset' => kafka_batch.unknown_last_offset?
+              )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/params/builders/params.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Params
+    # Due to the fact, that we create params related objects in couple contexts / places
+    # plus backends can build up them their own way we have this namespace.
+    # It allows to isolate actual params objects from their building process that can be
+    # context dependent.
+    module Builders
+      # Builder for params
+      module Params
+        class << self
+          # @param kafka_message [Kafka::FetchedMessage] message fetched from Kafka
+          # @param topic [Karafka::Routing::Topic] topic for which this message was fetched
+          # @return [Karafka::Params::Params] params object
+          def from_kafka_message(kafka_message, topic)
+            Karafka::Params::Params
+              .new
+              .merge!(
+                'create_time' => kafka_message.create_time,
+                'headers' => kafka_message.headers || {},
+                'is_control_record' => kafka_message.is_control_record,
+                'key' => kafka_message.key,
+                'offset' => kafka_message.offset,
+                'deserializer' => topic.deserializer,
+                'partition' => kafka_message.partition,
+                'receive_time' => Time.now,
+                'topic' => kafka_message.topic,
+                'payload' => kafka_message.value
+              )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/params/builders/params_batch.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Params
+    module Builders
+      # Builder for creating params batch instances
+      module ParamsBatch
+        class << self
+          # Creates params batch with params inside based on the incoming messages
+          # and the topic from which it comes
+          # @param kafka_messages [Array<Kafka::FetchedMessage>] raw fetched messages
+          # @param topic [Karafka::Routing::Topic] topic for which we're received messages
+          # @return [Karafka::Params::ParamsBatch<Karafka::Params::Params>] batch with params
+          def from_kafka_messages(kafka_messages, topic)
+            params_array = kafka_messages.map! do |message|
+              Karafka::Params::Builders::Params.from_kafka_message(message, topic)
+            end
+
+            Karafka::Params::ParamsBatch.new(params_array)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/params/metadata.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Params
+    # Simple metadata object that stores all non-message information received from Kafka cluster
+    # while fetching the data
+    class Metadata < Hash
+      # Attributes that should be accessible as methods as well (not only hash)
+      METHOD_ATTRIBUTES = %w[
+        batch_size
+        first_offset
+        highwater_mark_offset
+        last_offset
+        offset_lag
+        deserializer
+        partition
+        topic
+      ].freeze
+
+      private_constant :METHOD_ATTRIBUTES
+
+      METHOD_ATTRIBUTES.each do |attr|
+        # Defines a method call accessor to a particular hash field.
+        define_method(attr) do
+          self[attr]
+        end
+      end
+
+      # @return [Boolean] is the last offset known or unknown
+      def unknown_last_offset?
+        self['unknown_last_offset']
+      end
+    end
+  end
+end

data/lib/karafka/params/params.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Params namespace encapsulating all the logic that is directly related to params handling
+  module Params
+    # It provides lazy loading not only until the first usage, but also allows us to skip
+    # using deserializer until we execute our logic. That way we can operate with
+    # heavy-deserialization data without slowing down the whole application.
+    class Params < Hash
+      # Params attributes that should be available via a method call invocation for Kafka
+      # client compatibility.
+      # Kafka passes internally Kafka::FetchedMessage object and the ruby-kafka consumer
+      # uses those fields via method calls, so in order to be able to pass there our params
+      # objects, have to have same api.
+      METHOD_ATTRIBUTES = %w[
+        create_time
+        headers
+        is_control_record
+        key
+        offset
+        deserializer
+        deserialized
+        partition
+        receive_time
+        topic
+        payload
+      ].freeze
+
+      private_constant :METHOD_ATTRIBUTES
+
+      METHOD_ATTRIBUTES.each do |attr|
+        # Defines a method call accessor to a particular hash field.
+        # @note Won't work for complex key names that contain spaces, etc
+        # @param key [Symbol] name of a field that we want to retrieve with a method call
+        # @example
+        #   key_attr_reader :example
+        #   params.example #=> 'my example payload'
+        define_method(attr) do
+          self[attr]
+        end
+      end
+
+      # @return [Karafka::Params::Params] This method will trigger deserializer execution. If we
+      #   decide to retrieve data, deserializer will be executed to get data. Output of that will
+      #   be merged to the current object. This object will be also marked as already deserialized,
+      #   so we won't deserialize it again.
+      def deserialize!
+        return self if self['deserialized']
+
+        self['deserialized'] = true
+        self['payload'] = deserialize
+        self
+      end
+
+      private
+
+      # @return [Object] deserialized data
+      def deserialize
+        Karafka.monitor.instrument('params.params.deserialize', caller: self) do
+          self['deserializer'].call(self)
+        end
+      rescue ::StandardError => e
+        Karafka.monitor.instrument('params.params.deserialize.error', caller: self, error: e)
+        raise e
+      end
+    end
+  end
+end

data/lib/karafka/params/params_batch.rb

@@ -3,43 +3,58 @@
 module Karafka
   module Params
     # Params batch represents a set of messages received from Kafka.
-    # @note Params internally are lazy loaded before first use. That way we can skip parsing
-    #   process if we have after_fetch that rejects some incoming messages without using params
-    #   It can be also used when handling really heavy data (in terms of parsing).
+    # @note Params internally are lazy loaded before first use. That way we can skip
+    #   deserialization process if we have after_fetch that rejects some incoming messages
+    #   without using params It can be also used when handling really heavy data.
     class ParamsBatch
       include Enumerable
 
-      # Builds up a params batch based on raw kafka messages
-      # @param messages_batch [Array<Kafka::FetchedMessage>] messages batch
-      # @param topic_parser [Class] topic parser for unparsing messages values
-      def initialize(messages_batch, topic_parser)
-        @params_batch = messages_batch.map! do |message|
-          Karafka::Params::Params.build(message, topic_parser)
-        end
+      # @param params_array [Array<Karafka::Params::Params>] array with karafka params
+      # @return [Karafka::Params::ParamsBatch] lazy evaluated params batch object
+      def initialize(params_array)
+        @params_array = params_array
       end
 
-      # @yieldparam [Karafka::Params::Params] each parsed and loaded params instance
-      # @note Invocation of this method will cause loading and parsing each param after another.
-      #   If you want to get access without parsing, please access params_batch directly
+      # @yieldparam [Karafka::Params::Params] each deserialized and loaded params instance
+      # @note Invocation of this method will cause loading and deserializing each param after
+      #   another. If you want to get access without deserializing, please access params_array
+      #   directly
       def each
-        @params_batch.each { |param| yield(param.retrieve!) }
+        @params_array.each { |param| yield(param.deserialize!) }
       end
 
       # @return [Array<Karafka::Params::Params>] returns all the params in a loaded state, so they
       #   can be used for batch insert, etc. Without invoking all, up until first use, they won't
-      #   be parsed
-      def parsed
+      #   be deserialized
+      def deserialize!
         each(&:itself)
       end
 
-      # @return [Karafka::Params::Params] last element after the unparsing process
+      # @return [Array<Object>] array with deserialized payloads. This method can be useful when
+      #   we don't care about metadata and just want to extract all the data payloads from the
+      #   batch
+      def payloads
+        deserialize!.map(&:payload)
+      end
+
+      # @return [Karafka::Params::Params] first element after the deserialization process
+      def first
+        @params_array.first.deserialize!
+      end
+
+      # @return [Karafka::Params::Params] last element after the deserialization process
       def last
-        @params_batch.last.retrieve!
+        @params_array.last.deserialize!
       end
 
-      # @return [Array<Karafka::Params::Params>] pure array with params (not parsed)
+      # @return [Array<Karafka::Params::Params>] pure array with params (not deserialized)
       def to_a
-        @params_batch
+        @params_array
+      end
+
+      # @return [Integer] number of messages in the batch
+      def size
+        @params_array.size
       end
     end
   end

data/lib/karafka/patches/ruby_kafka.rb

@@ -1,34 +1,47 @@
 # frozen_string_literal: true
 
 module Karafka
+  # Namespace for various other libs patches
   module Patches
     # Patches for Ruby Kafka gem
     module RubyKafka
       # This patch allows us to inject business logic in between fetches and before the consumer
       # stop, so we can perform stop commit or anything else that we need since
       # ruby-kafka fetch loop does not allow that directly
-      # We don't wan't to use poll ruby-kafka api as it brings many more problems that we would
+      # We don't won't to use poll ruby-kafka api as it brings many more problems that we would
       # have to take care of. That way, nothing like that ever happens but we get the control
       # over the stopping process that we need (since we're the once that initiate it for each
       # thread)
       def consumer_loop
         super do
-          consumers = Karafka::Persistence::Consumer
-                      .all
+          consumers = Karafka::Persistence::Consumers
+                      .current
                       .values
                       .flat_map(&:values)
-                      .select { |ctrl| ctrl.respond_to?(:run_callbacks) }
+                      .select { |consumer| consumer.class.respond_to?(:after_fetch) }
 
-          if Karafka::App.stopped?
-            consumers.each { |ctrl| ctrl.run_callbacks :before_stop }
+          if Karafka::App.stopping?
+            publish_event(consumers, 'before_stop')
             Karafka::Persistence::Client.read.stop
           else
-            consumers.each { |ctrl| ctrl.run_callbacks :before_poll }
+            publish_event(consumers, 'before_poll')
             yield
-            consumers.each { |ctrl| ctrl.run_callbacks :after_poll }
+            publish_event(consumers, 'after_poll')
           end
         end
       end
+
+      private
+
+      # Notifies consumers about particular events happening
+      # @param consumers [Array<Object>] all consumers that want to be notified about an event
+      # @param event_name [String] name of the event that happened
+      def publish_event(consumers, event_name)
+        consumers.each do |consumer|
+          key = "consumers.#{Helpers::Inflector.map(consumer.class.to_s)}.#{event_name}"
+          Karafka::App.monitor.instrument(key, context: consumer)
+        end
+      end
     end
   end
 end

data/lib/karafka/persistence/client.rb

@@ -7,18 +7,22 @@ module Karafka
       # Thread.current key under which we store current thread messages consumer client
       PERSISTENCE_SCOPE = :client
 
-      # @param client [Karafka::Connection::Client] messages consumer client of
-      #   a current thread
-      # @return [Karafka::Connection::Client] persisted messages consumer client
-      def self.write(client)
-        Thread.current[PERSISTENCE_SCOPE] = client
-      end
+      private_constant :PERSISTENCE_SCOPE
+
+      class << self
+        # @param client [Karafka::Connection::Client] messages consumer client of
+        #   a current thread
+        # @return [Karafka::Connection::Client] persisted messages consumer client
+        def write(client)
+          Thread.current[PERSISTENCE_SCOPE] = client
+        end
 
-      # @return [Karafka::Connection::Client] persisted messages consumer client
-      # @raise [Karafka::Errors::MissingConsumer] raised when no thread messages consumer
-      #   client but we try to use it anyway
-      def self.read
-        Thread.current[PERSISTENCE_SCOPE] || raise(Errors::MissingClient)
+        # @return [Karafka::Connection::Client] persisted messages consumer client
+        # @raise [Karafka::Errors::MissingClientError] raised when no thread messages consumer
+        #   client but we try to use it anyway
+        def read
+          Thread.current[PERSISTENCE_SCOPE] || raise(Errors::MissingClientError)
+        end
       end
     end
   end

data/lib/karafka/persistence/{consumer.rb → consumers.rb}

@@ -7,16 +7,18 @@ module Karafka
     # Module used to provide a persistent cache across batch requests for a given
     # topic and partition to store some additional details when the persistent mode
     # for a given topic is turned on
-    class Consumer
+    class Consumers
       # Thread.current scope under which we store consumers data
       PERSISTENCE_SCOPE = :consumers
 
+      private_constant :PERSISTENCE_SCOPE
+
       class << self
-        # @return [Hash] current thread persistence scope hash with all the consumers
-        def all
-          # @note This does not need to be threadsafe (Hash) as it is always executed in a
-          # current thread context
-          Thread.current[PERSISTENCE_SCOPE] ||= Hash.new { |hash, key| hash[key] = {} }
+        # @return [Hash] current thread's persistence scope hash with all the consumers
+        def current
+          Thread.current[PERSISTENCE_SCOPE] ||= Concurrent::Hash.new do |hash, key|
+            hash[key] = Concurrent::Hash.new
+          end
         end
 
         # Used to build (if block given) and/or fetch a current consumer instance that will be
@@ -25,12 +27,17 @@ module Karafka
         # @param topic [Karafka::Routing::Topic] topic instance for which we might cache
         # @param partition [Integer] number of partition for which we want to cache
         def fetch(topic, partition)
-          # We always store a current instance for callback reasons
-          if topic.persistent
-            all[topic][partition] ||= topic.consumer.new
-          else
-            all[topic][partition] = topic.consumer.new
-          end
+          current[topic][partition] ||= topic.consumer.new(topic)
+        end
+
+        # Removes all persisted instances of consumers from the consumer cache
+        # @note This is used to reload consumers instances when code reloading in development mode
+        #   is present. This should not be used in production.
+        def clear
+          Thread
+            .list
+            .select { |thread| thread[PERSISTENCE_SCOPE] }
+            .each { |thread| thread[PERSISTENCE_SCOPE].clear }
         end
       end
     end

data/lib/karafka/persistence/topics.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Persistence
+    # Local cache for routing topics
+    # We use it in order not to build string instances and remap incoming topic upon each
+    # message / message batches received
+    class Topics
+      # Thread.current scope under which we store topics data
+      PERSISTENCE_SCOPE = :topics
+
+      private_constant :PERSISTENCE_SCOPE
+
+      class << self
+        # @return [Concurrent::Hash] hash with all the topics from given groups
+        def current
+          Thread.current[PERSISTENCE_SCOPE] ||= Concurrent::Hash.new do |hash, key|
+            hash[key] = Concurrent::Hash.new
+          end
+        end
+
+        # @param group_id [String] group id for which we fetch a topic representation
+        # @param raw_topic_name [String] raw topic name (before remapping) for which we fetch a
+        #   topic representation
+        # @return [Karafka::Routing::Topics] remapped topic representation that can be used further
+        #   on when working with given parameters
+        def fetch(group_id, raw_topic_name)
+          current[group_id][raw_topic_name] ||= begin
+            # We map from incoming topic name, as it might be namespaced, etc.
+            # @see topic_mapper internal docs
+            mapped_topic_name = Karafka::App.config.topic_mapper.incoming(raw_topic_name)
+            Routing::Router.find("#{group_id}_#{mapped_topic_name}")
+          end
+        end
+
+        # Clears the whole topics cache for all the threads
+        # This is used for in-development code reloading as we need to get rid of all the
+        # preloaded and cached instances of objects to make it work
+        def clear
+          Thread
+            .list
+            .select { |thread| thread[PERSISTENCE_SCOPE] }
+            .each { |thread| thread[PERSISTENCE_SCOPE].clear }
+        end
+      end
+    end
+  end
+end