karafka 1.4.0 → 2.0.10

data/lib/karafka/instrumentation/vendors/datadog/listener.rb

@@ -0,0 +1,232 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    # Namespace for vendor specific instrumentation
+    module Vendors
+      # Datadog specific instrumentation
+      module Datadog
+        # Listener that can be used to subscribe to Karafka to receive stats via StatsD
+        # and/or Datadog
+        #
+        # @note You need to setup the `dogstatsd-ruby` client and assign it
+        class Listener
+          include ::Karafka::Core::Configurable
+          extend Forwardable
+
+          def_delegators :config, :client, :rd_kafka_metrics, :namespace, :default_tags
+
+          # Value object for storing a single rdkafka metric publishing details
+          RdKafkaMetric = Struct.new(:type, :scope, :name, :key_location)
+
+          # Namespace under which the DD metrics should be published
+          setting :namespace, default: 'karafka'
+
+          # Datadog client that we should use to publish the metrics
+          setting :client
+
+          # Default tags we want to publish (for example hostname)
+          # Format as followed (example for hostname): `["host:#{Socket.gethostname}"]`
+          setting :default_tags, default: []
+
+          # All the rdkafka metrics we want to publish
+          #
+          # By default we publish quite a lot so this can be tuned
+          # Note, that the once with `_d` come from Karafka, not rdkafka or Kafka
+          setting :rd_kafka_metrics, default: [
+            # Client metrics
+            RdKafkaMetric.new(:count, :root, 'messages.consumed', 'rxmsgs_d'),
+            RdKafkaMetric.new(:count, :root, 'messages.consumed.bytes', 'rxmsg_bytes'),
+
+            # Broker metrics
+            RdKafkaMetric.new(:count, :brokers, 'consume.attempts', 'txretries_d'),
+            RdKafkaMetric.new(:count, :brokers, 'consume.errors', 'txerrs_d'),
+            RdKafkaMetric.new(:count, :brokers, 'receive.errors', 'rxerrs_d'),
+            RdKafkaMetric.new(:count, :brokers, 'connection.connects', 'connects_d'),
+            RdKafkaMetric.new(:count, :brokers, 'connection.disconnects', 'disconnects_d'),
+            RdKafkaMetric.new(:gauge, :brokers, 'network.latency.avg', %w[rtt avg]),
+            RdKafkaMetric.new(:gauge, :brokers, 'network.latency.p95', %w[rtt p95]),
+            RdKafkaMetric.new(:gauge, :brokers, 'network.latency.p99', %w[rtt p99])
+          ].freeze
+
+          configure
+
+          # @param block [Proc] configuration block
+          def initialize(&block)
+            configure
+            setup(&block) if block
+          end
+
+          # @param block [Proc] configuration block
+          # @note We define this alias to be consistent with `WaterDrop#setup`
+          def setup(&block)
+            configure(&block)
+          end
+
+          # Hooks up to WaterDrop instrumentation for emitted statistics
+          #
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_statistics_emitted(event)
+            statistics = event[:statistics]
+
+            rd_kafka_metrics.each do |metric|
+              report_metric(metric, statistics)
+            end
+          end
+
+          # Increases the errors count by 1
+          #
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_error_occurred(event)
+            extra_tags = ["type:#{event[:type]}"]
+
+            if event.payload[:caller].respond_to?(:messages)
+              metadata = event.payload[:caller].messages.metadata
+
+              extra_tags += [
+                "topic:#{metadata.topic}",
+                "partition:#{metadata.partition}"
+              ]
+            end
+
+            count('error_occurred', 1, tags: default_tags + extra_tags)
+          end
+
+          # Reports how many messages we've polled and how much time did we spend on it
+          #
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_connection_listener_fetch_loop_received(event)
+            time_taken = event[:time]
+            messages_count = event[:messages_buffer].size
+
+            histogram('listener.polling.time_taken', time_taken, tags: default_tags)
+            histogram('listener.polling.messages', messages_count, tags: default_tags)
+          end
+
+          # Here we report majority of things related to processing as we have access to the
+          # consumer
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_consumer_consumed(event)
+            messages = event.payload[:caller].messages
+            metadata = messages.metadata
+
+            tags = default_tags + [
+              "topic:#{metadata.topic}",
+              "partition:#{metadata.partition}"
+            ]
+
+            count('consumer.messages', messages.count, tags: tags)
+            count('consumer.batches', 1, tags: tags)
+            gauge('consumer.offset', metadata.last_offset, tags: tags)
+            histogram('consumer.consumed.time_taken', event[:time], tags: tags)
+            histogram('consumer.batch_size', messages.count, tags: tags)
+            histogram('consumer.processing_lag', metadata.processing_lag, tags: tags)
+            histogram('consumer.consumption_lag', metadata.consumption_lag, tags: tags)
+          end
+
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_consumer_revoked(event)
+            messages = event.payload[:caller].messages
+            metadata = messages.metadata
+
+            tags = default_tags + [
+              "topic:#{metadata.topic}",
+              "partition:#{metadata.partition}"
+            ]
+
+            count('consumer.revoked', 1, tags: tags)
+          end
+
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_consumer_shutdown(event)
+            messages = event.payload[:caller].messages
+            metadata = messages.metadata
+
+            tags = default_tags + [
+              "topic:#{metadata.topic}",
+              "partition:#{metadata.partition}"
+            ]
+
+            count('consumer.shutdown', 1, tags: tags)
+          end
+
+          # Worker related metrics
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_worker_process(event)
+            jq_stats = event[:jobs_queue].statistics
+
+            gauge('worker.total_threads', Karafka::App.config.concurrency, tags: default_tags)
+            histogram('worker.processing', jq_stats[:processing], tags: default_tags)
+            histogram('worker.enqueued_jobs', jq_stats[:enqueued], tags: default_tags)
+          end
+
+          # We report this metric before and after processing for higher accuracy
+          # Without this, the utilization would not be fully reflected
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_worker_processed(event)
+            jq_stats = event[:jobs_queue].statistics
+
+            histogram('worker.processing', jq_stats[:processing], tags: default_tags)
+          end
+
+          private
+
+          %i[
+            count
+            gauge
+            histogram
+            increment
+            decrement
+          ].each do |metric_type|
+            class_eval <<~METHODS, __FILE__, __LINE__ + 1
+              def #{metric_type}(key, *args)
+                client.#{metric_type}(
+                  namespaced_metric(key),
+                  *args
+                )
+              end
+            METHODS
+          end
+
+          # Wraps metric name in listener's namespace
+          # @param metric_name [String] RdKafkaMetric name
+          # @return [String]
+          def namespaced_metric(metric_name)
+            "#{namespace}.#{metric_name}"
+          end
+
+          # Reports a given metric statistics to Datadog
+          # @param metric [RdKafkaMetric] metric value object
+          # @param statistics [Hash] hash with all the statistics emitted
+          def report_metric(metric, statistics)
+            case metric.scope
+            when :root
+              public_send(
+                metric.type,
+                metric.name,
+                statistics.fetch(*metric.key_location),
+                tags: default_tags
+              )
+            when :brokers
+              statistics.fetch('brokers').each_value do |broker_statistics|
+                # Skip bootstrap nodes
+                # Bootstrap nodes have nodeid -1, other nodes have positive
+                # node ids
+                next if broker_statistics['nodeid'] == -1
+
+                public_send(
+                  metric.type,
+                  metric.name,
+                  broker_statistics.dig(*metric.key_location),
+                  tags: default_tags + ["broker:#{broker_statistics['nodename']}"]
+                )
+              end
+            else
+              raise ArgumentError, metric.scope
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Karafka
+  # @note Since we can only have one statistics callbacks manager and one error callbacks manager
+  # we use WaterDrops one that is already configured.
+  module Instrumentation
+    class << self
+      # Returns a manager for statistics callbacks
+      # @return [::WaterDrop::CallbacksManager]
+      def statistics_callbacks
+        ::WaterDrop::Instrumentation.statistics_callbacks
+      end
+
+      # Returns a manager for error callbacks
+      # @return [::WaterDrop::CallbacksManager]
+      def error_callbacks
+        ::WaterDrop::Instrumentation.error_callbacks
+      end
+    end
+  end
+end

data/lib/karafka/licenser.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Checks the license presence for pro and loads pro components when needed (if any)
+  class Licenser
+    # Location in the gem where we store the public key
+    PUBLIC_KEY_LOCATION = File.join(Karafka.gem_root, 'certs', 'karafka-pro.pem')
+
+    private_constant :PUBLIC_KEY_LOCATION
+
+    # Tries to prepare license and verifies it
+    #
+    # @param license_config [Karafka::Core::Configurable::Node] config related to the licensing
+    def prepare_and_verify(license_config)
+      prepare(license_config)
+      verify(license_config)
+    end
+
+    private
+
+    # @param license_config [Karafka::Core::Configurable::Node] config related to the licensing
+    def prepare(license_config)
+      # If there is token, no action needed
+      # We support a case where someone would put the token in instead of using one from the
+      # license. That's in case there are limitations to using external package sources, etc
+      return if license_config.token
+
+      begin
+        license_config.token || require('karafka-license')
+      rescue LoadError
+        return
+      end
+
+      license_config.token = Karafka::License.token
+    end
+
+    # Check license and setup license details (if needed)
+    # @param license_config [Karafka::Core::Configurable::Node] config related to the licensing
+    def verify(license_config)
+      # If no license, it will just run LGPL components without anything extra
+      return unless license_config.token
+
+      public_key = OpenSSL::PKey::RSA.new(File.read(PUBLIC_KEY_LOCATION))
+
+      # We gsub and strip in case someone copy-pasted it as a multi line string
+      formatted_token = license_config.token.strip.delete("\n").delete(' ')
+      decoded_token = Base64.decode64(formatted_token)
+
+      begin
+        data = public_key.public_decrypt(decoded_token)
+      rescue OpenSSL::OpenSSLError
+        data = nil
+      end
+
+      details = data ? JSON.parse(data) : raise_invalid_license_token(license_config)
+
+      license_config.entity = details.fetch('entity')
+    end
+
+    # Raises an error with info, that used token is invalid
+    # @param license_config [Karafka::Core::Configurable::Node]
+    def raise_invalid_license_token(license_config)
+      # We set it to false so `Karafka.pro?` method behaves as expected
+      license_config.token = false
+
+      raise(
+        Errors::InvalidLicenseTokenError,
+        <<~MSG.tr("\n", ' ')
+          License key you provided is invalid.
+          Please reach us at contact@karafka.io or visit https://karafka.io to obtain a valid one.
+        MSG
+      )
+    end
+  end
+end

data/lib/karafka/messages/batch_metadata.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Messages
+    # Simple batch metadata object that stores all non-message information received from Kafka
+    # cluster while fetching the data.
+    #
+    # @note This metadata object refers to per batch metadata, not `#message.metadata`
+    BatchMetadata = Struct.new(
+      :size,
+      :first_offset,
+      :last_offset,
+      :deserializer,
+      :partition,
+      :topic,
+      :created_at,
+      :scheduled_at,
+      :processed_at,
+      keyword_init: true
+    ) do
+      # This lag describes how long did it take for a message to be consumed from the moment it was
+      # created
+      def consumption_lag
+        time_distance_in_ms(processed_at, created_at)
+      end
+
+      # This lag describes how long did a batch have to wait before it was picked up by one of the
+      # workers
+      def processing_lag
+        time_distance_in_ms(processed_at, scheduled_at)
+      end
+
+      private
+
+      # Computes time distance in between two times in ms
+      #
+      # @param time1 [Time]
+      # @param time2 [Time]
+      # @return [Integer] distance in between two times in ms
+      def time_distance_in_ms(time1, time2)
+        ((time1 - time2) * 1_000).round
+      end
+    end
+  end
+end

data/lib/karafka/messages/builders/batch_metadata.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Messages
+    module Builders
+      # Builder for creating batch metadata object based on the batch informations.
+      module BatchMetadata
+        class << self
+          # Creates metadata based on the kafka batch data.
+          #
+          # @param messages [Array<Karafka::Messages::Message>] messages array
+          # @param topic [Karafka::Routing::Topic] topic for which we've fetched the batch
+          # @param scheduled_at [Time] moment when the batch was scheduled for processing
+          # @return [Karafka::Messages::BatchMetadata] batch metadata object
+          #
+          # @note We do not set `processed_at` as this needs to be assigned when the batch is
+          #   picked up for processing.
+          def call(messages, topic, scheduled_at)
+            Karafka::Messages::BatchMetadata.new(
+              size: messages.count,
+              first_offset: messages.first.offset,
+              last_offset: messages.last.offset,
+              deserializer: topic.deserializer,
+              partition: messages.first.partition,
+              topic: topic.name,
+              # We go with the assumption that the creation of the whole batch is the last message
+              # creation time
+              created_at: messages.last.timestamp,
+              # When this batch was built and scheduled for execution
+              scheduled_at: scheduled_at,
+              # This needs to be set to a correct value prior to processing starting
+              processed_at: nil
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/messages/builders/message.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Messages
+    # Builders encapsulate logic related to creating messages related objects.
+    module Builders
+      # Builder of a single message based on raw rdkafka message.
+      module Message
+        class << self
+          # @param kafka_message [Rdkafka::Consumer::Message] raw fetched message
+          # @param topic [Karafka::Routing::Topic] topic for which this message was fetched
+          # @param received_at [Time] moment when we've received the message
+          # @return [Karafka::Messages::Message] message object with payload and metadata
+          def call(kafka_message, topic, received_at)
+            # @see https://github.com/appsignal/rdkafka-ruby/issues/168
+            kafka_message.headers.transform_keys!(&:to_s)
+
+            metadata = Karafka::Messages::Metadata.new(
+              timestamp: kafka_message.timestamp,
+              headers: kafka_message.headers,
+              key: kafka_message.key,
+              offset: kafka_message.offset,
+              deserializer: topic.deserializer,
+              partition: kafka_message.partition,
+              topic: topic.name,
+              received_at: received_at
+            ).freeze
+
+            # Karafka messages cannot be frozen because of the lazy deserialization feature
+            Karafka::Messages::Message.new(
+              kafka_message.payload,
+              metadata
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/messages/builders/messages.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Messages
+    module Builders
+      # Builder for creating message batch instances.
+      module Messages
+        class << self
+          # Creates messages batch with messages inside based on the incoming messages and the
+          # topic from which it comes.
+          #
+          # @param messages [Array<Karafka::Messages::Message>] karafka messages array
+          # @param topic [Karafka::Routing::Topic] topic for which we're received messages
+          # @param received_at [Time] moment in time when the messages were received
+          # @return [Karafka::Messages::Messages] messages batch object
+          def call(messages, topic, received_at)
+            # We cannot freeze the batch metadata because it is altered with the processed_at time
+            # prior to the consumption. It is being frozen there
+            metadata = BatchMetadata.call(
+              messages,
+              topic,
+              received_at
+            )
+
+            Karafka::Messages::Messages.new(
+              messages,
+              metadata
+            ).freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/{params/params.rb → messages/message.rb}

@@ -1,12 +1,12 @@
 # frozen_string_literal: true
 
 module Karafka
-  # Params namespace encapsulating all the logic that is directly related to params handling
-  module Params
+  # Messages namespace encapsulating all the logic that is directly related to messages handling
+  module Messages
     # It provides lazy loading not only until the first usage, but also allows us to skip
     # using deserializer until we execute our logic. That way we can operate with
     # heavy-deserialization data without slowing down the whole application.
-    class Params
+    class Message
       extend Forwardable
 
       attr_reader :raw_payload, :metadata
@@ -14,7 +14,7 @@ module Karafka
       def_delegators :metadata, *Metadata.members
 
       # @param raw_payload [Object] incoming payload before deserialization
-      # @param metadata [Karafka::Params::Metadata] message metadata object
+      # @param metadata [Karafka::Messages::Metadata] message metadata object
       def initialize(raw_payload, metadata)
         @raw_payload = raw_payload
         @metadata = metadata
@@ -33,21 +33,16 @@ module Karafka
         @payload
       end
 
-      # @return [Boolean] did given params payload were deserialized already
+      # @return [Boolean] did we deserialize payload already
       def deserialized?
         @deserialized
       end
 
       private
 
-      # @return [Object] tries de-serializes data
+      # @return [Object] deserialized data
       def deserialize
-        Karafka.monitor.instrument('params.params.deserialize', caller: self) do
-          metadata.deserializer.call(self)
-        end
-      rescue ::StandardError => e
-        Karafka.monitor.instrument('params.params.deserialize.error', caller: self, error: e)
-        raise e
+        metadata.deserializer.call(self)
       end
     end
   end

data/lib/karafka/messages/messages.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Messages
+    # Messages batch represents a set of messages received from Kafka of a single topic partition.
+    class Messages
+      include Enumerable
+
+      attr_reader :metadata
+
+      # @param messages_array [Array<Karafka::Messages::Message>] array with karafka messages
+      # @param metadata [Karafka::Messages::BatchMetadata]
+      # @return [Karafka::Messages::Messages] lazy evaluated messages batch object
+      def initialize(messages_array, metadata)
+        @messages_array = messages_array
+        @metadata = metadata
+      end
+
+      # @param block [Proc] block we want to execute per each message
+      # @note Invocation of this method will not cause loading and deserializing of messages.
+      def each(&block)
+        @messages_array.each(&block)
+      end
+
+      # Runs deserialization of all the messages and returns them
+      # @return [Array<Karafka::Messages::Message>]
+      def deserialize!
+        each(&:payload)
+      end
+
+      # @return [Array<Object>] array with deserialized payloads. This method can be useful when
+      #   we don't care about metadata and just want to extract all the data payloads from the
+      #   batch
+      def payloads
+        map(&:payload)
+      end
+
+      # @return [Array<String>] array with raw, not deserialized payloads
+      def raw_payloads
+        map(&:raw_payload)
+      end
+
+      # @return [Karafka::Messages::Message] first message
+      def first
+        @messages_array.first
+      end
+
+      # @return [Karafka::Messages::Message] last message
+      def last
+        @messages_array.last
+      end
+
+      # @return [Integer] number of messages in the batch
+      def size
+        @messages_array.size
+      end
+
+      # @return [Array<Karafka::Messages::Message>] pure array with messages
+      def to_a
+        @messages_array
+      end
+    end
+  end
+end

data/lib/karafka/{params → messages}/metadata.rb

@@ -1,18 +1,16 @@
 # frozen_string_literal: true
 
 module Karafka
-  module Params
-    # Single message / params metadata details that can be accessed without the need for the
-    # payload deserialization
+  module Messages
+    # Single message metadata details that can be accessed without the need of deserialization.
     Metadata = Struct.new(
-      :create_time,
+      :timestamp,
       :headers,
-      :is_control_record,
       :key,
       :offset,
       :deserializer,
       :partition,
-      :receive_time,
+      :received_at,
       :topic,
       keyword_init: true
     )

data/lib/karafka/messages/seek.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Messages
+    # "Fake" message that we use as an abstraction layer when seeking back.
+    # This allows us to encapsulate a seek with a simple abstraction
+    Seek = Struct.new(:topic, :partition, :offset)
+  end
+end

data/lib/karafka/patches/rdkafka/consumer.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Patches to external components
+  module Patches
+    # Rdkafka related patches
+    module Rdkafka
+      # Rdkafka::Consumer patches
+      module Consumer
+        # A method that allows us to get the native kafka producer name
+        # @return [String] producer instance name
+        # @note We need this to make sure that we allocate proper dispatched events only to
+        #   callback listeners that should publish them
+        def name
+          @name ||= ::Rdkafka::Bindings.rd_kafka_name(@native_kafka)
+        end
+      end
+    end
+  end
+end
+
+::Rdkafka::Consumer.include ::Karafka::Patches::Rdkafka::Consumer