waterdrop 2.0.7 → 2.6.11

data/lib/{water_drop → waterdrop}/config.rb

@@ -5,11 +5,14 @@
 module WaterDrop
   # Configuration object for setting up all options required by WaterDrop
   class Config
-    include Dry::Configurable
+    include ::Karafka::Core::Configurable
 
     # Defaults for kafka settings, that will be overwritten only if not present already
     KAFKA_DEFAULTS = {
-      'client.id' => 'waterdrop'
+      'client.id': 'waterdrop',
+      # emit librdkafka statistics every five seconds. This is used in instrumentation.
+      # When disabled, part of metrics will not be published and available.
+      'statistics.interval.ms': 5_000
     }.freeze
 
     private_constant :KAFKA_DEFAULTS
@@ -22,7 +25,7 @@ module WaterDrop
     setting(
       :id,
       default: false,
-      constructor: ->(id) { id || SecureRandom.uuid }
+      constructor: ->(id) { id || SecureRandom.hex(6) }
     )
     # option [Instance] logger that we want to use
     # @note Due to how rdkafka works, this setting is global for all the producers
@@ -47,13 +50,41 @@ module WaterDrop
     #   delivery report. In a really robust systems, this describes the min-delivery time
     #   for a single sync message when produced in isolation
     setting :wait_timeout, default: 0.005 # 5 milliseconds
+    # option [Boolean] should we upon detecting full librdkafka queue backoff and retry or should
+    #   we raise an exception.
+    #   When this is set to `true`, upon full queue, we won't raise an error. There will be error
+    #   in the `error.occurred` notification pipeline with a proper type as while this is
+    #   recoverable, in a high number it still may mean issues.
+    #   Waiting is one of the recommended strategies.
+    setting :wait_on_queue_full, default: true
+    # option [Integer] how long (in seconds) should we backoff before a retry when queue is full
+    #   The retry will happen with the same message and backoff should give us some time to
+    #   dispatch previously buffered messages.
+    setting :wait_backoff_on_queue_full, default: 0.1
+    # option [Numeric] how many seconds should we wait with the backoff on queue having space for
+    # more messages before re-raising the error.
+    setting :wait_timeout_on_queue_full, default: 10
+    # option [Numeric] How long to wait before retrying a retryable transaction related error
+    setting :wait_backoff_on_transaction_command, default: 0.5
+    # option [Numeric] How many times to retry a retryable transaction related error before
+    #   giving up
+    setting :max_attempts_on_transaction_command, default: 5
+
     # option [Boolean] should we send messages. Setting this to false can be really useful when
     #   testing and or developing because when set to false, won't actually ping Kafka but will
     #   run all the validations, etc
     setting :deliver, default: true
+    # option [Class] class for usage when creating the underlying client used to dispatch messages
+    setting :client_class, default: Clients::Rdkafka
     # rdkafka options
     # @see https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
     setting :kafka, default: {}
+    # Middleware chain that can be expanded with useful middleware steps
+    setting(
+      :middleware,
+      default: false,
+      constructor: ->(middleware) { middleware || WaterDrop::Middleware.new }
+    )
 
     # Configuration method
     # @yield Runs a block of code providing a config singleton instance to it
@@ -63,10 +94,13 @@ module WaterDrop
         yield(config)
 
         merge_kafka_defaults!(config)
-        validate!(config.to_h)
+
+        Contracts::Config.new.validate!(config.to_h, Errors::ConfigurationInvalidError)
 
         ::Rdkafka::Config.logger = config.logger
       end
+
+      self
     end
 
     private
@@ -74,7 +108,7 @@ module WaterDrop
     # Propagates the kafka setting defaults unless they are already present
     # This makes it easier to set some values that users usually don't change but still allows them
     # to overwrite the whole hash if they want to
-    # @param config [Dry::Configurable::Config] dry config of this producer
+    # @param config [Karafka::Core::Configurable::Node] config of this producer
     def merge_kafka_defaults!(config)
       KAFKA_DEFAULTS.each do |key, value|
         next if config.kafka.key?(key)
@@ -82,16 +116,5 @@ module WaterDrop
         config.kafka[key] = value
       end
     end
-
-    # Validates the configuration and if anything is wrong, will raise an exception
-    # @param config_hash [Hash] config hash with setup details
-    # @raise [WaterDrop::Errors::ConfigurationInvalidError] raised when something is wrong with
-    #   the configuration
-    def validate!(config_hash)
-      result = Contracts::Config.new.call(config_hash)
-      return true if result.success?
-
-      raise Errors::ConfigurationInvalidError, result.errors.to_h
-    end
   end
 end

data/lib/waterdrop/contracts/config.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Contracts
+    # Contract with validation rules for WaterDrop configuration details
+    class Config < ::Karafka::Core::Contractable::Contract
+      configure do |config|
+        config.error_messages = YAML.safe_load(
+          File.read(
+            File.join(WaterDrop.gem_root, 'config', 'locales', 'errors.yml')
+          )
+        ).fetch('en').fetch('validations').fetch('config')
+      end
+
+      required(:id) { |val| val.is_a?(String) && !val.empty? }
+      required(:logger) { |val| !val.nil? }
+      required(:deliver) { |val| [true, false].include?(val) }
+      required(:max_payload_size) { |val| val.is_a?(Integer) && val >= 1 }
+      required(:max_wait_timeout) { |val| val.is_a?(Numeric) && val >= 0 }
+      required(:wait_timeout) { |val| val.is_a?(Numeric) && val.positive? }
+      required(:kafka) { |val| val.is_a?(Hash) && !val.empty? }
+      required(:wait_on_queue_full) { |val| [true, false].include?(val) }
+      required(:wait_backoff_on_queue_full) { |val| val.is_a?(Numeric) && val >= 0 }
+      required(:wait_timeout_on_queue_full) { |val| val.is_a?(Numeric) && val >= 0 }
+
+      # rdkafka allows both symbols and strings as keys for config but then casts them to strings
+      # This can be confusing, so we expect all keys to be symbolized
+      virtual do |config, errors|
+        next true unless errors.empty?
+
+        errors = []
+
+        config
+          .fetch(:kafka)
+          .keys
+          .reject { |key| key.is_a?(Symbol) }
+          .each { |key| errors << [[:kafka, key], :kafka_key_must_be_a_symbol] }
+
+        errors
+      end
+    end
+  end
+end

data/lib/waterdrop/contracts/message.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Contracts
+    # Contract with validation rules for validating that all the message options that
+    # we provide to producer are valid and usable
+    class Message < ::Karafka::Core::Contractable::Contract
+      configure do |config|
+        config.error_messages = YAML.safe_load(
+          File.read(
+            File.join(WaterDrop.gem_root, 'config', 'locales', 'errors.yml')
+          )
+        ).fetch('en').fetch('validations').fetch('message')
+      end
+
+      # Regex to check that topic has a valid format
+      TOPIC_REGEXP = /\A(\w|-|\.)+\z/
+
+      private_constant :TOPIC_REGEXP
+
+      attr_reader :max_payload_size
+
+      # @param max_payload_size [Integer] max payload size
+      def initialize(max_payload_size:)
+        super()
+        @max_payload_size = max_payload_size
+      end
+
+      required(:topic) do |val|
+        (val.is_a?(String) || val.is_a?(Symbol)) && TOPIC_REGEXP.match?(val.to_s)
+      end
+
+      required(:payload) { |val| val.nil? || val.is_a?(String) }
+      optional(:key) { |val| val.nil? || (val.is_a?(String) && !val.empty?) }
+      optional(:partition) { |val| val.is_a?(Integer) && val >= -1 }
+      optional(:partition_key) { |val| val.nil? || (val.is_a?(String) && !val.empty?) }
+      optional(:timestamp) { |val| val.nil? || (val.is_a?(Time) || val.is_a?(Integer)) }
+      optional(:headers) { |val| val.nil? || val.is_a?(Hash) }
+
+      virtual do |message, errors|
+        next true unless errors.empty?
+        next true unless message.key?(:headers)
+        next true if message[:headers].nil?
+
+        errors = []
+
+        message.fetch(:headers).each do |key, value|
+          errors << [%i[headers], :invalid_key_type] unless key.is_a?(String)
+          errors << [%i[headers], :invalid_value_type] unless value.is_a?(String)
+        end
+
+        errors
+      end
+
+      virtual do |message, errors, validator|
+        next true unless errors.empty?
+        next if message[:payload].nil? # tombstone payload
+        next true if message[:payload].bytesize <= validator.max_payload_size
+
+        [[%i[payload], :max_size]]
+      end
+    end
+  end
+end

data/lib/{water_drop → waterdrop}/errors.rb

@@ -29,15 +29,22 @@ module WaterDrop
     # contact us as it is an error.
     StatusInvalidError = Class.new(BaseError)
 
-    # Raised when during messages flushing something bad happened
-    class FlushFailureError < BaseError
-      attr_reader :dispatched_messages
+    # Raised when there is an inline error during single message produce operations
+    ProduceError = Class.new(BaseError)
 
-      # @param dispatched_messages [Array<Rdkafka::Producer::DeliveryHandle>] handlers of the
+    # Raise it within a transaction to abort it
+    AbortTransaction = Class.new(BaseError)
+
+    # Raised when during messages producing something bad happened inline
+    class ProduceManyError < ProduceError
+      attr_reader :dispatched
+
+      # @param dispatched [Array<Rdkafka::Producer::DeliveryHandle>] handlers of the
       #   messages that we've dispatched
-      def initialize(dispatched_messages)
-        super()
-        @dispatched_messages = dispatched_messages
+      # @param message [String] error message
+      def initialize(dispatched, message)
+        super(message)
+        @dispatched = dispatched
       end
     end
   end

data/lib/waterdrop/instrumentation/callbacks/delivery.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Instrumentation
+    module Callbacks
+      # Creates a callable that we want to run upon each message delivery or failure
+      #
+      # @note We don't have to provide client_name here as this callback is per client instance
+      #
+      # @note We do not consider `message.purge` as an error for transactional producers, because
+      #   this is a standard behaviour for not yet dispatched messages on aborted transactions.
+      #   We do however still want to instrument it for traceability.
+      class Delivery
+        # Error emitted when a message was not yet dispatched and was purged from the queue
+        RD_KAFKA_RESP_PURGE_QUEUE = -152
+
+        # Error emitted when a message was purged while it was dispatched
+        RD_KAFKA_RESP_PURGE_INFLIGHT = -151
+
+        # Errors related to queue purging that is expected in transactions
+        PURGE_ERRORS = [RD_KAFKA_RESP_PURGE_INFLIGHT, RD_KAFKA_RESP_PURGE_QUEUE].freeze
+
+        private_constant :RD_KAFKA_RESP_PURGE_QUEUE, :RD_KAFKA_RESP_PURGE_INFLIGHT, :PURGE_ERRORS
+
+        # @param producer_id [String] id of the current producer
+        # @param transactional [Boolean] is this handle for a transactional or regular producer
+        # @param monitor [WaterDrop::Instrumentation::Monitor] monitor we are using
+        def initialize(producer_id, transactional, monitor)
+          @producer_id = producer_id
+          @transactional = transactional
+          @monitor = monitor
+        end
+
+        # Emits delivery details to the monitor
+        # @param delivery_report [Rdkafka::Producer::DeliveryReport] delivery report
+        def call(delivery_report)
+          error_code = delivery_report.error.to_i
+
+          if error_code.zero?
+            instrument_acknowledged(delivery_report)
+
+          elsif @transactional && PURGE_ERRORS.include?(error_code)
+            instrument_purged(delivery_report)
+          else
+            instrument_error(delivery_report)
+          end
+        end
+
+        private
+
+        # @param delivery_report [Rdkafka::Producer::DeliveryReport] delivery report
+        def instrument_acknowledged(delivery_report)
+          @monitor.instrument(
+            'message.acknowledged',
+            caller: self,
+            producer_id: @producer_id,
+            offset: delivery_report.offset,
+            partition: delivery_report.partition,
+            topic: delivery_report.topic_name,
+            delivery_report: delivery_report
+          )
+        end
+
+        # @param delivery_report [Rdkafka::Producer::DeliveryReport] delivery report
+        def instrument_purged(delivery_report)
+          @monitor.instrument(
+            'message.purged',
+            caller: self,
+            error: build_error(delivery_report),
+            producer_id: @producer_id,
+            offset: delivery_report.offset,
+            partition: delivery_report.partition,
+            topic: delivery_report.topic_name,
+            delivery_report: delivery_report
+          )
+        end
+
+        # @param delivery_report [Rdkafka::Producer::DeliveryReport] delivery report
+        def instrument_error(delivery_report)
+          @monitor.instrument(
+            'error.occurred',
+            caller: self,
+            error: build_error(delivery_report),
+            producer_id: @producer_id,
+            offset: delivery_report.offset,
+            partition: delivery_report.partition,
+            topic: delivery_report.topic_name,
+            delivery_report: delivery_report,
+            type: 'librdkafka.dispatch_error'
+          )
+        end
+
+        # Builds appropriate rdkafka error
+        # @param delivery_report [Rdkafka::Producer::DeliveryReport] delivery report
+        # @return [::Rdkafka::RdkafkaError]
+        def build_error(delivery_report)
+          ::Rdkafka::RdkafkaError.new(delivery_report.error)
+        end
+      end
+    end
+  end
+end

data/lib/{water_drop → waterdrop}/instrumentation/callbacks/error.rb

@@ -18,15 +18,19 @@ module WaterDrop
         # @param client_name [String] rdkafka client name
         # @param error [Rdkafka::Error] error that occurred
         # @note It will only instrument on errors of the client of our producer
+        # @note When there is a particular message produce error (not internal error), the error
+        #   is shipped via the delivery callback, not via error callback.
         def call(client_name, error)
           # Emit only errors related to our client
           # Same as with statistics (mor explanation there)
           return unless @client_name == client_name
 
           @monitor.instrument(
-            'error.emitted',
+            'error.occurred',
+            caller: self,
+            error: error,
             producer_id: @producer_id,
-            error: error
+            type: 'librdkafka.error'
           )
         end
       end

data/lib/{water_drop → waterdrop}/instrumentation/callbacks/statistics.rb

@@ -17,7 +17,7 @@ module WaterDrop
           @producer_id = producer_id
           @client_name = client_name
           @monitor = monitor
-          @statistics_decorator = StatisticsDecorator.new
+          @statistics_decorator = ::Karafka::Core::Monitoring::StatisticsDecorator.new
         end
 
         # Emits decorated statistics to the monitor

data/lib/{water_drop/instrumentation/stdout_listener.rb → waterdrop/instrumentation/logger_listener.rb}

@@ -1,15 +1,25 @@
 # frozen_string_literal: true
 
 module WaterDrop
+  # WaterDrop instrumentation related module
   module Instrumentation
     # Default listener that hooks up to our instrumentation and uses its events for logging
     # It can be removed/replaced or anything without any harm to the Waterdrop flow
     # @note It is a module as we can use it then as a part of the Karafka framework listener
     #   as well as we can use it standalone
-    class StdoutListener
-      # @param logger [Object] stdout logger we want to use
-      def initialize(logger)
+    class LoggerListener
+      # @param logger [Object] logger we want to use
+      # @param log_messages [Boolean] Should we report the messages content (payload and metadata)
+      #   with each message operation.
+      #
+      #   This can be extensive, especially when producing a lot of messages. We provide this
+      #   despite the fact that we only report payloads in debug, because Rails by default operates
+      #   with debug level. This means, that when working with Rails in development, every single
+      #   payload dispatched will go to logs. In majority of the cases this is extensive and simply
+      #   floods the end user.
+      def initialize(logger, log_messages: true)
         @logger = logger
+        @log_messages = log_messages
       end
 
       # @param event [Dry::Events::Event] event that happened with the details
@@ -17,6 +27,9 @@ module WaterDrop
         message = event[:message]
 
         info(event, "Async producing of a message to '#{message[:topic]}' topic")
+
+        return unless log_messages?
+
         debug(event, message)
       end
 
@@ -25,6 +38,9 @@ module WaterDrop
         message = event[:message]
 
         info(event, "Sync producing of a message to '#{message[:topic]}' topic")
+
+        return unless log_messages?
+
         debug(event, message)
       end
 
@@ -34,6 +50,9 @@ module WaterDrop
         topics_count = messages.map { |message| "'#{message[:topic]}'" }.uniq.count
 
         info(event, "Async producing of #{messages.size} messages to #{topics_count} topics")
+
+        return unless log_messages?
+
         debug(event, messages)
       end
 
@@ -43,6 +62,9 @@ module WaterDrop
         topics_count = messages.map { |message| "'#{message[:topic]}'" }.uniq.count
 
         info(event, "Sync producing of #{messages.size} messages to #{topics_count} topics")
+
+        return unless log_messages?
+
         debug(event, messages)
       end
 
@@ -51,6 +73,9 @@ module WaterDrop
         message = event[:message]
 
         info(event, "Buffering of a message to '#{message[:topic]}' topic")
+
+        return unless log_messages?
+
         debug(event, [message])
       end
 
@@ -59,6 +84,9 @@ module WaterDrop
         messages = event[:messages]
 
         info(event, "Buffering of #{messages.size} messages")
+
+        return unless log_messages?
+
         debug(event, [messages, messages.size])
       end
 
@@ -67,15 +95,9 @@ module WaterDrop
         messages = event[:messages]
 
         info(event, "Async flushing of #{messages.size} messages from the buffer")
-        debug(event, messages)
-      end
 
-      # @param event [Dry::Events::Event] event that happened with the details
-      def on_buffer_flushed_async_error(event)
-        messages = event[:messages]
-        error = event[:error]
+        return unless log_messages?
 
-        error(event, "Async flushing of #{messages.size} failed due to: #{error}")
         debug(event, messages)
       end
 
@@ -84,34 +106,57 @@ module WaterDrop
         messages = event[:messages]
 
         info(event, "Sync flushing of #{messages.size} messages from the buffer")
+
+        return unless log_messages?
+
         debug(event, messages)
       end
 
       # @param event [Dry::Events::Event] event that happened with the details
-      def on_buffer_flushed_sync_error(event)
-        messages = event[:dispatched]
-        error = event[:error]
-
-        error(event, "Sync flushing of #{messages.size} failed due to: #{error}")
-        debug(event, messages)
+      def on_buffer_purged(event)
+        info(event, 'Successfully purging buffer')
       end
 
       # @param event [Dry::Events::Event] event that happened with the details
       def on_producer_closed(event)
-        info event, 'Closing producer'
-        debug event, ''
+        info(event, 'Closing producer')
       end
 
       # @param event [Dry::Events::Event] event that happened with the error details
-      def on_error_emitted(event)
+      def on_error_occurred(event)
         error = event[:error]
+        type = event[:type]
+
+        error(event, "Error occurred: #{error} - #{type}")
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_transaction_started(event)
+        info(event, 'Starting transaction')
+      end
 
-        error(event, "Background thread error emitted: #{error}")
-        debug(event, '')
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_transaction_aborted(event)
+        info(event, 'Aborting transaction')
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_transaction_committed(event)
+        info(event, 'Committing transaction')
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_transaction_finished(event)
+        info(event, 'Processing transaction')
       end
 
       private
 
+      # @return [Boolean] should we report the messages details in the debug mode.
+      def log_messages?
+        @log_messages
+      end
+
       # @param event [Dry::Events::Event] event that happened with the details
       # @param log_message [String] message we want to publish
       def debug(event, log_message)

data/lib/waterdrop/instrumentation/monitor.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Instrumentation
+    # WaterDrop instrumentation monitor that we use to publish events
+    # By default uses our internal notifications bus but can be used with
+    # `ActiveSupport::Notifications` as well
+    class Monitor < ::Karafka::Core::Monitoring::Monitor
+      # @param notifications_bus [Object] either our internal notifications bus or
+      #   `ActiveSupport::Notifications`
+      # @param namespace [String, nil] namespace for events or nil if no namespace
+      def initialize(
+        notifications_bus = WaterDrop::Instrumentation::Notifications.new,
+        namespace = nil
+      )
+        super(notifications_bus, namespace)
+      end
+    end
+  end
+end

data/lib/{water_drop/instrumentation/monitor.rb → waterdrop/instrumentation/notifications.rb}

@@ -2,13 +2,8 @@
 
 module WaterDrop
   module Instrumentation
-    # Monitor is used to hookup external monitoring services to monitor how WaterDrop works
-    # Since it is a pub-sub based on dry-monitor, you can use as many subscribers/loggers at the
-    # same time, which means that you might have for example file logging and NewRelic at the same
-    # time
-    # @note This class acts as a singleton because we are only permitted to have single monitor
-    #   per running process (just as logger)
-    class Monitor < Dry::Monitor::Notifications
+    # Instrumented is used to hookup external monitoring services to monitor how WaterDrop works
+    class Notifications < ::Karafka::Core::Monitoring::Notifications
       # List of events that we support in the system and to which a monitor client can hook up
       # @note The non-error once support timestamp benchmarking
       EVENTS = %w[
@@ -17,28 +12,31 @@ module WaterDrop
         message.produced_async
         message.produced_sync
         message.acknowledged
+        message.purged
         message.buffered
 
         messages.produced_async
         messages.produced_sync
         messages.buffered
 
+        transaction.started
+        transaction.committed
+        transaction.aborted
+        transaction.finished
+
         buffer.flushed_async
-        buffer.flushed_async.error
         buffer.flushed_sync
-        buffer.flushed_sync.error
+        buffer.purged
 
         statistics.emitted
 
-        error.emitted
+        error.occurred
       ].freeze
 
-      private_constant :EVENTS
-
       # @return [WaterDrop::Instrumentation::Monitor] monitor instance for system instrumentation
       def initialize
-        super(:waterdrop)
-        EVENTS.each(&method(:register_event))
+        super
+        EVENTS.each { |event| register_event(event) }
       end
     end
   end

data/lib/waterdrop/instrumentation/vendors/datadog/dashboard.json

@@ -0,0 +1 @@
+{"title":"WaterDrop producer example dashboard","description":"This dashboard include example setup for monitoring activity of your WaterDrop producer","widgets":[{"id":243951318,"definition":{"title":"Messages produced","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"produced sync","formula":"query1"},{"alias":"produced async","formula":"query2"},{"alias":"flushed sync","formula":"query3"},{"alias":"flushed async","formula":"query4"},{"alias":"acknowledged","formula":"query5"}],"response_format":"timeseries","queries":[{"query":"sum:waterdrop.produced_sync{*}.as_count()","data_source":"metrics","name":"query1"},{"query":"sum:waterdrop.produced_async{*}.as_count()","data_source":"metrics","name":"query2"},{"query":"sum:waterdrop.flushed_sync{*}.as_count()","data_source":"metrics","name":"query3"},{"query":"sum:waterdrop.flushed_async{*}.as_count()","data_source":"metrics","name":"query4"},{"query":"sum:waterdrop.acknowledged{*}.as_count()","data_source":"metrics","name":"query5"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"include_zero":true,"scale":"linear","label":"","min":"auto","max":"auto"}}},{"id":1979626566852990,"definition":{"title":"Messages buffer size","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"max","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"avg:waterdrop.buffer.size.max{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]}},{"id":243951221,"definition":{"title":"Kafka broker API calls","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"API calls","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"sum:waterdrop.calls{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"include_zero":true,"scale":"linear","label":"","min":"auto","max":"auto"}}},{"id":243951952,"definition":{"title":"Producer queue size","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Queue size average","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"max:waterdrop.queue.size.avg{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"},{"formulas":[{"alias":"Queue size max","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"max:waterdrop.queue.size.max{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"include_zero":true,"scale":"linear","label":"","min":"auto","max":"auto"}}},{"id":243951263,"definition":{"title":"Producer queue latency","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Average latency","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"avg:waterdrop.queue.latency.avg{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"},{"formulas":[{"alias":"Latency p95","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"avg:waterdrop.queue.latency.p95{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"},{"formulas":[{"alias":"Latency p99","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"avg:waterdrop.queue.latency.p99{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"include_zero":true,"scale":"linear","label":"","min":"auto","max":"auto"}}},{"id":243951276,"definition":{"title":"Producer network latency","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Average latency","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"avg:waterdrop.request_size.avg{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"},{"formulas":[{"alias":"Latency p95","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"avg:waterdrop.network.latency.p95{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"},{"formulas":[{"alias":"Latency p99","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"avg:waterdrop.network.latency.p99{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"include_zero":true,"scale":"linear","label":"","min":"auto","max":"auto"}}},{"id":243954928,"definition":{"title":"Producer errors","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"}],"response_format":"timeseries","queries":[{"query":"sum:waterdrop.error_occurred{*}.as_count()","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"include_zero":true,"scale":"linear","label":"","min":"auto","max":"auto"}}}],"template_variables":[],"layout_type":"ordered","is_read_only":false,"notify_list":[],"reflow_type":"auto","id":"rnr-kgh-dna"}