karafka 1.4.12 → 2.2.10

data/lib/karafka/instrumentation/vendors/appsignal/dashboard.json

@@ -0,0 +1,222 @@
+{
+  "title": "Karafka",
+  "description": "This dashboard gives an overview of the overall Karafka status.\nCheckout topics lag, throughput/performance per consumer,\nprocess/worker counts, and many other things.\n",
+  "visuals": [
+    {
+      "title": "Consumed messages",
+      "description": "",
+      "line_label": "%topic%[%consumer_group%]",
+      "display": "LINE",
+      "format": "number",
+      "draw_null_as_zero": true,
+      "metrics": [
+        {
+          "name": "karafka_consumer_messages",
+          "fields": [
+            {
+              "field": "COUNTER"
+            }
+          ],
+          "tags": [
+            {
+              "key": "consumer_group",
+              "value": "*"
+            },
+            {
+              "key": "topic",
+              "value": "*"
+            }
+          ]
+        }
+      ],
+      "type": "timeseries"
+    },
+    {
+      "title": "Consumed batches",
+      "line_label": "%topic%[%consumer_group%]",
+      "display": "LINE",
+      "format": "number",
+      "draw_null_as_zero": true,
+      "metrics": [
+        {
+          "name": "karafka_consumer_batches",
+          "fields": [
+            {
+              "field": "COUNTER"
+            }
+          ],
+          "tags": [
+            {
+              "key": "consumer_group",
+              "value": "*"
+            },
+            {
+              "key": "topic",
+              "value": "*"
+            }
+          ]
+        }
+      ],
+      "type": "timeseries"
+    },
+    {
+      "title": "Lags",
+      "line_label": "%topic%[%consumer_group%]",
+      "display": "LINE",
+      "format": "number",
+      "draw_null_as_zero": true,
+      "metrics": [
+        {
+          "name": "karafka_consumer_aggregated_lag",
+          "fields": [
+            {
+              "field": "GAUGE"
+            }
+          ],
+          "tags": [
+            {
+              "key": "consumer_group",
+              "value": "*"
+            },
+            {
+              "key": "topic",
+              "value": "*"
+            }
+          ]
+        }
+      ],
+      "type": "timeseries"
+    },
+    {
+      "title": "Errors and DLQ",
+      "description": "",
+      "line_label": "%name% %topic%[%consumer_group%]",
+      "display": "LINE",
+      "format": "number",
+      "draw_null_as_zero": true,
+      "metrics": [
+        {
+          "name": "karafka_consumer_errors",
+          "fields": [
+            {
+              "field": "COUNTER"
+            }
+          ],
+          "tags": [
+            {
+              "key": "consumer_group",
+              "value": "*"
+            },
+            {
+              "key": "topic",
+              "value": "*"
+            }
+          ]
+        },
+        {
+          "name": "karafka_consumer_dead",
+          "fields": [
+            {
+              "field": "COUNTER"
+            }
+          ],
+          "tags": [
+            {
+              "key": "consumer_group",
+              "value": "*"
+            },
+            {
+              "key": "topic",
+              "value": "*"
+            }
+          ]
+        }
+      ],
+      "type": "timeseries"
+    },
+    {
+      "title": "Connection stability",
+      "description": "",
+      "line_label": "%name%",
+      "display": "LINE",
+      "format": "number",
+      "draw_null_as_zero": true,
+      "metrics": [
+        {
+          "name": "karafka_connection_disconnects",
+          "fields": [
+            {
+              "field": "COUNTER"
+            }
+          ],
+          "tags": []
+        },
+        {
+          "name": "karafka_connection_connects",
+          "fields": [
+            {
+              "field": "COUNTER"
+            }
+          ],
+          "tags": []
+        },
+        {
+          "name": "karafka_requests_retries",
+          "fields": [
+            {
+              "field": "COUNTER"
+            }
+          ],
+          "tags": []
+        },
+        {
+          "name": "karafka_transmission_errors",
+          "fields": [
+            {
+              "field": "COUNTER"
+            }
+          ],
+          "tags": []
+        },
+        {
+          "name": "karafka_receive_errors",
+          "fields": [
+            {
+              "field": "COUNTER"
+            }
+          ],
+          "tags": []
+        }
+      ],
+      "type": "timeseries"
+    },
+    {
+      "title": "Concurrency",
+      "line_label": "%name%",
+      "display": "LINE",
+      "format": "number",
+      "draw_null_as_zero": true,
+      "metrics": [
+        {
+          "name": "karafka_processes_count",
+          "fields": [
+            {
+              "field": "COUNTER"
+            }
+          ],
+          "tags": []
+        },
+        {
+          "name": "karafka_threads_count",
+          "fields": [
+            {
+              "field": "COUNTER"
+            }
+          ],
+          "tags": []
+        }
+      ],
+      "type": "timeseries"
+    }
+  ]
+}

data/lib/karafka/instrumentation/vendors/appsignal/errors_listener.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+
+module Karafka
+  module Instrumentation
+    module Vendors
+      # Namespace for Appsignal instrumentation
+      module Appsignal
+        # Listener for reporting errors from both consumers and producers
+        # Since we have the same API for WaterDrop and Karafka, we can use one listener with
+        # independent instances
+        class ErrorsListener < Base
+          def_delegators :config, :client
+
+          setting :client, default: Client.new
+
+          configure
+
+          # Sends error details to Appsignal
+          #
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_error_occurred(event)
+            client.send_error(event[:error])
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb

@@ -0,0 +1,331 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+
+module Karafka
+  module Instrumentation
+    module Vendors
+      # Namespace for Appsignal instrumentation
+      module Appsignal
+        # Listener that ships metrics to Appsignal
+        class MetricsListener < Base
+          def_delegators :config, :client, :rd_kafka_metrics, :namespace
+
+          # Value object for storing a single rdkafka metric publishing details
+          RdKafkaMetric = Struct.new(:type, :scope, :name, :key_location)
+
+          setting :namespace, default: 'karafka'
+
+          setting :client, default: Client.new
+
+          setting :rd_kafka_metrics, default: [
+            # Broker metrics
+            RdKafkaMetric.new(:count, :brokers, 'requests_retries', 'txretries_d'),
+            RdKafkaMetric.new(:count, :brokers, 'transmission_errors', 'txerrs_d'),
+            RdKafkaMetric.new(:count, :brokers, 'receive_errors', 'rxerrs_d'),
+            RdKafkaMetric.new(:count, :brokers, 'connection_connects', 'connects_d'),
+            RdKafkaMetric.new(:count, :brokers, 'connection_disconnects', 'disconnects_d'),
+            RdKafkaMetric.new(:gauge, :brokers, 'network_latency_avg', %w[rtt avg]),
+            RdKafkaMetric.new(:gauge, :brokers, 'network_latency_p95', %w[rtt p95]),
+            RdKafkaMetric.new(:gauge, :brokers, 'network_latency_p99', %w[rtt p99]),
+
+            # Topics partitions metrics
+            RdKafkaMetric.new(:gauge, :topics, 'consumer_lag', 'consumer_lag_stored'),
+            RdKafkaMetric.new(:gauge, :topics, 'consumer_lag_delta', 'consumer_lag_stored_d')
+          ].freeze
+
+          # Metrics that sum values on topics levels and not on partition levels
+          setting :aggregated_rd_kafka_metrics, default: [
+            # Topic aggregated metrics
+            RdKafkaMetric.new(:gauge, :topics, 'consumer_aggregated_lag', 'consumer_lag_stored')
+          ].freeze
+
+          configure
+
+          # Before each consumption process, lets start a transaction associated with it
+          # We also set some basic metadata about the given consumption that can be useful for
+          # debugging
+          #
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_consumer_consume(event)
+            consumer = event.payload[:caller]
+
+            start_transaction(consumer, 'consume')
+
+            client.metadata = {
+              batch_size: consumer.messages.size,
+              first_offset: consumer.messages.metadata.first_offset,
+              last_offset: consumer.messages.metadata.last_offset,
+              consumer_group: consumer.topic.consumer_group.id,
+              topic: consumer.topic.name,
+              partition: consumer.partition,
+              attempt: consumer.coordinator.pause_tracker.attempt
+            }
+          end
+
+          # Once we're done with consumption, we bump counters about that
+          #
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_consumer_consumed(event)
+            consumer = event.payload[:caller]
+            messages = consumer.messages
+            metadata = messages.metadata
+
+            with_multiple_resolutions(consumer) do |tags|
+              count('consumer_messages', messages.size, tags)
+              count('consumer_batches', 1, tags)
+              gauge('consumer_offsets', metadata.last_offset, tags)
+            end
+
+            stop_transaction
+          end
+
+          # Register minute based probe only on app running. Otherwise if we would always register
+          # minute probe, it would report on processes using Karafka but not running the
+          # consumption process
+          #
+          # @param _event [Karafka::Core::Monitoring::Event]
+          def on_app_running(_event)
+            return if @probe_registered
+
+            @probe_registered = true
+
+            # Registers the minutely probe for one-every-minute metrics
+            client.register_probe(:karafka, -> { minute_probe })
+          end
+
+          # Keeps track of revocation user code execution
+          #
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_consumer_revoke(event)
+            consumer = event.payload[:caller]
+            start_transaction(consumer, 'revoked')
+          end
+
+          # Finishes the revocation transaction
+          #
+          # @param _event [Karafka::Core::Monitoring::Event]
+          def on_consumer_revoked(_event)
+            stop_transaction
+          end
+
+          # Keeps track of revocation user code execution
+          #
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_consumer_shutting_down(event)
+            consumer = event.payload[:caller]
+            start_transaction(consumer, 'shutdown')
+          end
+
+          # Finishes the shutdown transaction
+          #
+          # @param _event [Karafka::Core::Monitoring::Event]
+          def on_consumer_shutdown(_event)
+            stop_transaction
+          end
+
+          # Counts DLQ dispatches
+          #
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_dead_letter_queue_dispatched(event)
+            consumer = event.payload[:caller]
+
+            with_multiple_resolutions(consumer) do |tags|
+              count('consumer_dead', 1, tags)
+            end
+          end
+
+          # Reports on **any** error that occurs. This also includes non-user related errors
+          # originating from the framework.
+          #
+          # @param event [Karafka::Core::Monitoring::Event] error event details
+          def on_error_occurred(event)
+            # If this is a user consumption related error, we bump the counters for metrics
+            if event[:type] == 'consumer.consume.error'
+              consumer = event.payload[:caller]
+
+              with_multiple_resolutions(consumer) do |tags|
+                count('consumer_errors', 1, tags)
+              end
+            end
+
+            stop_transaction
+          end
+
+          # Hooks up to Karafka instrumentation for emitted statistics
+          #
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_statistics_emitted(event)
+            statistics = event[:statistics]
+            consumer_group_id = event[:consumer_group_id]
+
+            rd_kafka_metrics.each do |metric|
+              report_metric(metric, statistics, consumer_group_id)
+            end
+
+            report_aggregated_topics_metrics(statistics, consumer_group_id)
+          end
+
+          # Reports a given metric statistics to Appsignal
+          # @param metric [RdKafkaMetric] metric value object
+          # @param statistics [Hash] hash with all the statistics emitted
+          # @param consumer_group_id [String] cg in context which we operate
+          def report_metric(metric, statistics, consumer_group_id)
+            case metric.scope
+            when :root
+              # Do nothing on the root metrics as the same metrics are reported in a granular
+              # way from other places
+              nil
+            when :brokers
+              statistics.fetch('brokers').each_value do |broker_statistics|
+                # Skip bootstrap nodes
+                # Bootstrap nodes have nodeid -1, other nodes have positive
+                # node ids
+                next if broker_statistics['nodeid'] == -1
+
+                public_send(
+                  metric.type,
+                  metric.name,
+                  broker_statistics.dig(*metric.key_location),
+                  {
+                    broker: broker_statistics['nodename']
+                  }
+                )
+              end
+            when :topics
+              statistics.fetch('topics').each do |topic_name, topic_values|
+                topic_values['partitions'].each do |partition_name, partition_statistics|
+                  next if partition_name == '-1'
+                  # Skip until lag info is available
+                  next if partition_statistics['consumer_lag'] == -1
+                  next if partition_statistics['consumer_lag_stored'] == -1
+
+                  # Skip if we do not own the fetch assignment
+                  next if partition_statistics['fetch_state'] == 'stopped'
+                  next if partition_statistics['fetch_state'] == 'none'
+
+                  public_send(
+                    metric.type,
+                    metric.name,
+                    partition_statistics.dig(*metric.key_location),
+                    {
+                      consumer_group: consumer_group_id,
+                      topic: topic_name,
+                      partition: partition_name
+                    }
+                  )
+                end
+              end
+            else
+              raise ArgumentError, metric.scope
+            end
+          end
+
+          # Publishes aggregated topic-level metrics that are sum of per partition metrics
+          #
+          # @param statistics [Hash] hash with all the statistics emitted
+          # @param consumer_group_id [String] cg in context which we operate
+          def report_aggregated_topics_metrics(statistics, consumer_group_id)
+            config.aggregated_rd_kafka_metrics.each do |metric|
+              statistics.fetch('topics').each do |topic_name, topic_values|
+                sum = 0
+
+                topic_values['partitions'].each do |partition_name, partition_statistics|
+                  next if partition_name == '-1'
+                  # Skip until lag info is available
+                  next if partition_statistics['consumer_lag'] == -1
+                  next if partition_statistics['consumer_lag_stored'] == -1
+
+                  sum += partition_statistics.dig(*metric.key_location)
+                end
+
+                public_send(
+                  metric.type,
+                  metric.name,
+                  sum,
+                  {
+                    consumer_group: consumer_group_id,
+                    topic: topic_name
+                  }
+                )
+              end
+            end
+          end
+
+          # Increments a counter with a namespace key, value and tags
+          #
+          # @param key [String] key we want to use (without the namespace)
+          # @param value [Integer] count value
+          # @param tags [Hash] additional extra tags
+          def count(key, value, tags)
+            client.count(
+              namespaced_metric(key),
+              value,
+              tags
+            )
+          end
+
+          # Sets the gauge value
+          #
+          # @param key [String] key we want to use (without the namespace)
+          # @param value [Integer] gauge value
+          # @param tags [Hash] additional extra tags
+          def gauge(key, value, tags)
+            client.gauge(
+              namespaced_metric(key),
+              value,
+              tags
+            )
+          end
+
+          private
+
+          # Wraps metric name in listener's namespace
+          # @param metric_name [String] RdKafkaMetric name
+          # @return [String]
+          def namespaced_metric(metric_name)
+            "#{namespace}_#{metric_name}"
+          end
+
+          # Starts the transaction for monitoring user code
+          #
+          # @param consumer [Karafka::BaseConsumer] karafka consumer instance
+          # @param action_name [String] lifecycle user method name
+          def start_transaction(consumer, action_name)
+            client.start_transaction(
+              "#{consumer.class}##{action_name}"
+            )
+          end
+
+          # Stops the transaction wrapping user code
+          def stop_transaction
+            client.stop_transaction
+          end
+
+          # @param consumer [Karafka::BaseConsumer] Karafka consumer instance
+          def with_multiple_resolutions(consumer)
+            topic_name = consumer.topic.name
+            consumer_group_id = consumer.topic.consumer_group.id
+            partition = consumer.partition
+
+            tags = {
+              consumer_group: consumer_group_id,
+              topic: topic_name
+            }
+
+            yield(tags)
+            yield(tags.merge(partition: partition))
+          end
+
+          # Sends minute based probing metrics
+          def minute_probe
+            concurrency = Karafka::App.config.concurrency
+
+            count('processes_count', 1, {})
+            count('threads_count', concurrency, {})
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/vendors/datadog/dashboard.json

@@ -0,0 +1 @@
+{"title":"Karafka monitoring dashboard","description":"","widgets":[{"id":5988438511387100,"definition":{"title":"Workers poll","type":"group","show_title":true,"layout_type":"ordered","widgets":[{"id":8769294644934352,"definition":{"title":"Enqueued jobs","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Enqueued jobs","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"sum:karafka.worker.enqueued_jobs.avg{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":0,"y":0,"width":4,"height":2}},{"id":2714502141463873,"definition":{"title":"Workers usage","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Busy workers (p95)","formula":"query1"},{"alias":"Total workers","formula":"query2"}],"response_format":"timeseries","queries":[{"query":"sum:karafka.worker.processing.95percentile{*}","data_source":"metrics","name":"query1"},{"query":"sum:karafka.worker.total_threads{*}","data_source":"metrics","name":"query2"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":4,"y":0,"width":4,"height":2}},{"id":5370086629441984,"definition":{"title":"Workers % utilization","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"% workers utilization","formula":"(query1 / query2) * 100"}],"response_format":"timeseries","queries":[{"query":"sum:karafka.worker.processing.95percentile{*}","data_source":"metrics","name":"query1"},{"query":"sum:karafka.worker.total_threads{*}","data_source":"metrics","name":"query2"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":8,"y":0,"width":4,"height":2}}]},"layout":{"x":0,"y":0,"width":12,"height":3}},{"id":7444969424381053,"definition":{"title":"Stability & errors","type":"group","show_title":true,"layout_type":"ordered","widgets":[{"id":8304008422587936,"definition":{"title":"Client connects and disconnects","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"time":{},"type":"timeseries","requests":[{"formulas":[{"alias":"Connects","formula":"query1"},{"alias":"Disconnects","formula":"query2"}],"queries":[{"query":"sum:karafka.connection.connects{*} by {host,consumer_group}.as_count()","data_source":"metrics","name":"query1"},{"query":"sum:karafka.connection.disconnects{*} by {host,consumer_group}.as_count()","data_source":"metrics","name":"query2"}],"response_format":"timeseries","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"bars"}]},"layout":{"x":0,"y":0,"width":4,"height":2}},{"id":3722865443336921,"definition":{"title":"Errors encountered (any)","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"time":{},"type":"timeseries","requests":[{"formulas":[{"alias":"in-karafka errors","formula":"query1"},{"alias":"librdkafka consume errors","formula":"query2"},{"alias":"librdkafka receive errors","formula":"query3"}],"queries":[{"query":"sum:karafka.error_occurred{*} by {type,consumer_group}.as_count()","data_source":"metrics","name":"query1"},{"query":"sum:karafka.consume.errors{*} by {consumer_group}.as_count()","data_source":"metrics","name":"query2"},{"query":"sum:karafka.receive.errors{*} by {consumer_group}.as_count()","data_source":"metrics","name":"query3"}],"response_format":"timeseries","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":4,"y":0,"width":4,"height":2}},{"id":5477381252952760,"definition":{"title":"Processing errors","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"time":{},"type":"timeseries","requests":[{"formulas":[{"formula":"query1"}],"queries":[{"query":"sum:karafka.error_occurred{type:consumer.consume.error} by {partition,topic,consumer_group}.as_count()","data_source":"metrics","name":"query1"}],"response_format":"timeseries","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":8,"y":0,"width":4,"height":2}},{"id":2357301680769076,"definition":{"title":"Processing errors rate per topic","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"time":{},"type":"timeseries","requests":[{"formulas":[{"alias":"% error rate per topic","formula":"(query1 / (query1 + query2)) * 100"}],"queries":[{"query":"sum:karafka.error_occurred{type:consumer.consume.error} by {topic,partition,consumer_group}.as_count()","data_source":"metrics","name":"query1"},{"query":"sum:karafka.consumer.batches{*} by {topic,partition,consumer_group}.as_count()","data_source":"metrics","name":"query2"}],"response_format":"timeseries","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"bars"}]},"layout":{"x":0,"y":2,"width":4,"height":2}},{"id":3902930069982135,"definition":{"title":"Batches successful vs failures","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"time":{},"type":"timeseries","requests":[{"formulas":[{"alias":"Successfully processed batch","formula":"query1"},{"alias":"Batch processing with error","formula":"query2"}],"queries":[{"query":"sum:karafka.consumer.batches{*} by {partition,topic,consumer_group}.as_count()","data_source":"metrics","name":"query1"},{"query":"avg:karafka.error_occurred{type:consumer.consume.error} by {partition,topic,consumer_group}.as_count()","data_source":"metrics","name":"query2"}],"response_format":"timeseries","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":4,"y":2,"width":4,"height":2}},{"id":718749162159145,"definition":{"title":"Consumer instances revocations and shutdowns","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"time":{},"type":"timeseries","requests":[{"formulas":[{"alias":"Consumer instances revokations","formula":"query1"},{"alias":"Consumer instances shutdowns","formula":"query2"}],"queries":[{"query":"sum:karafka.consumer.revoked{*} by {consumer_group}.as_count()","data_source":"metrics","name":"query1"},{"query":"sum:karafka.consumer.shutdown{*} by {consumer_group}.as_count()","data_source":"metrics","name":"query2"}],"response_format":"timeseries","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":8,"y":2,"width":4,"height":2}}]},"layout":{"x":0,"y":3,"width":12,"height":5}},{"id":7288186528768428,"definition":{"title":"Topics overview","type":"group","show_title":true,"layout_type":"ordered","widgets":[{"id":1533435157804573,"definition":{"title":"Topics lags","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"time":{},"type":"timeseries","requests":[{"formulas":[{"alias":"Messages lag per topic partition","formula":"query1"}],"queries":[{"query":"avg:karafka.consumer.lags{*} by {partition,topic,consumer_group}","data_source":"metrics","name":"query1"}],"response_format":"timeseries","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":0,"y":0,"width":6,"height":2}},{"id":1411506453982604,"definition":{"title":"Topics lag trends","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"time":{},"type":"timeseries","requests":[{"formulas":[{"alias":"Topic partition consumption trend","formula":"query1"}],"queries":[{"query":"avg:karafka.consumer.lags_delta{*} by {partition,topic,consumer_group}","data_source":"metrics","name":"query1"}],"response_format":"timeseries","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":6,"y":0,"width":6,"height":2}}]},"layout":{"x":0,"y":0,"width":12,"height":3,"is_column_break":true}},{"id":8544040083223278,"definition":{"title":"Throughput ","type":"group","show_title":true,"layout_type":"ordered","widgets":[{"id":3740207481939733,"definition":{"title":"Offset lag changes","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"time":{},"type":"timeseries","requests":[{"formulas":[{"formula":"derivative(query1)"}],"queries":[{"query":"max:karafka.consumer.offset{*} by {topic,partition,consumer_group}","data_source":"metrics","name":"query1"}],"response_format":"timeseries","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":0,"y":0,"width":4,"height":2}},{"id":6319110548544878,"definition":{"title":"Batches processed per topic","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"time":{},"type":"timeseries","requests":[{"formulas":[{"alias":"Topic batches processed","formula":"query1"}],"queries":[{"query":"sum:karafka.consumer.batches{*} by {partition,topic,consumer_group}.as_count()","data_source":"metrics","name":"query1"}],"response_format":"timeseries","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":4,"y":0,"width":4,"height":2}},{"id":6232784865331443,"definition":{"title":"Messages consumed per topic","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"time":{},"type":"timeseries","requests":[{"formulas":[{"alias":"Messages consumed","formula":"query1"},{"alias":"Average batch size","formula":"query1 / query2"}],"queries":[{"query":"sum:karafka.consumer.messages{*} by {partition,topic,consumer_group}.as_count()","data_source":"metrics","name":"query1"},{"query":"sum:karafka.consumer.batches{*} by {partition,topic,consumer_group}.as_count()","data_source":"metrics","name":"query2"}],"response_format":"timeseries","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":8,"y":0,"width":4,"height":2}},{"id":2321394598982770,"definition":{"title":"Consumption lag (in seconds)","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"time":{},"type":"timeseries","requests":[{"formulas":[{"alias":"Consumption lag in s (max)","formula":"query2 / 1000"},{"alias":"Consumption lag in s (avg)","formula":"query3 / 1000"},{"alias":"Consumption lag in s (p95)","formula":"query1 / 1000"}],"queries":[{"query":"max:karafka.consumer.consumption_lag.max{*} by {consumer_group}","data_source":"metrics","name":"query2"},{"query":"max:karafka.consumer.consumption_lag.avg{*} by {consumer_group}","data_source":"metrics","name":"query3"},{"query":"max:karafka.consumer.consumption_lag.95percentile{*} by {consumer_group}","data_source":"metrics","name":"query1"}],"response_format":"timeseries","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":0,"y":2,"width":4,"height":2}},{"id":1062074781483741,"definition":{"title":"Processing lag (in ms)","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"time":{},"type":"timeseries","requests":[{"formulas":[{"alias":"Lag in ms (p95)","formula":"query1"},{"alias":"Lag in ms (max)","formula":"query2"},{"alias":"Lag in ms (avg)","formula":"query3"}],"queries":[{"query":"max:karafka.consumer.processing_lag.95percentile{*} by {consumer_group}","data_source":"metrics","name":"query1"},{"query":"max:karafka.consumer.processing_lag.max{*} by {consumer_group}","data_source":"metrics","name":"query2"},{"query":"max:karafka.consumer.processing_lag.avg{*} by {consumer_group}","data_source":"metrics","name":"query3"}],"response_format":"timeseries","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":4,"y":2,"width":4,"height":2}},{"id":7497794728674267,"definition":{"title":"Batch processing time","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"time":{},"type":"timeseries","requests":[{"formulas":[{"alias":"p95 time","formula":"query1"},{"alias":"max time","formula":"query2"},{"alias":"avg time","formula":"query3"}],"queries":[{"query":"max:karafka.consumer.consumed.time_taken.95percentile{*} by {topic,partition,consumer_group}","data_source":"metrics","name":"query1"},{"query":"max:karafka.consumer.consumed.time_taken.max{*} by {topic,partition,consumer_group}","data_source":"metrics","name":"query2"},{"query":"max:karafka.consumer.consumed.time_taken.avg{*} by {topic,partition,consumer_group}","data_source":"metrics","name":"query3"}],"response_format":"timeseries","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":8,"y":2,"width":4,"height":2}},{"id":4192833027984161,"definition":{"title":"Batch size per topic","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"time":{},"type":"timeseries","requests":[{"formulas":[{"alias":"Batch size p95","formula":"query1"},{"alias":"Batch size avg","formula":"query2"},{"alias":"Batch size max","formula":"query3"}],"queries":[{"query":"sum:karafka.consumer.batch_size.95percentile{*} by {partition,topic,consumer_group}","data_source":"metrics","name":"query1"},{"query":"sum:karafka.consumer.batch_size.avg{*} by {partition,topic,consumer_group}","data_source":"metrics","name":"query2"},{"query":"sum:karafka.consumer.batch_size.max{*} by {partition,topic,consumer_group}","data_source":"metrics","name":"query3"}],"response_format":"timeseries","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":0,"y":4,"width":4,"height":2}},{"id":4741598444771147,"definition":{"title":"Messages consumed overall","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"time":{},"type":"timeseries","requests":[{"formulas":[{"alias":"Messages consumed","formula":"query1"},{"alias":"Average batch size","formula":"query1 / query2"}],"queries":[{"query":"sum:karafka.consumer.messages{*} by {consumer_group}.as_count()","data_source":"metrics","name":"query1"},{"query":"sum:karafka.consumer.batches{*} by {consumer_group}.as_count()","data_source":"metrics","name":"query2"}],"response_format":"timeseries","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":4,"y":4,"width":4,"height":2}},{"id":4502534794102513,"definition":{"title":"Polling times (ms)","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"time":{},"type":"timeseries","requests":[{"formulas":[{"alias":"p95 ms","formula":"query1"},{"alias":"max ms","formula":"query2"},{"alias":"average ms","formula":"query3"}],"queries":[{"query":"avg:karafka.listener.polling.time_taken.95percentile{*} by {consumer_group}","data_source":"metrics","name":"query1"},{"query":"avg:karafka.listener.polling.time_taken.max{*} by {consumer_group}","data_source":"metrics","name":"query2"},{"query":"avg:karafka.listener.polling.time_taken.avg{*} by {consumer_group}","data_source":"metrics","name":"query3"}],"response_format":"timeseries","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":8,"y":4,"width":4,"height":2}}]},"layout":{"x":0,"y":3,"width":12,"height":7}}],"template_variables":[],"layout_type":"ordered","is_read_only":false,"notify_list":[],"reflow_type":"fixed","id":"s3u-z47-i6u"}