sbmt-kafka_consumer 2.0.0

data/lib/sbmt/kafka_consumer/client_configurer.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+class Sbmt::KafkaConsumer::ClientConfigurer
+  def self.configure!(**opts)
+    config = Sbmt::KafkaConsumer::Config.new
+    Karafka::App.setup do |karafka_config|
+      karafka_config.monitor = config.monitor_class.classify.constantize.new
+      karafka_config.logger = Sbmt::KafkaConsumer.logger
+      karafka_config.deserializer = config.deserializer_class.classify.constantize.new
+
+      karafka_config.client_id = config.client_id
+      karafka_config.consumer_mapper = config.consumer_mapper_class.classify.constantize.new
+      karafka_config.kafka = config.to_kafka_options
+
+      karafka_config.pause_timeout = config.pause_timeout * 1_000 if config.pause_timeout.present?
+      karafka_config.pause_max_timeout = config.pause_max_timeout * 1_000 if config.pause_max_timeout.present?
+      karafka_config.max_wait_time = config.max_wait_time * 1_000 if config.max_wait_time.present?
+      karafka_config.shutdown_timeout = config.shutdown_timeout * 1_000 if config.shutdown_timeout.present?
+
+      karafka_config.pause_with_exponential_backoff = config.pause_with_exponential_backoff if config.pause_with_exponential_backoff.present?
+
+      karafka_config.concurrency = (opts[:concurrency]) || config.concurrency
+
+      # Do not validate topics naming consistency
+      # see https://github.com/karafka/karafka/wiki/FAQ#why-am-i-seeing-a-needs-to-be-consistent-namespacing-style-error
+      karafka_config.strict_topics_namespacing = false
+
+      # Recreate consumers with each batch. This will allow Rails code reload to work in the
+      # development mode. Otherwise Karafka process would not be aware of code changes
+      karafka_config.consumer_persistence = !Rails.env.development?
+    end
+
+    Karafka.monitor.subscribe(config.logger_listener_class.classify.constantize.new)
+    Karafka.monitor.subscribe(config.metrics_listener_class.classify.constantize.new)
+
+    target_consumer_groups = if opts[:consumer_groups].blank?
+      config.consumer_groups
+    else
+      config.consumer_groups.select do |group|
+        opts[:consumer_groups].include?(group.id)
+      end
+    end
+
+    raise "No configured consumer groups found, exiting" if target_consumer_groups.blank?
+
+    # clear routes in case CLI runner tries to reconfigure them
+    # but railtie initializer had already executed and did the same
+    # otherwise we'll get duplicate routes error from sbmt-karafka internal config validation process
+    Karafka::App.routes.clear
+    Karafka::App.routes.draw do
+      target_consumer_groups.each do |cg|
+        consumer_group cg.name do
+          cg.topics.each do |t|
+            topic t.name do
+              active t.active
+              manual_offset_management t.manual_offset_management
+              consumer t.consumer.consumer_klass
+              deserializer t.deserializer.instantiate if t.deserializer.klass.present?
+              kafka t.kafka_options if t.kafka_options.present?
+            end
+          end
+        end
+      end
+    end
+  end
+
+  def self.routes
+    Karafka::App.routes.map do |cg|
+      topics = cg.topics.map { |t| {name: t.name, deserializer: t.deserializer} }
+      {group: cg.id, topics: topics}
+    end
+  end
+end

data/lib/sbmt/kafka_consumer/config/auth.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+class Sbmt::KafkaConsumer::Config::Auth < Dry::Struct
+  transform_keys(&:to_sym)
+
+  AVAILABLE_AUTH_KINDS = %w[plaintext sasl_plaintext].freeze
+  DEFAULT_AUTH_KIND = "plaintext"
+
+  AVAILABLE_SASL_MECHANISMS = %w[PLAIN SCRAM-SHA-256 SCRAM-SHA-512].freeze
+  DEFAULT_SASL_MECHANISM = "SCRAM-SHA-512"
+
+  attribute :kind, Sbmt::KafkaConsumer::Types::Strict::String
+    .default(DEFAULT_AUTH_KIND)
+    .enum(*AVAILABLE_AUTH_KINDS)
+  attribute? :sasl_mechanism, Sbmt::KafkaConsumer::Types::Strict::String
+    .default(DEFAULT_SASL_MECHANISM)
+    .enum(*AVAILABLE_SASL_MECHANISMS)
+  attribute? :sasl_username, Sbmt::KafkaConsumer::Types::Strict::String
+  attribute? :sasl_password, Sbmt::KafkaConsumer::Types::Strict::String
+
+  def to_kafka_options
+    ensure_options_are_valid
+
+    opts = {}
+
+    case kind
+    when "sasl_plaintext"
+      opts.merge!(
+        "security.protocol": kind,
+        "sasl.mechanism": sasl_mechanism,
+        "sasl.username": sasl_username,
+        "sasl.password": sasl_password
+      )
+    when "plaintext"
+      opts[:"security.protocol"] = kind
+    else
+      raise Anyway::Config::ValidationError, "unknown auth kind: #{kind}"
+    end
+
+    opts.symbolize_keys
+  end
+
+  private
+
+  def ensure_options_are_valid
+    raise Anyway::Config::ValidationError, "unknown auth kind: #{kind}" unless AVAILABLE_AUTH_KINDS.include?(kind)
+
+    case kind
+    when "sasl_plaintext"
+      raise Anyway::Config::ValidationError, "sasl_username is required for #{kind} auth kind" if sasl_username.blank?
+      raise Anyway::Config::ValidationError, "sasl_password is required for #{kind} auth kind" if sasl_password.blank?
+      raise Anyway::Config::ValidationError, "sasl_mechanism is required for #{kind} auth kind" if sasl_mechanism.blank?
+      raise Anyway::Config::ValidationError, "invalid sasl_mechanism for #{kind} auth kind, available options are: [#{AVAILABLE_SASL_MECHANISMS.join(",")}]" unless AVAILABLE_SASL_MECHANISMS.include?(sasl_mechanism)
+    end
+  end
+end

data/lib/sbmt/kafka_consumer/config/consumer.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class Sbmt::KafkaConsumer::Config::Consumer < Dry::Struct
+  transform_keys(&:to_sym)
+
+  attribute :klass, Sbmt::KafkaConsumer::Types::Strict::String
+  attribute :init_attrs, Sbmt::KafkaConsumer::Types::ConfigAttrs.optional.default({}.freeze)
+
+  def consumer_klass
+    target_klass = klass.constantize
+
+    return target_klass.consumer_klass if init_attrs.blank?
+
+    target_klass.consumer_klass(**init_attrs)
+  end
+end

data/lib/sbmt/kafka_consumer/config/consumer_group.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class Sbmt::KafkaConsumer::Config::ConsumerGroup < Dry::Struct
+  transform_keys(&:to_sym)
+
+  attribute :id, Sbmt::KafkaConsumer::Types::Strict::String
+  attribute :name, Sbmt::KafkaConsumer::Types::Strict::String
+  attribute :topics, Sbmt::KafkaConsumer::Types.Array(Sbmt::KafkaConsumer::Types::ConfigTopic)
+end

data/lib/sbmt/kafka_consumer/config/deserializer.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class Sbmt::KafkaConsumer::Config::Deserializer < Dry::Struct
+  transform_keys(&:to_sym)
+
+  attribute :klass, Sbmt::KafkaConsumer::Types::Strict::String
+    .optional
+    .default(Sbmt::KafkaConsumer::Serialization::NullDeserializer.to_s.freeze)
+  attribute :init_attrs, Sbmt::KafkaConsumer::Types::ConfigAttrs.optional.default({}.freeze)
+
+  def instantiate
+    return klass.constantize.new if init_attrs.blank?
+    klass.constantize.new(**init_attrs)
+  end
+end

data/lib/sbmt/kafka_consumer/config/kafka.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+class Sbmt::KafkaConsumer::Config::Kafka < Dry::Struct
+  transform_keys(&:to_sym)
+
+  # srv1:port1,srv2:port2,...
+  SERVERS_REGEXP = /^[a-z\d.\-:]+(,[a-z\d.\-:]+)*$/.freeze
+
+  attribute :servers, Sbmt::KafkaConsumer::Types::String.constrained(format: SERVERS_REGEXP)
+
+  # defaults are rdkafka's
+  # see https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md
+  attribute :heartbeat_timeout, Sbmt::KafkaConsumer::Types::Coercible::Integer.optional.default(5)
+  attribute :session_timeout, Sbmt::KafkaConsumer::Types::Coercible::Integer.optional.default(30)
+  attribute :reconnect_timeout, Sbmt::KafkaConsumer::Types::Coercible::Integer.optional.default(3)
+  attribute :connect_timeout, Sbmt::KafkaConsumer::Types::Coercible::Integer.optional.default(5)
+  attribute :socket_timeout, Sbmt::KafkaConsumer::Types::Coercible::Integer.optional.default(30)
+
+  attribute :kafka_options, Sbmt::KafkaConsumer::Types::ConfigAttrs.optional.default({}.freeze)
+
+  def to_kafka_options
+    # root options take precedence over kafka_options' ones
+    kafka_options.merge(
+      "bootstrap.servers": servers,
+      "heartbeat.interval.ms": heartbeat_timeout * 1_000,
+      "session.timeout.ms": session_timeout * 1_000,
+      "reconnect.backoff.max.ms": reconnect_timeout * 1_000,
+      "socket.connection.setup.timeout.ms": connect_timeout * 1_000,
+      "socket.timeout.ms": socket_timeout * 1_000
+    ).symbolize_keys
+  end
+end

data/lib/sbmt/kafka_consumer/config/metrics.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+class Sbmt::KafkaConsumer::Config::Metrics < Dry::Struct
+  transform_keys(&:to_sym)
+
+  attribute? :port, Sbmt::KafkaConsumer::Types::Coercible::Integer.optional
+  attribute :path, Sbmt::KafkaConsumer::Types::Strict::String
+    .optional
+    .default("/metrics")
+end

data/lib/sbmt/kafka_consumer/config/probes/endpoints.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class Sbmt::KafkaConsumer::Config::Probes::Endpoints < Dry::Struct
+  transform_keys(&:to_sym)
+
+  attribute :liveness, Sbmt::KafkaConsumer::Config::Probes::LivenessProbe.optional.default(
+    Sbmt::KafkaConsumer::Config::Probes::LivenessProbe.new.freeze
+  )
+
+  attribute :readiness, Sbmt::KafkaConsumer::Config::Probes::ReadinessProbe.optional.default(
+    Sbmt::KafkaConsumer::Config::Probes::ReadinessProbe.new.freeze
+  )
+end

data/lib/sbmt/kafka_consumer/config/probes/liveness_probe.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class Sbmt::KafkaConsumer::Config::Probes::LivenessProbe < Dry::Struct
+  transform_keys(&:to_sym)
+
+  attribute :enabled, Sbmt::KafkaConsumer::Types::Bool.optional.default(true)
+  attribute :path, Sbmt::KafkaConsumer::Types::Strict::String
+    .optional
+    .default("/liveness")
+  attribute :timeout, Sbmt::KafkaConsumer::Types::Coercible::Integer.optional.default(10)
+end

data/lib/sbmt/kafka_consumer/config/probes/readiness_probe.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+class Sbmt::KafkaConsumer::Config::Probes::ReadinessProbe < Dry::Struct
+  transform_keys(&:to_sym)
+
+  attribute :enabled, Sbmt::KafkaConsumer::Types::Bool.optional.default(true)
+  attribute :path, Sbmt::KafkaConsumer::Types::Strict::String
+    .optional
+    .default("/readiness/kafka_consumer")
+end

data/lib/sbmt/kafka_consumer/config/probes.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class Sbmt::KafkaConsumer::Config::Probes < Dry::Struct
+  transform_keys(&:to_sym)
+
+  attribute :port, Sbmt::KafkaConsumer::Types::Coercible::Integer.optional.default(9394)
+  attribute :endpoints, Endpoints.optional.default(Endpoints.new.freeze)
+end

data/lib/sbmt/kafka_consumer/config/topic.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class Sbmt::KafkaConsumer::Config::Topic < Dry::Struct
+  transform_keys(&:to_sym)
+
+  attribute :name, Sbmt::KafkaConsumer::Types::Strict::String
+  attribute :consumer, Sbmt::KafkaConsumer::Types::ConfigConsumer
+  attribute :deserializer, Sbmt::KafkaConsumer::Types::ConfigDeserializer
+    .optional
+    .default(Sbmt::KafkaConsumer::Config::Deserializer.new.freeze)
+  attribute :active, Sbmt::KafkaConsumer::Types::Bool.optional.default(true)
+  attribute :manual_offset_management, Sbmt::KafkaConsumer::Types::Bool.optional.default(true)
+  attribute? :kafka_options, Sbmt::KafkaConsumer::Types::ConfigAttrs.optional.default({}.freeze)
+end

data/lib/sbmt/kafka_consumer/config.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+class Sbmt::KafkaConsumer::Config < Anyway::Config
+  config_name :kafka_consumer
+
+  class << self
+    def coerce_to(struct)
+      lambda do |raw_attrs|
+        struct.new(**raw_attrs)
+      rescue Dry::Types::SchemaError => e
+        raise_validation_error "cannot parse #{struct}: #{e.message}"
+      end
+    end
+
+    def coerce_to_array_of(struct)
+      lambda do |raw_attrs|
+        raw_attrs.keys.map do |obj_title|
+          coerce_to(struct)
+            .call(**raw_attrs.fetch(obj_title)
+                            .merge(id: obj_title))
+        end
+      end
+    end
+  end
+
+  attr_config :client_id,
+    :pause_timeout, :pause_max_timeout, :pause_with_exponential_backoff,
+    :max_wait_time, :shutdown_timeout,
+    concurrency: 4, auth: {}, kafka: {}, consumer_groups: {}, probes: {}, metrics: {},
+    deserializer_class: "::Sbmt::KafkaConsumer::Serialization::NullDeserializer",
+    monitor_class: "::Sbmt::KafkaConsumer::Instrumentation::TracingMonitor",
+    logger_class: "::Sbmt::KafkaConsumer::Logger",
+    logger_listener_class: "::Sbmt::KafkaConsumer::Instrumentation::LoggerListener",
+    metrics_listener_class: "::Sbmt::KafkaConsumer::Instrumentation::YabedaMetricsListener",
+    consumer_mapper_class: "::Sbmt::KafkaConsumer::Routing::KarafkaV1ConsumerMapper"
+
+  required :client_id
+
+  on_load :validate_consumer_groups
+  on_load :set_default_metrics_port
+
+  coerce_types client_id: :string,
+    pause_timeout: :integer,
+    pause_max_timeout: :integer,
+    pause_with_exponential_backoff: :boolean,
+    max_wait_time: :integer,
+    shutdown_timeout: :integer,
+    concurrency: :integer
+
+  coerce_types kafka: coerce_to(Kafka)
+  coerce_types auth: coerce_to(Auth)
+  coerce_types probes: coerce_to(Probes)
+  coerce_types metrics: coerce_to(Metrics)
+  coerce_types consumer_groups: coerce_to_array_of(ConsumerGroup)
+
+  def to_kafka_options
+    kafka.to_kafka_options
+      .merge(auth.to_kafka_options)
+  end
+
+  private
+
+  def validate_consumer_groups
+    consumer_groups.each do |cg|
+      raise_validation_error "consumer group #{cg.id} must have at least one topic defined" if cg.topics.blank?
+      cg.topics.each do |t|
+        raise_validation_error "topic #{cg.id}.topics.name[#{t.name}] contains invalid consumer class: no const #{t.consumer.klass} defined" unless t.consumer.klass.safe_constantize
+        raise_validation_error "topic #{cg.id}.topics.name[#{t.name}] contains invalid deserializer class: no const #{t.deserializer.klass} defined" unless t.deserializer&.klass&.safe_constantize
+      end
+    end
+  end
+
+  def set_default_metrics_port
+    self.metrics = metrics.new(port: probes.port) unless metrics.port
+  end
+end

data/lib/sbmt/kafka_consumer/inbox_consumer.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaConsumer
+    class InboxConsumer < BaseConsumer
+      IDEMPOTENCY_HEADER_NAME = "Idempotency-Key"
+      DEFAULT_SOURCE = "KAFKA"
+
+      def self.consumer_klass(inbox_item:, event_name: nil, skip_on_error: false, name: nil)
+        Class.new(self) do
+          const_set(:INBOX_ITEM_CLASS_NAME, inbox_item)
+          const_set(:EVENT_NAME, event_name)
+          const_set(:SKIP_ON_ERROR, skip_on_error)
+
+          def self.name
+            superclass.name
+          end
+        end
+      end
+
+      def extra_message_attrs(_message)
+        {}
+      end
+
+      private
+
+      def process_message(message)
+        logger.tagged(inbox_name: inbox_name, event_name: event_name) do
+          ::Sbmt::KafkaConsumer.monitor.instrument(
+            "consumer.inbox.consumed_one", caller: self,
+            message: message,
+            message_uuid: message_uuid(message),
+            inbox_name: inbox_name,
+            event_name: event_name,
+            status: "success"
+          ) do
+            process_inbox_item(message)
+          end
+        end
+      end
+
+      def process_inbox_item(message)
+        result = Sbmt::Outbox::CreateInboxItem.call(
+          inbox_item_class,
+          attributes: message_attrs(message)
+        )
+
+        if result.failure?
+          raise "Failed consuming message for #{inbox_name}, message_uuid: #{message_uuid(message)}: #{result}"
+        end
+
+        item = result.success
+        item.track_metrics_after_consume if item.respond_to?(:track_metrics_after_consume)
+      rescue ActiveRecord::RecordNotUnique
+        instrument_error("Skipped duplicate message for #{inbox_name}, message_uuid: #{message_uuid(message)}", message, "duplicate")
+      rescue => ex
+        if skip_on_error
+          logger.warn("skipping unprocessable message for #{inbox_name}, message_uuid: #{message_uuid(message)}")
+          instrument_error(ex, message, "skipped")
+        else
+          instrument_error(ex, message)
+        end
+        raise ex
+      end
+
+      def message_attrs(message)
+        attrs = {
+          proto_payload: message.raw_payload,
+          options: {
+            headers: message.metadata.headers.dup,
+            group_id: topic.consumer_group.id,
+            topic: message.metadata.topic,
+            partition: message.metadata.partition,
+            source: DEFAULT_SOURCE
+          }
+        }
+
+        if message_uuid(message)
+          attrs[:uuid] = message_uuid(message)
+        end
+
+        # if message has no uuid, it will be generated later in Sbmt::Outbox::CreateInboxItem
+
+        attrs[:event_key] = if message.metadata.key.present?
+          message.metadata.key
+        elsif inbox_item_class.respond_to?(:event_key)
+          inbox_item_class.event_key(message)
+        else
+          # if message has no partitioning key
+          # set it to something random and monotonically increasing like offset
+          message.offset
+        end
+
+        attrs[:event_name] = event_name if inbox_item_class.has_attribute?(:event_name)
+
+        attrs.merge(extra_message_attrs(message))
+      end
+
+      def message_uuid(message)
+        message.metadata.headers.fetch(IDEMPOTENCY_HEADER_NAME, nil).presence
+      end
+
+      def inbox_item_class
+        @inbox_item_class ||= self.class::INBOX_ITEM_CLASS_NAME.constantize
+      end
+
+      def event_name
+        @event_name ||= self.class::EVENT_NAME
+      end
+
+      def inbox_name
+        inbox_item_class.box_name
+      end
+
+      def instrument_error(error, message, status = "failure")
+        ::Sbmt::KafkaConsumer.monitor.instrument(
+          "error.occurred",
+          error: error,
+          caller: self,
+          message: message,
+          inbox_name: inbox_name,
+          event_name: event_name,
+          status: status,
+          type: "consumer.inbox.consume_one"
+        )
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_consumer/instrumentation/base_monitor.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaConsumer
+    module Instrumentation
+      class BaseMonitor < Karafka::Instrumentation::Monitor
+        # karafka consuming is based around batch-processing
+        # so we need these per-message custom events
+        SBMT_KAFKA_CONSUMER_EVENTS = %w[
+          consumer.consumed_one
+          consumer.inbox.consumed_one
+        ].freeze
+
+        def initialize
+          super
+          SBMT_KAFKA_CONSUMER_EVENTS.each { |event_id| notifications_bus.register_event(event_id) }
+        end
+
+        def instrument(_event_id, _payload = EMPTY_HASH, &block)
+          super
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_consumer/instrumentation/chainable_monitor.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaConsumer
+    module Instrumentation
+      class ChainableMonitor < BaseMonitor
+        attr_reader :monitors
+
+        def initialize(monitors = [])
+          super()
+
+          @monitors = monitors
+        end
+
+        def instrument(event_id, payload = EMPTY_HASH, &block)
+          return super if monitors.empty?
+
+          chain = monitors.map { |monitor| monitor.new(event_id, payload) }
+          traverse_chain = proc do
+            if chain.empty?
+              super
+            else
+              chain.shift.trace(&traverse_chain)
+            end
+          end
+          traverse_chain.call
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_consumer/instrumentation/listener_helper.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaConsumer
+    module Instrumentation
+      module ListenerHelper
+        delegate :logger, to: ::Sbmt::KafkaConsumer
+
+        private
+
+        def consumer_tags(event)
+          message = event[:message]
+          {
+            topic: message.metadata.topic,
+            partition: message.metadata.partition
+          }
+        end
+
+        def inbox_tags(event)
+          {
+            inbox_name: event[:inbox_name],
+            event_name: event[:event_name],
+            status: event[:status]
+          }
+        end
+
+        def error_message(error)
+          if error.respond_to?(:message)
+            error.message
+          elsif error.respond_to?(:failure)
+            error.failure
+          else
+            error.to_s
+          end
+        end
+
+        def log_backtrace(error)
+          if error.respond_to?(:backtrace)
+            logger.error(error.backtrace.join("\n"))
+          elsif error.respond_to?(:trace)
+            logger.error(error.trace)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_consumer/instrumentation/liveness_listener.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaConsumer
+    module Instrumentation
+      class LivenessListener
+        include ListenerHelper
+        include KafkaConsumer::Probes::Probe
+
+        def initialize(timeout_sec: 10)
+          @consumer_groups = Karafka::App.routes.map(&:name)
+          @timeout_sec = timeout_sec
+          @polls = {}
+
+          setup_subscription
+        end
+
+        def probe(_env)
+          now = current_time
+          timed_out_polls = select_timed_out_polls(now)
+          return probe_ok groups: meta_from_polls(polls, now) if timed_out_polls.empty?
+
+          probe_error failed_groups: meta_from_polls(timed_out_polls, now)
+        end
+
+        def on_connection_listener_fetch_loop(event)
+          consumer_group = event.payload[:subscription_group].consumer_group
+          polls[consumer_group.name] = current_time
+        end
+
+        private
+
+        attr_reader :polls, :timeout_sec, :consumer_groups
+
+        def current_time
+          Time.now.utc
+        end
+
+        def select_timed_out_polls(now)
+          raise "consumer_groups are empty. Please set them up" if consumer_groups.empty?
+
+          consumer_groups.each_with_object({}) do |group, hash|
+            last_poll_at = polls[group]
+            next if last_poll_at && last_poll_at + timeout_sec >= now
+
+            hash[group] = last_poll_at
+          end
+        end
+
+        def meta_from_polls(polls, now)
+          polls.each_with_object({}) do |(group, last_poll_at), hash|
+            if last_poll_at.nil?
+              hash[group] = {had_poll: false}
+              next
+            end
+
+            hash[group] = {
+              had_poll: true,
+              last_poll_at: last_poll_at,
+              seconds_since_last_poll: (now - last_poll_at).to_i
+            }
+          end
+        end
+
+        def setup_subscription
+          Karafka::App.monitor.subscribe(self)
+        end
+      end
+    end
+  end
+end