deimos-kafka 1.0.0.pre.beta15

data/lib/deimos/message.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Deimos
+  # Basically a struct to hold the message as it's processed.
+  class Message
+    attr_accessor :payload, :key, :partition_key, :encoded_key,
+                  :encoded_payload, :topic, :producer_name
+
+    # @param payload [Hash]
+    # @param producer [Class]
+    def initialize(payload, producer, topic: nil, key: nil, partition_key: nil)
+      @payload = payload&.with_indifferent_access
+      @producer_name = producer.name
+      @topic = topic
+      @key = key
+      @partition_key = partition_key
+    end
+
+    # Add message_id and timestamp default values if they are in the
+    # schema and don't already have values.
+    # @param schema [Avro::Schema]
+    def add_fields(schema)
+      return if @payload.except(:payload_key, :partition_key).blank?
+
+      if schema.fields.any? { |f| f.name == 'message_id' }
+        @payload['message_id'] ||= SecureRandom.uuid
+      end
+      if schema.fields.any? { |f| f.name == 'timestamp' }
+        @payload['timestamp'] ||= Time.now.in_time_zone.to_s
+      end
+    end
+
+    # @param schema [Avro::Schema]
+    def coerce_fields(schema)
+      return if payload.nil?
+
+      @payload = SchemaCoercer.new(schema).coerce(@payload)
+    end
+
+    # @return [Hash]
+    def encoded_hash
+      {
+        topic: @topic,
+        key: @encoded_key,
+        partition_key: @partition_key || @encoded_key,
+        payload: @encoded_payload,
+        metadata: {
+          decoded_payload: @payload,
+          producer_name: @producer_name
+        }
+      }
+    end
+
+    # @return [Hash]
+    def to_h
+      {
+        topic: @topic,
+        key: @key,
+        partition_key: @partition_key || @key,
+        payload: @payload,
+        metadata: {
+          decoded_payload: @payload,
+          producer_name: @producer_name
+        }
+      }
+    end
+
+    # @param other [Message]
+    # @return [Boolean]
+    def ==(other)
+      self.to_h == other.to_h
+    end
+  end
+end

data/lib/deimos/metrics/datadog.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require 'deimos/metrics/provider'
+
+module Deimos
+  module Metrics
+    # A Metrics wrapper class for Datadog.
+    class Datadog < Metrics::Provider
+      # :nodoc:
+      def initialize(config, logger)
+        raise 'Metrics config must specify host_ip' if config[:host_ip].nil?
+        raise 'Metrics config must specify host_port' if config[:host_port].nil?
+        raise 'Metrics config must specify namespace' if config[:namespace].nil?
+
+        logger.info("DatadogMetricsProvider configured with: #{config}")
+        @client = Datadog::Statsd.new(
+          config[:host_ip],
+          config[:host_port]
+        )
+        @client.tags = config[:tags]
+        @client.namespace = config[:namespace]
+      end
+
+      # :nodoc:
+      def increment(metric_name, options={})
+        @client.increment(metric_name, options)
+      end
+
+      # :nodoc:
+      def gauge(metric_name, count, options={})
+        @client.gauge(metric_name, count, options)
+      end
+
+      # :nodoc:
+      def histogram(metric_name, count, options={})
+        @client.histogram(metric_name, count, options)
+      end
+
+      # :nodoc:
+      def time(metric_name, options={})
+        @client.time(metric_name, options) do
+          yield
+        end
+      end
+    end
+  end
+end

data/lib/deimos/metrics/mock.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'deimos/metrics/provider'
+
+module Deimos
+  module Metrics
+    # A mock Metrics wrapper which just logs the metrics
+    class Mock
+      # :nodoc:
+      def initialize(logger=nil)
+        @logger = logger || Logger.new(STDOUT)
+        @logger.info('MockMetricsProvider initialized')
+      end
+
+      # :nodoc:
+      def increment(metric_name, options)
+        @logger.info("MockMetricsProvider.increment: #{metric_name}, #{options}")
+      end
+
+      # :nodoc:
+      def gauge(metric_name, count, options)
+        @logger.info("MockMetricsProvider.gauge: #{metric_name}, #{count}, #{options}")
+      end
+
+      # :nodoc:
+      def histogram(metric_name, count, options)
+        @logger.info("MockMetricsProvider.histogram: #{metric_name}, #{count}, #{options}")
+      end
+
+      # :nodoc:
+      def time(metric_name, options={})
+        start_time = Time.now
+        yield
+        total_time = (Time.now - start_time).to_i
+        @logger.info("MockMetricsProvider.time: #{metric_name}, #{total_time}, #{options}")
+      end
+    end
+  end
+end

data/lib/deimos/metrics/provider.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# rubocop:disable Lint/UnusedMethodArgument
+module Deimos
+  module Metrics
+    # Base class for all metrics providers.
+    class Provider
+      # Send an counter increment metric
+      # @param metric_name [String] The name of the counter metric
+      # @param options [Hash] Any additional options, e.g. :tags
+      def increment(metric_name, options)
+        raise NotImplementedError
+      end
+
+      # Send an counter increment metric
+      # @param metric_name [String] The name of the counter metric
+      # @param options [Hash] Any additional options, e.g. :tags
+      def gauge(metric_name, count, options)
+        raise NotImplementedError
+      end
+
+      # Send an counter increment metric
+      # @param metric_name [String] The name of the counter metric
+      # @param options [Hash] Any additional options, e.g. :tags
+      def histogram(metric_name, count, options)
+        raise NotImplementedError
+      end
+
+      # Time a yielded block, and send a timer metric
+      # @param metric_name [String] The name of the metric
+      # @param options [Hash] Any additional options, e.g. :tags
+      def time(metric_name, options={})
+        raise NotImplementedError
+      end
+    end
+  end
+end
+# rubocop:enable Lint/UnusedMethodArgument

data/lib/deimos/monkey_patches/phobos_cli.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require 'phobos/cli/start'
+
+# :nodoc:
+module Phobos
+  # :nodoc:
+  module CLI
+    # :nodoc:
+    class Start
+      # :nodoc:
+      def validate_listeners!
+        Phobos.config.listeners.each do |listener|
+          handler = listener.handler
+          begin
+            handler.constantize
+          rescue NameError
+            error_exit("Handler '#{handler}' not defined")
+          end
+
+          delivery = listener.delivery
+          if delivery.nil?
+            Phobos::CLI.logger.warn do
+              Hash(message: "Delivery option should be specified, defaulting to 'batch'"\
+               ' - specify this option to silence this message')
+            end
+          elsif !Listener::DELIVERY_OPTS.include?(delivery)
+            error_exit("Invalid delivery option '#{delivery}'. Please specify one of: "\
+              "#{Listener::DELIVERY_OPTS.join(', ')}")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/deimos/monkey_patches/phobos_producer.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'phobos/producer'
+
+module Phobos
+  module Producer
+    # :nodoc:
+    class PublicAPI
+      # :nodoc:
+      def publish(topic, payload, key=nil, partition_key=nil)
+        class_producer.publish(topic, payload, key, partition_key)
+      end
+
+      # :nodoc:
+      def async_publish(topic, payload, key=nil, partition_key=nil)
+        class_producer.async_publish(topic, payload, key, partition_key)
+      end
+    end
+
+    # :nodoc:
+    module ClassMethods
+      # :nodoc:
+      class PublicAPI
+        # :nodoc:
+        def publish(topic, payload, key=nil, partition_key=nil)
+          publish_list([{ topic: topic, payload: payload, key: key,
+                          partition_key: partition_key }])
+        end
+
+        # :nodoc:
+        def async_publish(topic, payload, key=nil, partition_key=nil)
+          async_publish_list([{ topic: topic, payload: payload, key: key,
+                                partition_key: partition_key }])
+        end
+
+      private
+
+        # :nodoc:
+        def produce_messages(producer, messages)
+          messages.each do |message|
+            partition_key = message[:partition_key] || message[:key]
+            producer.produce(message[:payload],
+                             topic: message[:topic],
+                             key: message[:key],
+                             partition_key: partition_key)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/deimos/monkey_patches/ruby_kafka_heartbeat.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module Kafka
+  class Heartbeat
+    def initialize(group:, interval:, instrumenter:)
+      @group = group
+      @interval = interval
+      @last_heartbeat = Time.now
+      @instrumenter = instrumenter
+    end
+
+    def trigger!
+      @instrumenter.instrument('heartbeat.consumer',
+                               group_id: @group.group_id,
+                               topic_partitions: @group.assigned_partitions) do
+                                 @group.heartbeat
+                                 @last_heartbeat = Time.now
+                               end
+    end
+  end
+
+  class Client
+    def consumer(
+        group_id:,
+        session_timeout: 30,
+        offset_commit_interval: 10,
+        offset_commit_threshold: 0,
+        heartbeat_interval: 10,
+        offset_retention_time: nil,
+        fetcher_max_queue_size: 100
+      )
+      cluster = initialize_cluster
+
+      instrumenter = DecoratingInstrumenter.new(@instrumenter,
+                                                group_id: group_id)
+
+      # The Kafka protocol expects the retention time to be in ms.
+      retention_time = (offset_retention_time && offset_retention_time * 1_000) || -1
+
+      group = ConsumerGroup.new(
+        cluster: cluster,
+        logger: @logger,
+        group_id: group_id,
+        session_timeout: session_timeout,
+        retention_time: retention_time,
+        instrumenter: instrumenter
+      )
+
+      fetcher = Fetcher.new(
+        cluster: initialize_cluster,
+        group: group,
+        logger: @logger,
+        instrumenter: instrumenter,
+        max_queue_size: fetcher_max_queue_size
+      )
+
+      offset_manager = OffsetManager.new(
+        cluster: cluster,
+        group: group,
+        fetcher: fetcher,
+        logger: @logger,
+        commit_interval: offset_commit_interval,
+        commit_threshold: offset_commit_threshold,
+        offset_retention_time: offset_retention_time
+      )
+
+      heartbeat = Heartbeat.new(
+        group: group,
+        interval: heartbeat_interval,
+        instrumenter: instrumenter
+      )
+
+      Consumer.new(
+        cluster: cluster,
+        logger: @logger,
+        instrumenter: instrumenter,
+        group: group,
+        offset_manager: offset_manager,
+        fetcher: fetcher,
+        session_timeout: session_timeout,
+        heartbeat: heartbeat
+      )
+    end
+  end
+end

data/lib/deimos/monkey_patches/schema_store.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require 'avro_turf/schema_store'
+
+# Allows us to add in-memory schemas to the schema store in
+# addition to the ones stored in the file system.
+class AvroTurf::SchemaStore
+  attr_accessor :schemas
+
+  # @param schema_hash [Hash]
+  def add_schema(schema_hash)
+    name = schema_hash['name']
+    namespace = schema_hash['namespace']
+    full_name = Avro::Name.make_fullname(name, namespace)
+    return if @schemas.key?(full_name)
+
+    Avro::Schema.real_parse(schema_hash, @schemas)
+  end
+end

data/lib/deimos/producer.rb

@@ -0,0 +1,218 @@
+# frozen_string_literal: true
+
+require 'deimos/avro_data_encoder'
+require 'deimos/message'
+require 'deimos/shared_config'
+require 'deimos/schema_coercer'
+require 'phobos/producer'
+require 'active_support/notifications'
+
+# :nodoc:
+module Deimos
+  class << self
+    # Run a block without allowing any messages to be produced to Kafka.
+    # Optionally add a list of producer classes to limit the disabling to those
+    # classes.
+    # @param producer_classes [Array<Class>|Class]
+    def disable_producers(*producer_classes, &block)
+      if producer_classes.any?
+        _disable_producer_classes(producer_classes, &block)
+        return
+      end
+
+      if Thread.current[:frk_disable_all_producers] # nested disable block
+        yield
+        return
+      end
+
+      Thread.current[:frk_disable_all_producers] = true
+      yield
+      Thread.current[:frk_disable_all_producers] = false
+    end
+
+    # :nodoc:
+    def _disable_producer_classes(producer_classes)
+      Thread.current[:frk_disabled_producers] ||= Set.new
+      producers_to_disable = producer_classes -
+                             Thread.current[:frk_disabled_producers].to_a
+      Thread.current[:frk_disabled_producers] += producers_to_disable
+      yield
+      Thread.current[:frk_disabled_producers] -= producers_to_disable
+    end
+
+    # Are producers disabled? If a class is passed in, check only that class.
+    # Otherwise check if the global disable flag is set.
+    # @return [Boolean]
+    def producers_disabled?(producer_class=nil)
+      Thread.current[:frk_disable_all_producers] ||
+        Thread.current[:frk_disabled_producers]&.include?(producer_class)
+    end
+  end
+
+  # Producer to publish messages to a given kafka topic.
+  class Producer
+    include SharedConfig
+
+    MAX_BATCH_SIZE = 500
+
+    class << self
+      # @return [Hash]
+      def config
+        @config ||= {
+          encode_key: true,
+          namespace: Deimos.config.producer_schema_namespace
+        }
+      end
+
+      # Set the topic.
+      # @param topic [String]
+      # @return [String] the current topic if no argument given.
+      def topic(topic=nil)
+        if topic
+          config[:topic] = topic
+          return
+        end
+        # accessor
+        "#{Deimos.config.producer_topic_prefix}#{config[:topic]}"
+      end
+
+      # Override the default partition key (which is the payload key).
+      # @param _payload [Hash] the payload being passed into the produce method.
+      # Will include `payload_key` if it is part of the original payload.
+      # @return [String]
+      def partition_key(_payload)
+        nil
+      end
+
+      # Publish the payload to the topic.
+      # @param payload [Hash] with an optional payload_key hash key.
+      def publish(payload)
+        publish_list([payload])
+      end
+
+      # Publish a list of messages.
+      # @param payloads [Hash|Array<Hash>] with optional payload_key hash key.
+      # @param sync [Boolean] if given, override the default setting of
+      # whether to publish synchronously.
+      # @param force_send [Boolean] if true, ignore the configured backend
+      # and send immediately to Kafka.
+      def publish_list(payloads, sync: nil, force_send: false)
+        return if Deimos.config.seed_broker.blank? ||
+                  Deimos.config.disable_producers ||
+                  Deimos.producers_disabled?(self)
+
+        backend_class = determine_backend_class(sync, force_send)
+        Deimos.instrument(
+          'encode_messages',
+          producer: self,
+          topic: topic,
+          payloads: payloads
+        ) do
+          messages = Array(payloads).map { |p| Deimos::Message.new(p, self) }
+          messages.each(&method(:_process_message))
+          messages.in_groups_of(MAX_BATCH_SIZE, false) do |batch|
+            self.produce_batch(backend_class, batch)
+          end
+        end
+      end
+
+      # @param sync [Boolean]
+      # @param force_send [Boolean]
+      # @return [Class < Deimos::Backend]
+      def determine_backend_class(sync, force_send)
+        backend = if force_send
+                    :kafka
+                  else
+                    Deimos.config.publish_backend
+                  end
+        if backend == :kafka_async && sync
+          backend = :kafka
+        elsif backend == :kafka && sync == false
+          backend = :kafka_async
+        end
+        "Deimos::Backends::#{backend.to_s.classify}".constantize
+      end
+
+      # Send a batch to the backend.
+      # @param backend [Class < Deimos::Backend]
+      # @param batch [Array<Deimos::Message>]
+      def produce_batch(backend, batch)
+        backend.publish(producer_class: self, messages: batch)
+      end
+
+      # @return [AvroDataEncoder]
+      def encoder
+        @encoder ||= AvroDataEncoder.new(schema: config[:schema],
+                                         namespace: config[:namespace])
+      end
+
+      # @return [AvroDataEncoder]
+      def key_encoder
+        @key_encoder ||= AvroDataEncoder.new(schema: config[:key_schema],
+                                             namespace: config[:namespace])
+      end
+
+      # Override this in active record producers to add
+      # non-schema fields to check for updates
+      # @return [Array<String>] fields to check for updates
+      def watched_attributes
+        self.encoder.avro_schema.fields.map(&:name)
+      end
+
+    private
+
+      # @param message [Message]
+      def _process_message(message)
+        # this violates the Law of Demeter but it has to happen in a very
+        # specific order and requires a bunch of methods on the producer
+        # to work correctly.
+        message.add_fields(encoder.avro_schema)
+        message.partition_key = self.partition_key(message.payload)
+        message.key = _retrieve_key(message.payload)
+        # need to do this before _coerce_fields because that might result
+        # in an empty payload which is an *error* whereas this is intended.
+        message.payload = nil if message.payload.blank?
+        message.coerce_fields(encoder.avro_schema)
+        message.encoded_key = _encode_key(message.key)
+        message.topic = self.topic
+        message.encoded_payload = if message.payload.nil?
+                                    nil
+                                  else
+                                    encoder.encode(message.payload,
+                                                   topic: "#{config[:topic]}-value")
+                                  end
+      end
+
+      # @param key [Object]
+      # @return [String|Object]
+      def _encode_key(key)
+        if key.nil?
+          return nil if config[:no_keys] # no key is fine, otherwise it's a problem
+
+          raise 'No key given but a key is required! Use `key_config none: true` to avoid using keys.'
+        end
+        if config[:encode_key] && config[:key_field].nil? &&
+           config[:key_schema].nil?
+          raise 'No key config given - if you are not encoding keys, please use `key_config plain: true`'
+        end
+
+        if config[:key_field]
+          encoder.encode_key(config[:key_field], key, "#{config[:topic]}-key")
+        elsif config[:key_schema]
+          key_encoder.encode(key, topic: "#{config[:topic]}-key")
+        else
+          key
+        end
+      end
+
+      # @param payload [Hash]
+      # @return [String]
+      def _retrieve_key(payload)
+        key = payload.delete(:payload_key)
+        return key if key
+
+        config[:key_field] ? payload[config[:key_field]] : nil
+      end
+    end
+  end
+end