deimos-temp-fork 0.0.1

data/lib/deimos/metrics/datadog.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require 'deimos/metrics/provider'
+
+module Deimos
+  module Metrics
+    # A Metrics wrapper class for Datadog.
+    class Datadog < Metrics::Provider
+      # :nodoc:
+      def initialize(config, logger)
+        raise 'Metrics config must specify host_ip' if config[:host_ip].nil?
+        raise 'Metrics config must specify host_port' if config[:host_port].nil?
+        raise 'Metrics config must specify namespace' if config[:namespace].nil?
+
+        logger.info("DatadogMetricsProvider configured with: #{config}")
+        @client = Datadog::Statsd.new(
+          config[:host_ip],
+          config[:host_port]
+        )
+        @client.tags = config[:tags]
+        @client.namespace = config[:namespace]
+      end
+
+      # :nodoc:
+      def increment(metric_name, options={})
+        @client.increment(metric_name, options)
+      end
+
+      # :nodoc:
+      def gauge(metric_name, count, options={})
+        @client.gauge(metric_name, count, options)
+      end
+
+      # :nodoc:
+      def histogram(metric_name, count, options={})
+        @client.histogram(metric_name, count, options)
+      end
+
+      # :nodoc:
+      def time(metric_name, options={})
+        @client.time(metric_name, options) do
+          yield
+        end
+      end
+    end
+  end
+end

data/lib/deimos/metrics/mock.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'deimos/metrics/provider'
+
+module Deimos
+  module Metrics
+    # A mock Metrics wrapper which just logs the metrics
+    class Mock
+      # :nodoc:
+      def initialize(logger=nil)
+        @logger = logger || Logger.new(STDOUT)
+        @logger.info('MockMetricsProvider initialized')
+      end
+
+      # :nodoc:
+      def increment(metric_name, options={})
+        @logger.info("MockMetricsProvider.increment: #{metric_name}, #{options}")
+      end
+
+      # :nodoc:
+      def gauge(metric_name, count, options={})
+        @logger.info("MockMetricsProvider.gauge: #{metric_name}, #{count}, #{options}")
+      end
+
+      # :nodoc:
+      def histogram(metric_name, count, options={})
+        @logger.info("MockMetricsProvider.histogram: #{metric_name}, #{count}, #{options}")
+      end
+
+      # :nodoc:
+      def time(metric_name, options={})
+        start_time = Time.now
+        yield
+        total_time = (Time.now - start_time).to_i
+        @logger.info("MockMetricsProvider.time: #{metric_name}, #{total_time}, #{options}")
+      end
+    end
+  end
+end

data/lib/deimos/metrics/provider.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Deimos
+  module Metrics
+    # Base class for all metrics providers.
+    class Provider
+      # Send an counter increment metric
+      # @param metric_name [String] The name of the counter metric
+      # @param options [Hash] Any additional options, e.g. :tags
+      def increment(metric_name, options={})
+        raise NotImplementedError
+      end
+
+      # Send an counter increment metric
+      # @param metric_name [String] The name of the counter metric
+      # @param options [Hash] Any additional options, e.g. :tags
+      def gauge(metric_name, count, options={})
+        raise NotImplementedError
+      end
+
+      # Send an counter increment metric
+      # @param metric_name [String] The name of the counter metric
+      # @param options [Hash] Any additional options, e.g. :tags
+      def histogram(metric_name, count, options={})
+        raise NotImplementedError
+      end
+
+      # Time a yielded block, and send a timer metric
+      # @param metric_name [String] The name of the metric
+      # @param options [Hash] Any additional options, e.g. :tags
+      def time(metric_name, options={})
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/deimos/monkey_patches/phobos_cli.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require 'phobos/cli/start'
+
+# :nodoc:
+module Phobos
+  # :nodoc:
+  module CLI
+    # :nodoc:
+    class Start
+      # :nodoc:
+      def validate_listeners!
+        Phobos.config.listeners.each do |listener|
+          handler = listener.handler
+          begin
+            handler.constantize
+          rescue NameError
+            error_exit("Handler '#{handler}' not defined")
+          end
+
+          delivery = listener.delivery
+          if delivery.nil?
+            Phobos::CLI.logger.warn do
+              Hash(message: "Delivery option should be specified, defaulting to 'batch'"\
+               ' - specify this option to silence this message')
+            end
+          elsif !Listener::DELIVERY_OPTS.include?(delivery)
+            error_exit("Invalid delivery option '#{delivery}'. Please specify one of: "\
+              "#{Listener::DELIVERY_OPTS.join(', ')}")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/deimos/monkey_patches/phobos_producer.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'phobos/producer'
+
+module Phobos
+  module Producer
+    # :nodoc:
+    class PublicAPI
+      # :nodoc:
+      def publish(topic, payload, key=nil, partition_key=nil)
+        class_producer.publish(topic, payload, key, partition_key)
+      end
+
+      # :nodoc:
+      def async_publish(topic, payload, key=nil, partition_key=nil)
+        class_producer.async_publish(topic, payload, key, partition_key)
+      end
+    end
+
+    # :nodoc:
+    module ClassMethods
+      # :nodoc:
+      class PublicAPI
+        # :nodoc:
+        def publish(topic, payload, key=nil, partition_key=nil)
+          publish_list([{ topic: topic, payload: payload, key: key,
+                          partition_key: partition_key }])
+        end
+
+        # :nodoc:
+        def async_publish(topic, payload, key=nil, partition_key=nil)
+          async_publish_list([{ topic: topic, payload: payload, key: key,
+                                partition_key: partition_key }])
+        end
+
+      private
+
+        # :nodoc:
+        def produce_messages(producer, messages)
+          messages.each do |message|
+            partition_key = message[:partition_key] || message[:key]
+            producer.produce(message[:payload],
+                             topic: message[:topic],
+                             key: message[:key],
+                             partition_key: partition_key)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/deimos/poll_info.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Deimos
+  # ActiveRecord class to record the last time we polled the database.
+  # For use with DbPoller.
+  class PollInfo < ActiveRecord::Base
+    self.table_name = 'deimos_poll_info'
+  end
+end

data/lib/deimos/producer.rb

@@ -0,0 +1,224 @@
+# frozen_string_literal: true
+
+require 'deimos/message'
+require 'deimos/shared_config'
+require 'phobos/producer'
+require 'active_support/notifications'
+
+# :nodoc:
+module Deimos
+  class << self
+    # Run a block without allowing any messages to be produced to Kafka.
+    # Optionally add a list of producer classes to limit the disabling to those
+    # classes.
+    # @param producer_classes [Array<Class>|Class]
+    def disable_producers(*producer_classes, &block)
+      if producer_classes.any?
+        _disable_producer_classes(producer_classes, &block)
+        return
+      end
+
+      if Thread.current[:frk_disable_all_producers] # nested disable block
+        yield
+        return
+      end
+
+      begin
+        Thread.current[:frk_disable_all_producers] = true
+        yield
+      ensure
+        Thread.current[:frk_disable_all_producers] = false
+      end
+    end
+
+    # :nodoc:
+    def _disable_producer_classes(producer_classes)
+      Thread.current[:frk_disabled_producers] ||= Set.new
+      producers_to_disable = producer_classes -
+                             Thread.current[:frk_disabled_producers].to_a
+      Thread.current[:frk_disabled_producers] += producers_to_disable
+      yield
+      Thread.current[:frk_disabled_producers] -= producers_to_disable
+    end
+
+    # Are producers disabled? If a class is passed in, check only that class.
+    # Otherwise check if the global disable flag is set.
+    # @return [Boolean]
+    def producers_disabled?(producer_class=nil)
+      Thread.current[:frk_disable_all_producers] ||
+        Thread.current[:frk_disabled_producers]&.include?(producer_class)
+    end
+  end
+
+  # Producer to publish messages to a given kafka topic.
+  class Producer
+    include SharedConfig
+
+    MAX_BATCH_SIZE = 500
+
+    class << self
+      # @return [Hash]
+      def config
+        @config ||= {
+          encode_key: true,
+          namespace: Deimos.config.producers.schema_namespace
+        }
+      end
+
+      # Set the topic.
+      # @param topic [String]
+      # @return [String] the current topic if no argument given.
+      def topic(topic=nil)
+        if topic
+          config[:topic] = topic
+          return
+        end
+        # accessor
+        "#{Deimos.config.producers.topic_prefix}#{config[:topic]}"
+      end
+
+      # Override the default partition key (which is the payload key).
+      # @param _payload [Hash] the payload being passed into the produce method.
+      # Will include `payload_key` if it is part of the original payload.
+      # @return [String]
+      def partition_key(_payload)
+        nil
+      end
+
+      # Publish the payload to the topic.
+      # @param payload [Hash] with an optional payload_key hash key.
+      # @param topic [String] if specifying the topic
+      def publish(payload, topic: self.topic)
+        publish_list([payload], topic: topic)
+      end
+
+      # Publish a list of messages.
+      # @param payloads [Hash|Array<Hash>] with optional payload_key hash key.
+      # @param sync [Boolean] if given, override the default setting of
+      # whether to publish synchronously.
+      # @param force_send [Boolean] if true, ignore the configured backend
+      # and send immediately to Kafka.
+      # @param topic [String] if specifying the topic
+      def publish_list(payloads, sync: nil, force_send: false, topic: self.topic)
+        return if Deimos.config.kafka.seed_brokers.blank? ||
+                  Deimos.config.producers.disabled ||
+                  Deimos.producers_disabled?(self)
+
+        raise 'Topic not specified. Please specify the topic.' if topic.blank?
+
+        backend_class = determine_backend_class(sync, force_send)
+        Deimos.instrument(
+          'encode_messages',
+          producer: self,
+          topic: topic,
+          payloads: payloads
+        ) do
+          messages = Array(payloads).map { |p| Deimos::Message.new(p, self) }
+          messages.each { |m| _process_message(m, topic) }
+          messages.in_groups_of(MAX_BATCH_SIZE, false) do |batch|
+            self.produce_batch(backend_class, batch)
+          end
+        end
+      end
+
+      # @param sync [Boolean]
+      # @param force_send [Boolean]
+      # @return [Class < Deimos::Backend]
+      def determine_backend_class(sync, force_send)
+        backend = if force_send
+                    :kafka
+                  else
+                    Deimos.config.producers.backend
+                  end
+        if backend == :kafka_async && sync
+          backend = :kafka
+        elsif backend == :kafka && sync == false
+          backend = :kafka_async
+        end
+        "Deimos::Backends::#{backend.to_s.classify}".constantize
+      end
+
+      # Send a batch to the backend.
+      # @param backend [Class < Deimos::Backend]
+      # @param batch [Array<Deimos::Message>]
+      def produce_batch(backend, batch)
+        backend.publish(producer_class: self, messages: batch)
+      end
+
+      # @return [Deimos::SchemaBackends::Base]
+      def encoder
+        @encoder ||= Deimos.schema_backend(schema: config[:schema],
+                                           namespace: config[:namespace])
+      end
+
+      # @return [Deimos::SchemaBackends::Base]
+      def key_encoder
+        @key_encoder ||= Deimos.schema_backend(schema: config[:key_schema],
+                                               namespace: config[:namespace])
+      end
+
+      # Override this in active record producers to add
+      # non-schema fields to check for updates
+      # @return [Array<String>] fields to check for updates
+      def watched_attributes
+        self.encoder.schema_fields.map(&:name)
+      end
+
+    private
+
+      # @param message [Message]
+      # @param topic [String]
+      def _process_message(message, topic)
+        # this violates the Law of Demeter but it has to happen in a very
+        # specific order and requires a bunch of methods on the producer
+        # to work correctly.
+        message.add_fields(encoder.schema_fields.map(&:name))
+        message.partition_key = self.partition_key(message.payload)
+        message.key = _retrieve_key(message.payload)
+        # need to do this before _coerce_fields because that might result
+        # in an empty payload which is an *error* whereas this is intended.
+        message.payload = nil if message.payload.blank?
+        message.coerce_fields(encoder)
+        message.encoded_key = _encode_key(message.key)
+        message.topic = topic
+        message.encoded_payload = if message.payload.nil?
+                                    nil
+                                  else
+                                    encoder.encode(message.payload,
+                                                   topic: "#{Deimos.config.producers.topic_prefix}#{config[:topic]}-value")
+                                  end
+      end
+
+      # @param key [Object]
+      # @return [String|Object]
+      def _encode_key(key)
+        if key.nil?
+          return nil if config[:no_keys] # no key is fine, otherwise it's a problem
+
+          raise 'No key given but a key is required! Use `key_config none: true` to avoid using keys.'
+        end
+        if config[:encode_key] && config[:key_field].nil? &&
+           config[:key_schema].nil?
+          raise 'No key config given - if you are not encoding keys, please use `key_config plain: true`'
+        end
+
+        if config[:key_field]
+          encoder.encode_key(config[:key_field], key, topic: "#{Deimos.config.producers.topic_prefix}#{config[:topic]}-key")
+        elsif config[:key_schema]
+          key_encoder.encode(key, topic: "#{Deimos.config.producers.topic_prefix}#{config[:topic]}-key")
+        else
+          key
+        end
+      end
+
+      # @param payload [Hash]
+      # @return [String]
+      def _retrieve_key(payload)
+        key = payload.delete(:payload_key)
+        return key if key
+
+        config[:key_field] ? payload[config[:key_field]] : nil
+      end
+    end
+  end
+end