deimos-ruby 1.0.0.pre.beta22

data/lib/deimos/shared_config.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require 'active_support/concern'
+
+module Deimos
+  # Module that producers and consumers can share which sets up configuration.
+  module SharedConfig
+    extend ActiveSupport::Concern
+
+    # need to use this instead of class_methods to be backwards-compatible
+    # with Rails 3
+    module ClassMethods
+      # @return [Hash]
+      def config
+        return @config if @config
+
+        @config = {
+          encode_key: true
+        }
+        klass = self.superclass
+        while klass.respond_to?(:config)
+          klass_config = klass.config
+          if klass_config
+            # default is true for this so don't include it in the merge
+            klass_config.delete(:encode_key) if klass_config[:encode_key]
+            @config.merge!(klass_config) if klass.config
+          end
+          klass = klass.superclass
+        end
+        @config
+      end
+
+      # Set the schema.
+      # @param schema [String]
+      def schema(schema)
+        config[:schema] = schema
+      end
+
+      # Set the namespace.
+      # @param namespace [String]
+      def namespace(namespace)
+        config[:namespace] = namespace
+      end
+
+      # Set key configuration.
+      # @param field [Symbol] the name of a field to use in the value schema as
+      #   a generated key schema
+      # @param schema [String|Symbol] the name of a schema to use for the key
+      # @param plain [Boolean] if true, do not encode keys at all
+      # @param none [Boolean] if true, do not use keys at all
+      def key_config(plain: nil, field: nil, schema: nil, none: nil)
+        config[:no_keys] = none
+        config[:encode_key] = !plain && !none
+        config[:key_field] = field&.to_s
+        config[:key_schema] = schema
+      end
+    end
+  end
+end

data/lib/deimos/test_helpers.rb

@@ -0,0 +1,356 @@
+# frozen_string_literal: true
+
+require 'active_support/concern'
+require 'active_support/core_ext'
+require 'avro_turf'
+require 'deimos/tracing/mock'
+require 'deimos/metrics/mock'
+
+module Deimos
+  # Include this module in your RSpec spec_helper
+  # to stub out external dependencies
+  # and add methods to use to test encoding/decoding.
+  module TestHelpers
+    extend ActiveSupport::Concern
+
+    class << self
+      # @return [Array<Hash>]
+      def sent_messages
+        @sent_messages ||= []
+      end
+    end
+
+    included do
+      # @param encoder_schema [String]
+      # @param namespace [String]
+      # @return [Deimos::AvroDataEncoder]
+      def create_encoder(encoder_schema, namespace)
+        encoder = Deimos::AvroDataEncoder.new(schema: encoder_schema,
+                                              namespace: namespace)
+
+        # we added and_wrap_original to RSpec 2 but the regular block
+        # syntax wasn't working for some reason - block wasn't being passed
+        # to the method
+        block = proc do |m, *args|
+          m.call(*args)
+          args[0]
+        end
+        allow(encoder).to receive(:encode_local).and_wrap_original(&block)
+        allow(encoder).to receive(:encode) do |payload, schema: nil, topic: nil|
+          encoder.encode_local(payload, schema: schema)
+        end
+
+        block = proc do |m, *args|
+          m.call(*args)&.values&.first
+        end
+        allow(encoder).to receive(:encode_key).and_wrap_original(&block)
+        encoder
+      end
+
+      # @param decoder_schema [String]
+      # @param namespace [String]
+      # @return [Deimos::AvroDataDecoder]
+      def create_decoder(decoder_schema, namespace)
+        decoder = Deimos::AvroDataDecoder.new(schema: decoder_schema,
+                                              namespace: namespace)
+        allow(decoder).to receive(:decode_local) { |payload| payload }
+        allow(decoder).to receive(:decode) do |payload, schema: nil|
+          schema ||= decoder.schema
+          if schema && decoder.namespace
+            # Validate against local schema.
+            encoder = Deimos::AvroDataEncoder.new(schema: schema,
+                                                  namespace: decoder.namespace)
+            encoder.schema_store = decoder.schema_store
+            encoder.encode_local(payload)
+          end
+          payload
+        end
+        allow(decoder).to receive(:decode_key) do |payload, _key_id|
+          payload.values.first
+        end
+        decoder
+      end
+
+      RSpec.configure do |config|
+
+        config.before(:suite) do
+          Deimos.configure do |fr_config|
+            fr_config.logger = Logger.new(STDOUT)
+            fr_config.seed_broker ||= 'test_broker'
+            fr_config.tracer = Deimos::Tracing::Mock.new
+            fr_config.metrics = Deimos::Metrics::Mock.new
+          end
+        end
+
+      end
+      before(:each) do
+        client = double('client').as_null_object
+        allow(client).to receive(:time) do |*_args, &block|
+          block.call
+        end
+      end
+    end
+
+    # Stub all already-loaded producers and consumers fir unit testing purposes.
+    def stub_producers_and_consumers!
+      Deimos::TestHelpers.sent_messages.clear
+
+      allow(Deimos::Producer).to receive(:produce_batch) do |_, batch|
+        Deimos::TestHelpers.sent_messages.concat(batch.map(&:to_h))
+      end
+
+      Deimos::Producer.descendants.each do |klass|
+        next if klass == Deimos::ActiveRecordProducer # "abstract" class
+
+        stub_producer(klass)
+      end
+
+      Deimos::Consumer.descendants.each do |klass|
+        next if klass == Deimos::ActiveRecordConsumer # "abstract" class
+
+        stub_consumer(klass)
+      end
+    end
+
+    # Stub a given producer class.
+    # @param klass [Class < Deimos::Producer]
+    def stub_producer(klass)
+      allow(klass).to receive(:encoder) do
+        create_encoder(klass.config[:schema], klass.config[:namespace])
+      end
+      allow(klass).to receive(:key_encoder) do
+        create_encoder(klass.config[:key_schema], klass.config[:namespace])
+      end
+    end
+
+    # Stub a given consumer class.
+    # @param klass [Class < Deimos::Consumer]
+    def stub_consumer(klass)
+      allow(klass).to receive(:decoder) do
+        create_decoder(klass.config[:schema], klass.config[:namespace])
+      end
+      klass.class_eval do
+        alias_method(:old_consume, :consume) unless self.instance_methods.include?(:old_consume)
+      end
+      allow_any_instance_of(klass).to receive(:consume) do |instance, payload, metadata|
+        metadata[:key] = klass.new.decode_key(metadata[:key])
+        instance.old_consume(payload, metadata)
+      end
+    end
+
+    # get the difference of 2 hashes.
+    # @param hash1 [Hash]
+    # @param hash2 [Hash]
+    def _hash_diff(hash1, hash2)
+      if hash1.nil? || !hash1.is_a?(Hash)
+        hash2
+      elsif hash2.nil? || !hash2.is_a?(Hash)
+        hash1
+      else
+        hash1.dup.
+          delete_if { |k, v| hash2[k] == v }.
+          merge!(hash2.dup.delete_if { |k, _v| hash1.key?(k) })
+      end
+    end
+
+    # :nodoc:
+    def _frk_failure_message(topic, message, key=nil, partition_key=nil, was_negated=false)
+      messages = Deimos::TestHelpers.sent_messages.
+        select { |m| m[:topic] == topic }.
+        map { |m| m.except(:topic) }
+      message_string = ''
+      diff = nil
+      min_hash_diff = nil
+      if messages.any?
+        message_string = messages.map(&:inspect).join("\n")
+        min_hash_diff = messages.min_by { |m| _hash_diff(m, message).keys.size }
+        diff = RSpec::Expectations.differ.
+          diff_as_object(message, min_hash_diff[:payload])
+      end
+      description = if message.respond_to?(:description)
+                      message.description
+                    elsif message.nil?
+                      'nil'
+                    else
+                      message
+                    end
+      str = "Expected #{topic} #{'not ' if was_negated}to have sent #{description}"
+      str += " with key #{key}" if key
+      str += " with partition key #{partition_key}" if partition_key
+      str += "\nClosest message received: #{min_hash_diff}" if min_hash_diff
+      str += "\nDiff: #{diff}" if diff
+      str + "\nAll Messages received:\n#{message_string}"
+    end
+
+    RSpec::Matchers.define :have_sent do |msg, key=nil, partition_key=nil|
+      message = if msg.respond_to?(:with_indifferent_access)
+                  msg.with_indifferent_access
+                else
+                  msg
+                end
+      match do |topic|
+        Deimos::TestHelpers.sent_messages.any? do |m|
+          hash_matcher = RSpec::Matchers::BuiltIn::Match.new(message)
+          hash_matcher.send(:match,
+                            message,
+                            m[:payload]&.with_indifferent_access) &&
+            topic == m[:topic] &&
+            (key.present? ? key == m[:key] : true) &&
+            (partition_key.present? ? partition_key == m[:partition_key] : true)
+        end
+      end
+
+      if respond_to?(:failure_message)
+        failure_message do |topic|
+          _frk_failure_message(topic, message, key, partition_key)
+        end
+        failure_message_when_negated do |topic|
+          _frk_failure_message(topic, message, key, partition_key, true)
+        end
+      else
+        failure_message_for_should do |topic|
+          _frk_failure_message(topic, message, key, partition_key)
+        end
+        failure_message_for_should_not do |topic|
+          _frk_failure_message(topic, message, key, partition_key, true)
+        end
+      end
+    end
+
+    # Clear all sent messages - e.g. if we want to check that
+    # particular messages were sent or not sent after a point in time.
+    def clear_kafka_messages!
+      Deimos::TestHelpers.sent_messages.clear
+    end
+
+    # test that a message was sent on the given topic.
+    # DEPRECATED - use the "have_sent" matcher instead.
+    # @param message [Hash]
+    # @param topic [String]
+    # @param key [String|Integer]
+    # @return [Boolean]
+    def was_message_sent?(message, topic, key=nil)
+      Deimos::TestHelpers.sent_messages.any? do |m|
+        message == m[:payload] && m[:topic] == topic &&
+          (key.present? ? m[:key] == key : true)
+      end
+    end
+
+    # Test that a given handler will consume a given payload correctly, i.e.
+    # that the Avro schema is correct. If
+    # a block is given, that block will be executed when `consume` is called.
+    # Otherwise it will just confirm that `consume` is called at all.
+    # @param handler_class_or_topic [Class|String] Class which inherits from
+    # Deimos::Consumer or the topic as a string
+    # @param payload [Hash] the payload to consume
+    # @param call_original [Boolean] if true, allow the consume handler
+    # to continue as normal. Not compatible with a block.
+    # @param ignore_expectation [Boolean] Set to true to not place any
+    # expectations on the consumer. Primarily used internally to Deimos.
+    # @param key [Object] the key to use.
+    # @param partition_key [Object] the partition key to use.
+    def test_consume_message(handler_class_or_topic,
+                             payload,
+                             call_original: false,
+                             key: nil,
+                             partition_key: nil,
+                             skip_expectation: false,
+                             &block)
+      raise 'Cannot have both call_original and be given a block!' if call_original && block_given?
+
+      payload&.stringify_keys!
+      handler_class = if handler_class_or_topic.is_a?(String)
+                        _get_handler_class_from_topic(handler_class_or_topic)
+                      else
+                        handler_class_or_topic
+                      end
+      handler = handler_class.new
+      allow(handler_class).to receive(:new).and_return(handler)
+      listener = double('listener',
+                        handler_class: handler_class,
+                        encoding: nil)
+      key ||= _key_from_consumer(handler_class)
+      message = double('message',
+                       'key' => key,
+                       'partition_key' => partition_key,
+                       'partition' => 1,
+                       'offset' => 1,
+                       'value' => payload)
+
+      unless skip_expectation
+        expectation = expect(handler).to receive(:consume).
+          with(payload, anything, &block)
+        expectation.and_call_original if call_original
+      end
+      Phobos::Actions::ProcessMessage.new(
+        listener: listener,
+        message: message,
+        listener_metadata: { topic: 'my-topic' }
+      ).send(:process_message, payload)
+    end
+
+    # Check to see that a given message will fail due to Avro errors.
+    # @param handler_class [Class]
+    # @param payload [Hash]
+    def test_consume_invalid_message(handler_class, payload)
+      handler = handler_class.new
+      allow(handler_class).to receive(:new).and_return(handler)
+      listener = double('listener',
+                        handler_class: handler_class,
+                        encoding: nil)
+      message = double('message',
+                       key: _key_from_consumer(handler_class),
+                       partition_key: nil,
+                       partition: 1,
+                       offset: 1,
+                       value: payload)
+
+      expect {
+        Phobos::Actions::ProcessMessage.new(
+          listener: listener,
+          message: message,
+          listener_metadata: { topic: 'my-topic' }
+        ).send(:process_message, payload)
+      }.to raise_error(Avro::SchemaValidator::ValidationError)
+    end
+
+    # @param schema1 [String|Hash] a file path, JSON string, or
+    # hash representing a schema.
+    # @param schema2 [String|Hash] a file path, JSON string, or
+    # hash representing a schema.
+    # @return [Boolean] true if the schemas are compatible, false otherwise.
+    def self.schemas_compatible?(schema1, schema2)
+      json1, json2 = [schema1, schema2].map do |schema|
+        if schema.is_a?(String)
+          schema = File.read(schema) unless schema.strip.starts_with?('{') # file path
+          MultiJson.load(schema)
+        else
+          schema
+        end
+      end
+      avro_schema1 = Avro::Schema.real_parse(json1, {})
+      avro_schema2 = Avro::Schema.real_parse(json2, {})
+      Avro::SchemaCompatibility.mutual_read?(avro_schema1, avro_schema2)
+    end
+
+  private
+
+    def _key_from_consumer(consumer)
+      if consumer.config[:key_field] || consumer.config[:key_schema]
+        { 'test' => 1 }
+      else
+        1
+      end
+    end
+
+    # @param topic [String]
+    # @return [Class]
+    def _get_handler_class_from_topic(topic)
+      listeners = Phobos.config['listeners']
+      handler = listeners.find { |l| l.topic == topic }
+      raise "No consumer found in Phobos configuration for topic #{topic}!" if handler.nil?
+
+      handler.handler.constantize
+    end
+  end
+end

data/lib/deimos/tracing/datadog.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require 'deimos/tracing/provider'
+
+module Deimos
+  module Tracing
+    # Tracing wrapper class for Datadog.
+    class Datadog < Tracing::Provider
+      # :nodoc:
+      def initialize(config)
+        raise 'Tracing config must specify service_name' if config[:service_name].nil?
+
+        @service = config[:service_name]
+      end
+
+      # :nodoc:
+      def start(span_name, options={})
+        span = ::Datadog.tracer.trace(span_name)
+        span.service = @service
+        span.resource = options[:resource]
+        span
+      end
+
+      # :nodoc:
+      def finish(span)
+        span.finish
+      end
+
+      # :nodoc:
+      def set_error(span, exception)
+        span.set_error(exception)
+      end
+    end
+  end
+end

data/lib/deimos/tracing/mock.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'deimos/tracing/provider'
+
+module Deimos
+  module Tracing
+    # Class that mocks out tracing functionality
+    class Mock < Tracing::Provider
+      # :nodoc:
+      def initialize(logger=nil)
+        @logger = logger || Logger.new(STDOUT)
+        @logger.info('MockTracingProvider initialized')
+      end
+
+      # :nodoc:
+      def start(span_name, _options={})
+        @logger.info("Mock span '#{span_name}' started")
+        {
+          name: span_name,
+          started_at: Time.zone.now
+        }
+      end
+
+      # :nodoc:
+      def finish(span)
+        name = span[:name]
+        start = span[:started_at]
+        finish = Time.zone.now
+        @logger.info("Mock span '#{name}' finished: #{start} to #{finish}")
+      end
+
+      # :nodoc:
+      def set_error(span, exception)
+        span[:exception] = exception
+        name = span[:name]
+        @logger.info("Mock span '#{name}' set an error: #{exception}")
+      end
+    end
+  end
+end

data/lib/deimos/tracing/provider.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+# rubocop:disable Lint/UnusedMethodArgument
+module Deimos
+  module Tracing
+    # Base class for all tracing providers.
+    class Provider
+      # Returns a span object and starts the trace.
+      # @param span_name [String] The name of the span/trace
+      # @param options [Hash] Options for the span
+      # @return [Object] The span object
+      def start(span_name, options={})
+        raise NotImplementedError
+      end
+
+      # Finishes the trace on the span object.
+      # @param span [Object] The span to finish trace on
+      def finish(span)
+        raise NotImplementedError
+      end
+
+      # Set an error on the span.
+      # @param span [Object] The span to set error on
+      # @param exception [Exception] The exception that occurred
+      def set_error(span, exception)
+        raise NotImplementedError
+      end
+    end
+  end
+end
+# rubocop:enable Lint/UnusedMethodArgument

data/lib/deimos/utils/db_producer.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+module Deimos
+  module Utils
+    # Class which continually polls the database and sends Kafka messages.
+    class DbProducer
+      include Phobos::Producer
+      attr_accessor :id, :current_topic
+
+      BATCH_SIZE = 1000
+
+      # @param logger [Logger]
+      def initialize(logger=Logger.new(STDOUT))
+        @id = SecureRandom.uuid
+        @logger = logger
+        @logger.push_tags("DbProducer #{@id}") if @logger.respond_to?(:push_tags)
+      end
+
+      # Start the poll.
+      def start
+        @logger.info('Starting...')
+        @signal_to_stop = false
+        loop do
+          if @signal_to_stop
+            @logger.info('Shutting down')
+            break
+          end
+          send_pending_metrics
+          process_next_messages
+        end
+      end
+
+      # Stop the poll.
+      def stop
+        @logger.info('Received signal to stop')
+        @signal_to_stop = true
+      end
+
+      # Complete one loop of processing all messages in the DB.
+      def process_next_messages
+        topics = retrieve_topics
+        @logger.info("Found topics: #{topics}")
+        topics.each(&method(:process_topic))
+        sleep(0.5)
+      end
+
+      # @return [Array<String>]
+      def retrieve_topics
+        KafkaMessage.select('distinct topic').map(&:topic).uniq
+      end
+
+      # @param topic [String]
+      # @return [String] the topic that was locked, or nil if none were.
+      def process_topic(topic)
+        # If the topic is already locked, another producer is currently
+        # working on it. Move on to the next one.
+        unless KafkaTopicInfo.lock(topic, @id)
+          @logger.debug("Could not lock topic #{topic} - continuing")
+          return
+        end
+        @current_topic = topic
+        messages = retrieve_messages
+
+        while messages.any?
+          produce_messages(messages.map(&:phobos_message))
+          messages.first.class.where(id: messages.map(&:id)).delete_all
+          break if messages.size < BATCH_SIZE
+
+          KafkaTopicInfo.heartbeat(@current_topic, @id) # keep alive
+          send_pending_metrics
+          messages = retrieve_messages
+        end
+        KafkaTopicInfo.clear_lock(@current_topic, @id)
+      rescue StandardError => e
+        @logger.error("Error processing messages for topic #{@current_topic}: #{e.class.name}: #{e.message} #{e.backtrace.join("\n")}")
+        KafkaTopicInfo.register_error(@current_topic, @id)
+      end
+
+      # @return [Array<Deimos::KafkaMessage>]
+      def retrieve_messages
+        KafkaMessage.where(topic: @current_topic).order(:id).limit(BATCH_SIZE)
+      end
+
+      # Send metrics to Datadog.
+      def send_pending_metrics
+        first_message = KafkaMessage.first
+        time_diff = first_message ? Time.zone.now - KafkaMessage.first.created_at : 0
+        Deimos.config.metrics&.gauge('pending_db_messages_max_wait', time_diff)
+      end
+
+      # @param batch [Array<Hash>]
+      def produce_messages(batch)
+        batch_size = batch.size
+        begin
+          batch.in_groups_of(batch_size, false).each do |group|
+            @logger.debug("Publishing #{group.size} messages to #{@current_topic}")
+            producer.publish_list(group)
+            Deimos.config.metrics&.increment(
+              'publish',
+              tags: %W(status:success topic:#{@current_topic}),
+              by: group.size
+            )
+            @logger.info("Sent #{group.size} messages to #{@current_topic}")
+          end
+        rescue Kafka::BufferOverflow
+          raise if batch_size == 1
+
+          @logger.error("Buffer overflow when publishing #{batch.size} in groups of #{batch_size}, retrying...")
+          if batch_size < 10
+            batch_size = 1
+          else
+            batch_size /= 10
+          end
+          if self.class.producer.respond_to?(:sync_producer_shutdown) # Phobos 1.8.3
+            self.class.producer.sync_producer_shutdown
+          end
+          retry
+        end
+      end
+    end
+  end
+end