deimos-ruby 1.7.0.pre.beta1 → 1.8.1.pre.beta3

data/lib/deimos/schema_backends/base.rb

@@ -3,13 +3,15 @@
 module Deimos
   # Represents a field in the schema.
   class SchemaField
-    attr_accessor :name, :type
+    attr_accessor :name, :type, :enum_values
 
     # @param name [String]
     # @param type [Object]
-    def initialize(name, type)
+    # @param enum_values [Array<String>]
+    def initialize(name, type, enum_values=[])
       @name = name
       @type = type
+      @enum_values = enum_values
     end
   end
 
@@ -69,6 +71,12 @@ module Deimos
         :mock
       end
 
+      # The content type to use when encoding / decoding requests over HTTP via ActionController.
+      # @return [String]
+      def self.content_type
+        raise NotImplementedError
+      end
+
       # Encode a payload. To be defined by subclass.
       # @param payload [Hash]
       # @param schema [Symbol|String]
@@ -109,6 +117,17 @@ module Deimos
         raise NotImplementedError
       end
 
+      # Given a field definition, return the SQL type that might be used in
+      # ActiveRecord table creation - e.g. for Avro, a `long` type would
+      # return `:bigint`. There are also special values that need to be returned:
+      # `:array`, `:map` and `:record`, for types representing those structures.
+      # `:enum` is also recognized.
+      # @param field [SchemaField]
+      # @return [Symbol]
+      def sql_type(field)
+        raise NotImplementedError
+      end
+
       # Encode a message key. To be defined by subclass.
       # @param key [String|Hash] the value to use as the key.
       # @param key_id [Symbol|String] the field name of the key.

data/lib/deimos/utils/db_poller.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 require 'deimos/poll_info'
-require 'deimos/utils/executor'
-require 'deimos/utils/signal_handler'
+require 'sigurd/executor'
+require 'sigurd/signal_handler'
 
 module Deimos
   module Utils
@@ -22,10 +22,10 @@ module Deimos
         pollers = Deimos.config.db_poller_objects.map do |poller_config|
           self.new(poller_config)
         end
-        executor = Deimos::Utils::Executor.new(pollers,
-                                               sleep_seconds: 5,
-                                               logger: Deimos.config.logger)
-        signal_handler = Deimos::Utils::SignalHandler.new(executor)
+        executor = Sigurd::Executor.new(pollers,
+                                        sleep_seconds: 5,
+                                        logger: Deimos.config.logger)
+        signal_handler = Sigurd::SignalHandler.new(executor)
         signal_handler.run!
       end
 

data/lib/deimos/utils/db_producer.rb

@@ -9,6 +9,8 @@ module Deimos
       attr_accessor :id, :current_topic
 
       BATCH_SIZE = 1000
+      DELETE_BATCH_SIZE = 10
+      MAX_DELETE_ATTEMPTS = 3
 
       # @param logger [Logger]
       def initialize(logger=Logger.new(STDOUT))
@@ -48,6 +50,7 @@ module Deimos
         topics = retrieve_topics
         @logger.info("Found topics: #{topics}")
         topics.each(&method(:process_topic))
+        KafkaTopicInfo.ping_empty_topics(topics)
         sleep(0.5)
       end
 
@@ -87,13 +90,13 @@ module Deimos
           begin
             produce_messages(compacted_messages.map(&:phobos_message))
           rescue Kafka::BufferOverflow, Kafka::MessageSizeTooLarge, Kafka::RecordListTooLarge
-            Deimos::KafkaMessage.where(id: messages.map(&:id)).delete_all
+            delete_messages(messages)
             @logger.error('Message batch too large, deleting...')
             @logger.error(Deimos::KafkaMessage.decoded(messages))
             raise
           end
         end
-        Deimos::KafkaMessage.where(id: messages.map(&:id)).delete_all
+        delete_messages(messages)
         Deimos.config.metrics&.increment(
           'db_producer.process',
           tags: %W(topic:#{@current_topic}),
@@ -106,6 +109,27 @@ module Deimos
         true
       end
 
+      # @param messages [Array<Deimos::KafkaMessage>]
+      def delete_messages(messages)
+        attempts = 1
+        begin
+          messages.in_groups_of(DELETE_BATCH_SIZE, false).each do |batch|
+            Deimos::KafkaMessage.where(topic: batch.first.topic,
+                                       id: batch.map(&:id)).
+              delete_all
+          end
+        rescue StandardError => e
+          if (e.message =~ /Lock wait/i || e.message =~ /Lost connection/i) &&
+             attempts <= MAX_DELETE_ATTEMPTS
+            attempts += 1
+            ActiveRecord::Base.connection.verify!
+            sleep(1)
+            retry
+          end
+          raise
+        end
+      end
+
       # @return [Array<Deimos::KafkaMessage>]
       def retrieve_messages
         KafkaMessage.where(topic: @current_topic).order(:id).limit(BATCH_SIZE)
@@ -126,15 +150,33 @@ module Deimos
         metrics = Deimos.config.metrics
         return unless metrics
 
+        topics = KafkaTopicInfo.select(%w(topic last_processed_at))
         messages = Deimos::KafkaMessage.
           select('count(*) as num_messages, min(created_at) as earliest, topic').
-          group(:topic)
-        if messages.none?
-          metrics.gauge('pending_db_messages_max_wait', 0)
-        end
-        messages.each do |record|
-          time_diff = Time.zone.now - record.earliest
-          metrics.gauge('pending_db_messages_max_wait', time_diff,
+          group(:topic).
+          index_by(&:topic)
+        topics.each do |record|
+          message_record = messages[record.topic]
+          # We want to record the last time we saw any activity, meaning either
+          # the oldest message, or the last time we processed, whichever comes
+          # last.
+          if message_record
+            record_earliest = record.earliest
+            # SQLite gives a string here
+            if record_earliest.is_a?(String)
+              record_earliest = Time.zone.parse(record_earliest)
+            end
+
+            earliest = [record.last_processed_at, record_earliest].max
+            time_diff = Time.zone.now - earliest
+            metrics.gauge('pending_db_messages_max_wait', time_diff,
+                          tags: ["topic:#{record.topic}"])
+          else
+            # no messages waiting
+            metrics.gauge('pending_db_messages_max_wait', 0,
+                          tags: ["topic:#{record.topic}"])
+          end
+          metrics.gauge('pending_db_messages_count', message_record&.num_messages || 0,
                         tags: ["topic:#{record.topic}"])
         end
       end
@@ -170,11 +212,11 @@ module Deimos
           end
 
           @logger.error("Got error #{e.class.name} when publishing #{batch.size} in groups of #{batch_size}, retrying...")
-          if batch_size < 10
-            batch_size = 1
-          else
-            batch_size /= 10
-          end
+          batch_size = if batch_size < 10
+                         1
+                       else
+                         (batch_size / 10).to_i
+                       end
           shutdown_producer
           retry
         end
@@ -183,7 +225,7 @@ module Deimos
       # @param batch [Array<Deimos::KafkaMessage>]
       # @return [Array<Deimos::KafkaMessage>]
       def compact_messages(batch)
-        return batch unless batch.first&.key.present?
+        return batch if batch.first&.key.blank?
 
         topic = batch.first.topic
         return batch if config.compact_topics != :all &&

data/lib/deimos/utils/deadlock_retry.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Deimos
+  module Utils
+    # Utility class to retry a given block if a a deadlock is encountered.
+    # Supports Postgres and MySQL deadlocks and lock wait timeouts.
+    class DeadlockRetry
+      class << self
+        # Maximum number of times to retry the block after encountering a deadlock
+        RETRY_COUNT = 2
+
+        # Need to match on error messages to support older Rails versions
+        DEADLOCK_MESSAGES = [
+          # MySQL
+          'Deadlock found when trying to get lock',
+          'Lock wait timeout exceeded',
+
+          # Postgres
+          'deadlock detected'
+        ].freeze
+
+        # Retry the given block when encountering a deadlock. For any other
+        # exceptions, they are reraised. This is used to handle cases where
+        # the database may be busy but the transaction would succeed if
+        # retried later. Note that your block should be idempotent and it will
+        # be wrapped in a transaction.
+        # Sleeps for a random number of seconds to prevent multiple transactions
+        # from retrying at the same time.
+        # @param tags [Array] Tags to attach when logging and reporting metrics.
+        # @yield Yields to the block that may deadlock.
+        def wrap(tags=[])
+          count = RETRY_COUNT
+
+          begin
+            ActiveRecord::Base.transaction do
+              yield
+            end
+          rescue ActiveRecord::StatementInvalid => e
+            # Reraise if not a known deadlock
+            raise if DEADLOCK_MESSAGES.none? { |m| e.message.include?(m) }
+
+            # Reraise if all retries exhausted
+            raise if count <= 0
+
+            Deimos.config.logger.warn(
+              message: 'Deadlock encountered when trying to execute query. '\
+                "Retrying. #{count} attempt(s) remaining",
+              tags: tags
+            )
+
+            Deimos.config.metrics&.increment(
+              'deadlock',
+              tags: tags
+            )
+
+            count -= 1
+
+            # Sleep for a random amount so that if there are multiple
+            # transactions deadlocking, they don't all retry at the same time
+            sleep(Random.rand(5.0) + 0.5)
+
+            retry
+          end
+        end
+      end
+    end
+  end
+end

data/lib/deimos/utils/lag_reporter.rb

@@ -29,30 +29,21 @@ module Deimos
           self.topics[topic.to_s].report_lag(partition)
         end
 
-        # @param topic [String]
-        # @param partition [Integer]
-        # @param lag [Integer]
-        def assign_lag(topic, partition, lag)
-          self.topics[topic.to_s] ||= Topic.new(topic, self)
-          self.topics[topic.to_s].assign_lag(partition, lag)
-        end
-
-        # Figure out the current lag by asking Kafka based on the current offset.
         # @param topic [String]
         # @param partition [Integer]
         # @param offset [Integer]
-        def compute_lag(topic, partition, offset)
+        def assign_current_offset(topic, partition, offset)
           self.topics[topic.to_s] ||= Topic.new(topic, self)
-          self.topics[topic.to_s].compute_lag(partition, offset)
+          self.topics[topic.to_s].assign_current_offset(partition, offset)
         end
       end
 
-      # Topic which has a hash of partition => last known offset lag
+      # Topic which has a hash of partition => last known current offsets
       class Topic
         # @return [String]
         attr_accessor :topic_name
         # @return [Hash<Integer, Integer>]
-        attr_accessor :partition_offset_lags
+        attr_accessor :partition_current_offsets
         # @return [ConsumerGroup]
         attr_accessor :consumer_group
 
@@ -61,35 +52,33 @@ module Deimos
         def initialize(topic_name, group)
           self.topic_name = topic_name
           self.consumer_group = group
-          self.partition_offset_lags = {}
+          self.partition_current_offsets = {}
         end
 
         # @param partition [Integer]
-        # @param lag [Integer]
-        def assign_lag(partition, lag)
-          self.partition_offset_lags[partition.to_i] = lag
+        def assign_current_offset(partition, offset)
+          self.partition_current_offsets[partition.to_i] = offset
         end
 
         # @param partition [Integer]
-        # @param offset [Integer]
         def compute_lag(partition, offset)
-          return if self.partition_offset_lags[partition.to_i]
-
           begin
             client = Phobos.create_kafka_client
             last_offset = client.last_offset_for(self.topic_name, partition)
-            assign_lag(partition, [last_offset - offset, 0].max)
+            lag = last_offset - offset
           rescue StandardError # don't do anything, just wait
             Deimos.config.logger.
               debug("Error computing lag for #{self.topic_name}, will retry")
           end
+          lag || 0
         end
 
         # @param partition [Integer]
         def report_lag(partition)
-          lag = self.partition_offset_lags[partition.to_i]
-          return unless lag
+          current_offset = self.partition_current_offsets[partition.to_i]
+          return unless current_offset
 
+          lag = compute_lag(partition, current_offset)
           group = self.consumer_group.id
           Deimos.config.logger.
             debug("Sending lag: #{group}/#{partition}: #{lag}")
@@ -109,16 +98,20 @@ module Deimos
           @groups = {}
         end
 
+        # offset_lag = event.payload.fetch(:offset_lag)
+        # group_id = event.payload.fetch(:group_id)
+        # topic = event.payload.fetch(:topic)
+        # partition = event.payload.fetch(:partition)
         # @param payload [Hash]
         def message_processed(payload)
-          lag = payload[:offset_lag]
+          offset = payload[:offset] || payload[:last_offset]
           topic = payload[:topic]
           group = payload[:group_id]
           partition = payload[:partition]
 
           synchronize do
             @groups[group.to_s] ||= ConsumerGroup.new(group)
-            @groups[group.to_s].assign_lag(topic, partition, lag)
+            @groups[group.to_s].assign_current_offset(topic, partition, offset)
           end
         end
 
@@ -131,7 +124,7 @@ module Deimos
 
           synchronize do
             @groups[group.to_s] ||= ConsumerGroup.new(group)
-            @groups[group.to_s].compute_lag(topic, partition, offset)
+            @groups[group.to_s].assign_current_offset(topic, partition, offset)
           end
         end
 

data/lib/deimos/utils/schema_controller_mixin.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module Deimos
+  module Utils
+    # Mixin to automatically decode schema-encoded payloads when given the correct content type,
+    # and provide the `render_schema` method to encode the payload for responses.
+    module SchemaControllerMixin
+      extend ActiveSupport::Concern
+
+      included do
+        Mime::Type.register('avro/binary', :avro)
+
+        attr_accessor :payload
+
+        if respond_to?(:before_filter)
+          before_filter(:decode_schema, if: :schema_format?)
+        else
+          before_action(:decode_schema, if: :schema_format?)
+        end
+      end
+
+      # :nodoc:
+      module ClassMethods
+        # @return [Hash<String, Hash<Symbol, String>>]
+        def schema_mapping
+          @schema_mapping ||= {}
+        end
+
+        # Indicate which schemas should be assigned to actions.
+        # @param actions [Symbol]
+        # @param request [String]
+        # @param response [String]
+        def schemas(*actions, request: nil, response: nil)
+          actions.each do |action|
+            request ||= action.to_s.titleize
+            response ||= action.to_s.titleize
+            schema_mapping[action.to_s] = { request: request, response: response }
+          end
+        end
+
+        # @return [Hash<Symbol, String>]
+        def namespaces
+          @namespaces ||= {}
+        end
+
+        # Set the namespace for both requests and responses.
+        # @param name [String]
+        def namespace(name)
+          request_namespace(name)
+          response_namespace(name)
+        end
+
+        # Set the namespace for requests.
+        # @param name [String]
+        def request_namespace(name)
+          namespaces[:request] = name
+        end
+
+        # Set the namespace for repsonses.
+        # @param name [String]
+        def response_namespace(name)
+          namespaces[:response] = name
+        end
+      end
+
+      # @return [Boolean]
+      def schema_format?
+        request.content_type == Deimos.schema_backend_class.content_type
+      end
+
+      # Get the namespace from either an existing instance variable, or tease it out of the schema.
+      # @param type [Symbol] :request or :response
+      # @return [Array<String, String>] the namespace and schema.
+      def parse_namespace(type)
+        namespace = self.class.namespaces[type]
+        schema = self.class.schema_mapping[params['action']][type]
+        if schema.nil?
+          raise "No #{type} schema defined for #{params[:controller]}##{params[:action]}!"
+        end
+
+        if namespace.nil?
+          last_period = schema.rindex('.')
+          namespace, schema = schema.split(last_period)
+        end
+        if namespace.nil? || schema.nil?
+          raise "No request namespace defined for #{params[:controller]}##{params[:action]}!"
+        end
+
+        [namespace, schema]
+      end
+
+      # Decode the payload with the parameters.
+      def decode_schema
+        namespace, schema = parse_namespace(:request)
+        decoder = Deimos.schema_backend(schema: schema, namespace: namespace)
+        @payload = decoder.decode(request.body.read).with_indifferent_access
+        request.body.rewind if request.body.respond_to?(:rewind)
+      end
+
+      # Render a hash into a payload as specified by the configured schema and namespace.
+      # @param payload [Hash]
+      def render_schema(payload, schema: nil, namespace: nil)
+        namespace, schema = parse_namespace(:response) if !schema && !namespace
+        encoder = Deimos.schema_backend(schema: schema, namespace: namespace)
+        encoded = encoder.encode(payload)
+        response.headers['Content-Type'] = encoder.class.content_type
+        send_data(encoded)
+      end
+    end
+  end
+end