deimos-ruby 1.6.3 → 1.8.1.pre.beta1

data/lib/deimos/utils/db_producer.rb

@@ -2,12 +2,15 @@
 
 module Deimos
   module Utils
-    # Class which continually polls the database and sends Kafka messages.
+    # Class which continually polls the kafka_messages table
+    # in the database and sends Kafka messages.
     class DbProducer
       include Phobos::Producer
       attr_accessor :id, :current_topic
 
       BATCH_SIZE = 1000
+      DELETE_BATCH_SIZE = 10
+      MAX_DELETE_ATTEMPTS = 3
 
       # @param logger [Logger]
       def initialize(logger=Logger.new(STDOUT))
@@ -47,6 +50,7 @@ module Deimos
         topics = retrieve_topics
         @logger.info("Found topics: #{topics}")
         topics.each(&method(:process_topic))
+        KafkaTopicInfo.ping_empty_topics(topics)
         sleep(0.5)
       end
 
@@ -86,13 +90,13 @@ module Deimos
           begin
             produce_messages(compacted_messages.map(&:phobos_message))
           rescue Kafka::BufferOverflow, Kafka::MessageSizeTooLarge, Kafka::RecordListTooLarge
-            Deimos::KafkaMessage.where(id: messages.map(&:id)).delete_all
+            delete_messages(messages)
             @logger.error('Message batch too large, deleting...')
             @logger.error(Deimos::KafkaMessage.decoded(messages))
             raise
           end
         end
-        Deimos::KafkaMessage.where(id: messages.map(&:id)).delete_all
+        delete_messages(messages)
         Deimos.config.metrics&.increment(
           'db_producer.process',
           tags: %W(topic:#{@current_topic}),
@@ -105,6 +109,27 @@ module Deimos
         true
       end
 
+      # @param messages [Array<Deimos::KafkaMessage>]
+      def delete_messages(messages)
+        attempts = 1
+        begin
+          messages.in_groups_of(DELETE_BATCH_SIZE, false).each do |batch|
+            Deimos::KafkaMessage.where(topic: batch.first.topic,
+                                       id: batch.map(&:id)).
+              delete_all
+          end
+        rescue StandardError => e
+          if (e.message =~ /Lock wait/i || e.message =~ /Lost connection/i) &&
+             attempts <= MAX_DELETE_ATTEMPTS
+            attempts += 1
+            ActiveRecord::Base.connection.verify!
+            sleep(1)
+            retry
+          end
+          raise
+        end
+      end
+
       # @return [Array<Deimos::KafkaMessage>]
       def retrieve_messages
         KafkaMessage.where(topic: @current_topic).order(:id).limit(BATCH_SIZE)
@@ -125,15 +150,33 @@ module Deimos
         metrics = Deimos.config.metrics
         return unless metrics
 
+        topics = KafkaTopicInfo.select(%w(topic last_processed_at))
         messages = Deimos::KafkaMessage.
           select('count(*) as num_messages, min(created_at) as earliest, topic').
-          group(:topic)
-        if messages.none?
-          metrics.gauge('pending_db_messages_max_wait', 0)
-        end
-        messages.each do |record|
-          time_diff = Time.zone.now - record.earliest
-          metrics.gauge('pending_db_messages_max_wait', time_diff,
+          group(:topic).
+          index_by(&:topic)
+        topics.each do |record|
+          message_record = messages[record.topic]
+          # We want to record the last time we saw any activity, meaning either
+          # the oldest message, or the last time we processed, whichever comes
+          # last.
+          if message_record
+            record_earliest = record.earliest
+            # SQLite gives a string here
+            if record_earliest.is_a?(String)
+              record_earliest = Time.zone.parse(record_earliest)
+            end
+
+            earliest = [record.last_processed_at, record_earliest].max
+            time_diff = Time.zone.now - earliest
+            metrics.gauge('pending_db_messages_max_wait', time_diff,
+                          tags: ["topic:#{record.topic}"])
+          else
+            # no messages waiting
+            metrics.gauge('pending_db_messages_max_wait', 0,
+                          tags: ["topic:#{record.topic}"])
+          end
+          metrics.gauge('pending_db_messages_count', message_record&.num_messages || 0,
                         tags: ["topic:#{record.topic}"])
         end
       end
@@ -169,11 +212,11 @@ module Deimos
           end
 
           @logger.error("Got error #{e.class.name} when publishing #{batch.size} in groups of #{batch_size}, retrying...")
-          if batch_size < 10
-            batch_size = 1
-          else
-            batch_size /= 10
-          end
+          batch_size = if batch_size < 10
+                         1
+                       else
+                         (batch_size / 10).to_i
+                       end
           shutdown_producer
           retry
         end
@@ -182,7 +225,7 @@ module Deimos
       # @param batch [Array<Deimos::KafkaMessage>]
       # @return [Array<Deimos::KafkaMessage>]
       def compact_messages(batch)
-        return batch unless batch.first&.key.present?
+        return batch if batch.first&.key.blank?
 
         topic = batch.first.topic
         return batch if config.compact_topics != :all &&

data/lib/deimos/utils/deadlock_retry.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Deimos
+  module Utils
+    # Utility class to retry a given block if a a deadlock is encountered.
+    # Supports Postgres and MySQL deadlocks and lock wait timeouts.
+    class DeadlockRetry
+      class << self
+        # Maximum number of times to retry the block after encountering a deadlock
+        RETRY_COUNT = 2
+
+        # Need to match on error messages to support older Rails versions
+        DEADLOCK_MESSAGES = [
+          # MySQL
+          'Deadlock found when trying to get lock',
+          'Lock wait timeout exceeded',
+
+          # Postgres
+          'deadlock detected'
+        ].freeze
+
+        # Retry the given block when encountering a deadlock. For any other
+        # exceptions, they are reraised. This is used to handle cases where
+        # the database may be busy but the transaction would succeed if
+        # retried later. Note that your block should be idempotent and it will
+        # be wrapped in a transaction.
+        # Sleeps for a random number of seconds to prevent multiple transactions
+        # from retrying at the same time.
+        # @param tags [Array] Tags to attach when logging and reporting metrics.
+        # @yield Yields to the block that may deadlock.
+        def wrap(tags=[])
+          count = RETRY_COUNT
+
+          begin
+            ActiveRecord::Base.transaction do
+              yield
+            end
+          rescue ActiveRecord::StatementInvalid => e
+            # Reraise if not a known deadlock
+            raise if DEADLOCK_MESSAGES.none? { |m| e.message.include?(m) }
+
+            # Reraise if all retries exhausted
+            raise if count <= 0
+
+            Deimos.config.logger.warn(
+              message: 'Deadlock encountered when trying to execute query. '\
+                "Retrying. #{count} attempt(s) remaining",
+              tags: tags
+            )
+
+            Deimos.config.metrics&.increment(
+              'deadlock',
+              tags: tags
+            )
+
+            count -= 1
+
+            # Sleep for a random amount so that if there are multiple
+            # transactions deadlocking, they don't all retry at the same time
+            sleep(Random.rand(5.0) + 0.5)
+
+            retry
+          end
+        end
+      end
+    end
+  end
+end

data/lib/deimos/utils/lag_reporter.rb

@@ -29,30 +29,21 @@ module Deimos
           self.topics[topic.to_s].report_lag(partition)
         end
 
-        # @param topic [String]
-        # @param partition [Integer]
-        # @param lag [Integer]
-        def assign_lag(topic, partition, lag)
-          self.topics[topic.to_s] ||= Topic.new(topic, self)
-          self.topics[topic.to_s].assign_lag(partition, lag)
-        end
-
-        # Figure out the current lag by asking Kafka based on the current offset.
         # @param topic [String]
         # @param partition [Integer]
         # @param offset [Integer]
-        def compute_lag(topic, partition, offset)
+        def assign_current_offset(topic, partition, offset)
           self.topics[topic.to_s] ||= Topic.new(topic, self)
-          self.topics[topic.to_s].compute_lag(partition, offset)
+          self.topics[topic.to_s].assign_current_offset(partition, offset)
         end
       end
 
-      # Topic which has a hash of partition => last known offset lag
+      # Topic which has a hash of partition => last known current offsets
       class Topic
         # @return [String]
         attr_accessor :topic_name
         # @return [Hash<Integer, Integer>]
-        attr_accessor :partition_offset_lags
+        attr_accessor :partition_current_offsets
         # @return [ConsumerGroup]
         attr_accessor :consumer_group
 
@@ -61,35 +52,33 @@ module Deimos
         def initialize(topic_name, group)
           self.topic_name = topic_name
           self.consumer_group = group
-          self.partition_offset_lags = {}
+          self.partition_current_offsets = {}
         end
 
         # @param partition [Integer]
-        # @param lag [Integer]
-        def assign_lag(partition, lag)
-          self.partition_offset_lags[partition.to_i] = lag
+        def assign_current_offset(partition, offset)
+          self.partition_current_offsets[partition.to_i] = offset
         end
 
         # @param partition [Integer]
-        # @param offset [Integer]
         def compute_lag(partition, offset)
-          return if self.partition_offset_lags[partition.to_i]
-
           begin
             client = Phobos.create_kafka_client
             last_offset = client.last_offset_for(self.topic_name, partition)
-            assign_lag(partition, [last_offset - offset, 0].max)
+            lag = last_offset - offset
           rescue StandardError # don't do anything, just wait
             Deimos.config.logger.
               debug("Error computing lag for #{self.topic_name}, will retry")
           end
+          lag || 0
         end
 
         # @param partition [Integer]
         def report_lag(partition)
-          lag = self.partition_offset_lags[partition.to_i]
-          return unless lag
+          current_offset = self.partition_current_offsets[partition.to_i]
+          return unless current_offset
 
+          lag = compute_lag(partition, current_offset)
           group = self.consumer_group.id
           Deimos.config.logger.
             debug("Sending lag: #{group}/#{partition}: #{lag}")
@@ -109,16 +98,20 @@ module Deimos
           @groups = {}
         end
 
+        # offset_lag = event.payload.fetch(:offset_lag)
+        # group_id = event.payload.fetch(:group_id)
+        # topic = event.payload.fetch(:topic)
+        # partition = event.payload.fetch(:partition)
         # @param payload [Hash]
         def message_processed(payload)
-          lag = payload[:offset_lag]
+          offset = payload[:offset] || payload[:last_offset]
           topic = payload[:topic]
           group = payload[:group_id]
           partition = payload[:partition]
 
           synchronize do
             @groups[group.to_s] ||= ConsumerGroup.new(group)
-            @groups[group.to_s].assign_lag(topic, partition, lag)
+            @groups[group.to_s].assign_current_offset(topic, partition, offset)
           end
         end
 
@@ -131,7 +124,7 @@ module Deimos
 
           synchronize do
             @groups[group.to_s] ||= ConsumerGroup.new(group)
-            @groups[group.to_s].compute_lag(topic, partition, offset)
+            @groups[group.to_s].assign_current_offset(topic, partition, offset)
           end
         end
 

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.6.3'
+  VERSION = '1.8.1-beta1'
 end

data/lib/generators/deimos/active_record/templates/migration.rb.tt

@@ -0,0 +1,28 @@
+class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
+  def up
+    if table_exists?(:<%= table_name %>)
+      warn "<%= table_name %> already exists, exiting"
+      return
+    end
+    create_table :<%= table_name %> do |t|
+      <%- fields.each do |key| -%>
+      <%- next if %w(id message_id timestamp).include?(key.name) -%>
+      <%- sql_type = schema_base.sql_type(key)
+         if %w(record array map).include?(sql_type)
+           conn = ActiveRecord::Base.connection
+           sql_type = conn.respond_to?(:supports_json?) && conn.supports_json? ? :json : :string
+         end
+      -%>
+      t.<%= sql_type %> :<%= key.name %>
+      <%- end -%>
+    end
+
+    # TODO add indexes as necessary
+  end
+
+  def down
+    return unless table_exists?(:<%= table_name %>)
+    drop_table :<%= table_name %>
+  end
+
+end

data/lib/generators/deimos/active_record/templates/model.rb.tt

@@ -0,0 +1,5 @@
+class <%= table_name.classify %> < ApplicationRecord
+<%- fields.select { |f| f.enum_values.any? }.each do |field| -%>
+  enum <%= field.name %>: {<%= field.enum_values.map { |v| "#{v}: '#{v}'"}.join(', ') %>}
+<% end -%>
+end

data/lib/generators/deimos/active_record_generator.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require 'rails/generators'
+require 'rails/generators/active_record/migration'
+require 'rails/version'
+
+# Generates a new consumer.
+module Deimos
+  module Generators
+    # Generator for ActiveRecord model and migration.
+    class ActiveRecordGenerator < Rails::Generators::Base
+      include Rails::Generators::Migration
+      if Rails.version < '4'
+        extend(ActiveRecord::Generators::Migration)
+      else
+        include ActiveRecord::Generators::Migration
+      end
+      source_root File.expand_path('active_record/templates', __dir__)
+
+      argument :table_name, desc: 'The table to create.', required: true
+      argument :full_schema, desc: 'The fully qualified schema name.', required: true
+
+      no_commands do
+
+        # @return [String]
+        def db_migrate_path
+          if defined?(Rails.application) && Rails.application
+            paths = Rails.application.config.paths['db/migrate']
+            paths.respond_to?(:to_ary) ? paths.to_ary.first : paths.to_a.first
+          else
+            'db/migrate'
+          end
+        end
+
+        # @return [String]
+        def migration_version
+          "[#{ActiveRecord::Migration.current_version}]"
+        rescue StandardError
+          ''
+        end
+
+        # @return [String]
+        def table_class
+          self.table_name.classify
+        end
+
+        # @return [String]
+        def schema
+          last_dot = self.full_schema.rindex('.')
+          self.full_schema[last_dot + 1..-1]
+        end
+
+        # @return [String]
+        def namespace
+          last_dot = self.full_schema.rindex('.')
+          self.full_schema[0...last_dot]
+        end
+
+        # @return [Deimos::SchemaBackends::Base]
+        def schema_base
+          @schema_base ||= Deimos.schema_backend_class.new(schema: schema, namespace: namespace)
+        end
+
+        # @return [Array<SchemaField>]
+        def fields
+          schema_base.schema_fields
+        end
+
+      end
+
+      desc 'Generate migration for a table based on an existing schema.'
+      # :nodoc:
+      def generate
+        migration_template('migration.rb', "db/migrate/create_#{table_name.underscore}.rb")
+        template('model.rb', "app/models/#{table_name.underscore}.rb")
+      end
+    end
+  end
+end

data/lib/generators/deimos/db_backend/templates/migration

@@ -16,6 +16,7 @@ class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version
       t.datetime :locked_at
       t.boolean :error, null: false, default: false
       t.integer :retries, null: false, default: 0
+      t.datetime :last_processed_at
     end
     add_index :kafka_topic_info, :topic, unique: true
     add_index :kafka_topic_info, [:locked_by, :error]

data/lib/generators/deimos/db_backend/templates/rails3_migration

@@ -16,6 +16,7 @@ class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version
       t.datetime :locked_at
       t.boolean :error, null: false, default: false
       t.integer :retries, null: false, default: 0
+      t.datetime :last_processed_at
     end
     add_index :kafka_topic_info, :topic, unique: true
     add_index :kafka_topic_info, [:locked_by, :error]