deimos-ruby 1.8.0.pre.beta1 → 1.8.1.pre.beta4

data/deimos-ruby.gemspec

@@ -23,7 +23,6 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency('ruby-kafka', '~> 0.7')
   spec.add_runtime_dependency('sigurd', '0.0.1')
 
-  spec.add_development_dependency('activerecord', '~> 5.2')
   spec.add_development_dependency('activerecord-import')
   spec.add_development_dependency('avro', '~> 1.9')
   spec.add_development_dependency('database_cleaner', '~> 1.7')
@@ -34,10 +33,11 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency('guard-rubocop', '~> 1')
   spec.add_development_dependency('mysql2', '~> 0.5')
   spec.add_development_dependency('pg', '~> 1.1')
-  spec.add_development_dependency('rails', '~> 5.2', '>= 5.2.4.2')
+  spec.add_development_dependency('rails', '~> 6')
   spec.add_development_dependency('rake', '~> 13')
   spec.add_development_dependency('rspec', '~> 3')
   spec.add_development_dependency('rspec_junit_formatter', '~>0.3')
+  spec.add_development_dependency('rspec-rails', '~> 4')
   spec.add_development_dependency('rubocop', '~> 0.72')
   spec.add_development_dependency('rubocop-rspec', '~> 1.27')
   spec.add_development_dependency('sqlite3', '~> 1.3')

data/lib/deimos.rb

@@ -23,14 +23,15 @@ require 'deimos/monkey_patches/phobos_producer'
 require 'deimos/monkey_patches/phobos_cli'
 
 require 'deimos/railtie' if defined?(Rails)
+require 'deimos/utils/schema_controller_mixin' if defined?(ActionController)
 
 if defined?(ActiveRecord)
   require 'deimos/kafka_source'
   require 'deimos/kafka_topic_info'
   require 'deimos/backends/db'
-  require 'sigurd/signal_handler.rb'
-  require 'sigurd/executor.rb'
-  require 'deimos/utils/db_producer.rb'
+  require 'sigurd/signal_handler'
+  require 'sigurd/executor'
+  require 'deimos/utils/db_producer'
   require 'deimos/utils/db_poller'
 end
 

data/lib/deimos/consume/batch_consumption.rb

@@ -6,6 +6,7 @@ module Deimos
     # delivery. Payloads are decoded then consumers are invoked with arrays
     # of messages to be handled at once
     module BatchConsumption
+      extend ActiveSupport::Concern
       include Phobos::BatchHandler
 
       # :nodoc:
@@ -17,6 +18,7 @@ module Deimos
               decode_key(message.key)
             end
           end
+          metadata[:first_offset] = batch.first&.offset
 
           payloads = batch.map do |message|
             message.payload ? self.class.decoder.decode(message.payload) : nil

data/lib/deimos/consume/message_consumption.rb

@@ -5,6 +5,7 @@ module Deimos
     # Methods used by message-by-message (non-batch) consumers. These consumers
     # are invoked for every individual message.
     module MessageConsumption
+      extend ActiveSupport::Concern
       include Phobos::Handler
 
       # :nodoc:

data/lib/deimos/instrumentation.rb

@@ -46,13 +46,18 @@ module Deimos
 
       messages = exception.failed_messages
       messages.group_by(&:topic).each do |topic, batch|
-        next if batch.empty?
+        producer = Deimos::Producer.descendants.find { |c| c.topic == topic }
+        next if batch.empty? || !producer
 
-        producer = batch.first.metadata[:producer_name]
-        payloads = batch.map { |m| m.metadata[:decoded_payload] }
+        decoder = Deimos.schema_backend(schema: producer.config[:schema],
+                                        namespace: producer.config[:namespace])
+        payloads = batch.map { |m| decoder.decode(m.value) }
 
-        Deimos.config.metrics&.count('publish_error', payloads.size,
-                                     tags: %W(topic:#{topic}))
+        Deimos.config.metrics&.increment(
+          'publish_error',
+          tags: %W(topic:#{topic}),
+          by: payloads.size
+        )
         Deimos.instrument(
           'produce_error',
           producer: producer,

data/lib/deimos/kafka_topic_info.rb

@@ -13,7 +13,7 @@ module Deimos
       def lock(topic, lock_id)
         # Try to create it - it's fine if it already exists
         begin
-          self.create(topic: topic)
+          self.create(topic: topic, last_processed_at: Time.zone.now)
         rescue ActiveRecord::RecordNotUnique
           # continue on
         end
@@ -52,7 +52,26 @@ module Deimos
       # @param lock_id [String]
       def clear_lock(topic, lock_id)
         self.where(topic: topic, locked_by: lock_id).
-          update_all(locked_by: nil, locked_at: nil, error: false, retries: 0)
+          update_all(locked_by: nil,
+                     locked_at: nil,
+                     error: false,
+                     retries: 0,
+                     last_processed_at: Time.zone.now)
+      end
+
+      # Update all topics that aren't currently locked and have no messages
+      # waiting. It's OK if some messages get inserted in the middle of this
+      # because the point is that at least within a few milliseconds of each
+      # other, it wasn't locked and had no messages, meaning the topic
+      # was in a good state.
+      # @param except_topics [Array<String>] the list of topics we've just
+      # realized had messages in them, meaning all other topics were empty.
+      def ping_empty_topics(except_topics)
+        records = KafkaTopicInfo.where(locked_by: nil).
+          where('topic not in(?)', except_topics)
+        records.each do |info|
+          info.update_attribute(:last_processed_at, Time.zone.now)
+        end
       end
 
       # The producer calls this if it gets an error sending messages. This

data/lib/deimos/schema_backends/avro_base.rb

@@ -33,6 +33,30 @@ module Deimos
         decode(payload, schema: @key_schema['name'])[field_name]
       end
 
+      # :nodoc:
+      def sql_type(field)
+        type = field.type.type
+        return type if %w(array map record).include?(type)
+
+        if type == :union
+          non_null = field.type.schemas.reject { |f| f.type == :null }
+          if non_null.size > 1
+            warn("WARNING: #{field.name} has more than one non-null type. Picking the first for the SQL type.")
+          end
+          return non_null.first.type
+        end
+        return type.to_sym if %w(float boolean).include?(type)
+        return :integer if type == 'int'
+        return :bigint if type == 'long'
+
+        if type == 'double'
+          warn('Avro `double` type turns into SQL `float` type. Please ensure you have the correct `limit` set.')
+          return :float
+        end
+
+        :string
+      end
+
       # @override
       def coerce_field(field, value)
         AvroSchemaCoercer.new(avro_schema).coerce_type(field.type, value)
@@ -40,7 +64,10 @@ module Deimos
 
       # @override
       def schema_fields
-        avro_schema.fields.map { |field| SchemaField.new(field.name, field.type) }
+        avro_schema.fields.map do |field|
+          enum_values = field.type.type == 'enum' ? field.type.symbols : []
+          SchemaField.new(field.name, field.type, enum_values)
+        end
       end
 
       # @override
@@ -55,6 +82,11 @@ module Deimos
         :avro_validation
       end
 
+      # @override
+      def self.content_type
+        'avro/binary'
+      end
+
     private
 
       # @param schema [String]

data/lib/deimos/schema_backends/avro_schema_coercer.rb

@@ -10,18 +10,37 @@ module Deimos
       @schema = schema
     end
 
-    # @param type [Symbol]
+    # Coerce sub-records in a payload to match the schema.
+    # @param type [Avro::Schema::UnionSchema]
+    # @param val [Object]
+    # @return [Object]
+    def coerce_union(type, val)
+      union_types = type.schemas.map { |s| s.type.to_sym }
+      return nil if val.nil? && union_types.include?(:null)
+
+      schema_type = type.schemas.find { |s| s.type.to_sym != :null }
+      coerce_type(schema_type, val)
+    end
+
+    # Coerce sub-records in a payload to match the schema.
+    # @param type [Avro::Schema::RecordSchema]
+    # @param val [Object]
+    # @return [Object]
+    def coerce_record(type, val)
+      record = val.map do |name, value|
+        field = type.fields.find { |f| f.name == name }
+        coerce_type(field.type, value)
+      end
+      val.keys.zip(record).to_h
+    end
+
+    # Coerce values in a payload to match the schema.
+    # @param type [Avro::Schema]
     # @param val [Object]
     # @return [Object]
     def coerce_type(type, val)
       int_classes = [Time, ActiveSupport::TimeWithZone]
       field_type = type.type.to_sym
-      if field_type == :union
-        union_types = type.schemas.map { |s| s.type.to_sym }
-        return nil if val.nil? && union_types.include?(:null)
-
-        field_type = union_types.find { |t| t != :null }
-      end
 
       case field_type
       when :int, :long
@@ -32,14 +51,12 @@ module Deimos
         else
           val # this will fail
         end
-
       when :float, :double
         if val.is_a?(Numeric) || _is_float_string?(val)
           val.to_f
         else
           val # this will fail
         end
-
       when :string
         if val.respond_to?(:to_str)
           val.to_s
@@ -54,6 +71,10 @@ module Deimos
         else
           true
         end
+      when :union
+        coerce_union(type, val)
+      when :record
+        coerce_record(type, val)
       else
         val
       end

data/lib/deimos/schema_backends/base.rb

@@ -3,13 +3,15 @@
 module Deimos
   # Represents a field in the schema.
   class SchemaField
-    attr_accessor :name, :type
+    attr_accessor :name, :type, :enum_values
 
     # @param name [String]
     # @param type [Object]
-    def initialize(name, type)
+    # @param enum_values [Array<String>]
+    def initialize(name, type, enum_values=[])
       @name = name
       @type = type
+      @enum_values = enum_values
     end
   end
 
@@ -69,6 +71,12 @@ module Deimos
         :mock
       end
 
+      # The content type to use when encoding / decoding requests over HTTP via ActionController.
+      # @return [String]
+      def self.content_type
+        raise NotImplementedError
+      end
+
       # Encode a payload. To be defined by subclass.
       # @param payload [Hash]
       # @param schema [Symbol|String]
@@ -109,6 +117,17 @@ module Deimos
         raise NotImplementedError
       end
 
+      # Given a field definition, return the SQL type that might be used in
+      # ActiveRecord table creation - e.g. for Avro, a `long` type would
+      # return `:bigint`. There are also special values that need to be returned:
+      # `:array`, `:map` and `:record`, for types representing those structures.
+      # `:enum` is also recognized.
+      # @param field [SchemaField]
+      # @return [Symbol]
+      def sql_type(field)
+        raise NotImplementedError
+      end
+
       # Encode a message key. To be defined by subclass.
       # @param key [String|Hash] the value to use as the key.
       # @param key_id [Symbol|String] the field name of the key.

data/lib/deimos/utils/db_producer.rb

@@ -9,6 +9,8 @@ module Deimos
       attr_accessor :id, :current_topic
 
       BATCH_SIZE = 1000
+      DELETE_BATCH_SIZE = 10
+      MAX_DELETE_ATTEMPTS = 3
 
       # @param logger [Logger]
       def initialize(logger=Logger.new(STDOUT))
@@ -48,6 +50,7 @@ module Deimos
         topics = retrieve_topics
         @logger.info("Found topics: #{topics}")
         topics.each(&method(:process_topic))
+        KafkaTopicInfo.ping_empty_topics(topics)
         sleep(0.5)
       end
 
@@ -87,13 +90,13 @@ module Deimos
           begin
             produce_messages(compacted_messages.map(&:phobos_message))
           rescue Kafka::BufferOverflow, Kafka::MessageSizeTooLarge, Kafka::RecordListTooLarge
+            delete_messages(messages)
             @logger.error('Message batch too large, deleting...')
             @logger.error(Deimos::KafkaMessage.decoded(messages))
-            Deimos::KafkaMessage.where(id: messages.map(&:id)).delete_all
             raise
           end
         end
-        Deimos::KafkaMessage.where(id: messages.map(&:id)).delete_all
+        delete_messages(messages)
         Deimos.config.metrics&.increment(
           'db_producer.process',
           tags: %W(topic:#{@current_topic}),
@@ -106,6 +109,27 @@ module Deimos
         true
       end
 
+      # @param messages [Array<Deimos::KafkaMessage>]
+      def delete_messages(messages)
+        attempts = 1
+        begin
+          messages.in_groups_of(DELETE_BATCH_SIZE, false).each do |batch|
+            Deimos::KafkaMessage.where(topic: batch.first.topic,
+                                       id: batch.map(&:id)).
+              delete_all
+          end
+        rescue StandardError => e
+          if (e.message =~ /Lock wait/i || e.message =~ /Lost connection/i) &&
+             attempts <= MAX_DELETE_ATTEMPTS
+            attempts += 1
+            ActiveRecord::Base.connection.verify!
+            sleep(1)
+            retry
+          end
+          raise
+        end
+      end
+
       # @return [Array<Deimos::KafkaMessage>]
       def retrieve_messages
         KafkaMessage.where(topic: @current_topic).order(:id).limit(BATCH_SIZE)
@@ -126,19 +150,33 @@ module Deimos
         metrics = Deimos.config.metrics
         return unless metrics
 
+        topics = KafkaTopicInfo.select(%w(topic last_processed_at))
         messages = Deimos::KafkaMessage.
           select('count(*) as num_messages, min(created_at) as earliest, topic').
-          group(:topic)
-        if messages.none?
-          metrics.gauge('pending_db_messages_max_wait', 0)
-        end
-        messages.each do |record|
-          earliest = record.earliest
-          # SQLite gives a string here
-          earliest = Time.zone.parse(earliest) if earliest.is_a?(String)
-
-          time_diff = Time.zone.now - earliest
-          metrics.gauge('pending_db_messages_max_wait', time_diff,
+          group(:topic).
+          index_by(&:topic)
+        topics.each do |record|
+          message_record = messages[record.topic]
+          # We want to record the last time we saw any activity, meaning either
+          # the oldest message, or the last time we processed, whichever comes
+          # last.
+          if message_record
+            record_earliest = record.earliest
+            # SQLite gives a string here
+            if record_earliest.is_a?(String)
+              record_earliest = Time.zone.parse(record_earliest)
+            end
+
+            earliest = [record.last_processed_at, record_earliest].max
+            time_diff = Time.zone.now - earliest
+            metrics.gauge('pending_db_messages_max_wait', time_diff,
+                          tags: ["topic:#{record.topic}"])
+          else
+            # no messages waiting
+            metrics.gauge('pending_db_messages_max_wait', 0,
+                          tags: ["topic:#{record.topic}"])
+          end
+          metrics.gauge('pending_db_messages_count', message_record&.num_messages || 0,
                         tags: ["topic:#{record.topic}"])
         end
       end
@@ -174,11 +212,11 @@ module Deimos
           end
 
           @logger.error("Got error #{e.class.name} when publishing #{batch.size} in groups of #{batch_size}, retrying...")
-          if batch_size < 10
-            batch_size = 1
-          else
-            batch_size /= 10
-          end
+          batch_size = if batch_size < 10
+                         1
+                       else
+                         (batch_size / 10).to_i
+                       end
           shutdown_producer
           retry
         end
@@ -187,7 +225,7 @@ module Deimos
       # @param batch [Array<Deimos::KafkaMessage>]
       # @return [Array<Deimos::KafkaMessage>]
       def compact_messages(batch)
-        return batch unless batch.first&.key.present?
+        return batch if batch.first&.key.blank?
 
         topic = batch.first.topic
         return batch if config.compact_topics != :all &&

data/lib/deimos/utils/schema_controller_mixin.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module Deimos
+  module Utils
+    # Mixin to automatically decode schema-encoded payloads when given the correct content type,
+    # and provide the `render_schema` method to encode the payload for responses.
+    module SchemaControllerMixin
+      extend ActiveSupport::Concern
+
+      included do
+        Mime::Type.register('avro/binary', :avro)
+
+        attr_accessor :payload
+
+        if respond_to?(:before_filter)
+          before_filter(:decode_schema, if: :schema_format?)
+        else
+          before_action(:decode_schema, if: :schema_format?)
+        end
+      end
+
+      # :nodoc:
+      module ClassMethods
+        # @return [Hash<String, Hash<Symbol, String>>]
+        def schema_mapping
+          @schema_mapping ||= {}
+        end
+
+        # Indicate which schemas should be assigned to actions.
+        # @param actions [Symbol]
+        # @param request [String]
+        # @param response [String]
+        def schemas(*actions, request: nil, response: nil)
+          actions.each do |action|
+            request ||= action.to_s.titleize
+            response ||= action.to_s.titleize
+            schema_mapping[action.to_s] = { request: request, response: response }
+          end
+        end
+
+        # @return [Hash<Symbol, String>]
+        def namespaces
+          @namespaces ||= {}
+        end
+
+        # Set the namespace for both requests and responses.
+        # @param name [String]
+        def namespace(name)
+          request_namespace(name)
+          response_namespace(name)
+        end
+
+        # Set the namespace for requests.
+        # @param name [String]
+        def request_namespace(name)
+          namespaces[:request] = name
+        end
+
+        # Set the namespace for repsonses.
+        # @param name [String]
+        def response_namespace(name)
+          namespaces[:response] = name
+        end
+      end
+
+      # @return [Boolean]
+      def schema_format?
+        request.content_type == Deimos.schema_backend_class.content_type
+      end
+
+      # Get the namespace from either an existing instance variable, or tease it out of the schema.
+      # @param type [Symbol] :request or :response
+      # @return [Array<String, String>] the namespace and schema.
+      def parse_namespace(type)
+        namespace = self.class.namespaces[type]
+        schema = self.class.schema_mapping[params['action']][type]
+        if schema.nil?
+          raise "No #{type} schema defined for #{params[:controller]}##{params[:action]}!"
+        end
+
+        if namespace.nil?
+          last_period = schema.rindex('.')
+          namespace, schema = schema.split(last_period)
+        end
+        if namespace.nil? || schema.nil?
+          raise "No request namespace defined for #{params[:controller]}##{params[:action]}!"
+        end
+
+        [namespace, schema]
+      end
+
+      # Decode the payload with the parameters.
+      def decode_schema
+        namespace, schema = parse_namespace(:request)
+        decoder = Deimos.schema_backend(schema: schema, namespace: namespace)
+        @payload = decoder.decode(request.body.read).with_indifferent_access
+        request.body.rewind if request.body.respond_to?(:rewind)
+      end
+
+      # Render a hash into a payload as specified by the configured schema and namespace.
+      # @param payload [Hash]
+      def render_schema(payload, schema: nil, namespace: nil)
+        namespace, schema = parse_namespace(:response) if !schema && !namespace
+        encoder = Deimos.schema_backend(schema: schema, namespace: namespace)
+        encoded = encoder.encode(payload)
+        response.headers['Content-Type'] = encoder.class.content_type
+        send_data(encoded)
+      end
+    end
+  end
+end