deimos-ruby 1.0.0 → 1.1.0.pre.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 65652367bda550248f06a89305f831f4edeaef1cdd1fc2f87f21b80b5c124adf
-  data.tar.gz: b58ca940506703e6c1f39edd1a8b9c058d711fa47925696e799593e7d741ce7d
+  metadata.gz: b9cb2755df1f6588b130a9171d427a54301e7e5258d6f414d8e9950393b0a361
+  data.tar.gz: dc475fcb4e450a693501b6c3a148c93dec90e0c40212e40de991e2aa7b8afb4f
 SHA512:
-  metadata.gz: b646a55133358dfe0e188cdc47338610b327eb5bd4a56cfb1e329d73f1798c00f85ac9b07eb6c675fceee1b0dedecd9f5d2c1028c10bc5d270856d33ba55a271
-  data.tar.gz: 8c4d4557d4a1a4aebfc9d55159c68c26864a87d065e68b14975f1843318851b024cc711ee12ed85ba1b0dcc1a14d7cedf80eba2be5961fdcd0d33dae3aa7141b
+  metadata.gz: 8b7594655e6051986ddc55efc7a5df32fafd3e445324b8d2592eeb406cbf607cf401b6c6ab9c4d7c76d6d8cfa3c8890d47400231dec15f330dbffb0341644a59
+  data.tar.gz: bc909ec5cb051ee58e6f7840a1d191e9881f72e3442d9eba35be14e28fada2ff070f318fd5d73d46eca3bdc05f7ed05e0b0ac4d8a98f8ff4f29377854dd129ca

data/.rubocop.yml

@@ -179,6 +179,9 @@ Style/GuardClause:
 Style/HashSyntax:
   EnforcedStyle: ruby19_no_mixed_keys
 
+Style/IfUnlessModifier:
+  Enabled: false
+
 # Allow the following:
 # var x = "foo" +
 #   "bar"

data/CHANGELOG.md

@@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+# [1.1.0-beta1] - 2019-09-10
+- Added BatchConsumer.
+
 ## [1.0.0] - 2019-09-03
 - Official release of Deimos 1.0!
 

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    deimos-ruby (1.0.0)
+    deimos-ruby (1.1.0.pre.beta1)
       avro-patches (~> 0.3)
       avro_turf (~> 0.8)
-      phobos (~> 1.8)
+      phobos (~> 1.8.2.pre.beta2)
       ruby-kafka (~> 0.7)
 
 GEM
@@ -87,7 +87,7 @@ GEM
     parser (2.6.3.0)
       ast (~> 2.4.0)
     pg (1.1.3)
-    phobos (1.8.1)
+    phobos (1.8.2.pre.beta2)
       activesupport (>= 3.0.0)
       concurrent-ruby (>= 1.0.2)
       concurrent-ruby-ext (>= 1.0.2)

data/README.md

@@ -21,6 +21,7 @@ Built on Phobos and hence Ruby-Kafka.
         * [Kafka Message Keys](#kafka-message-keys)
    * [Consumers](#consumers)
    * [Rails Integration](#rails-integration)
+   * [Database Backend](#database-backend)
    * [Running Consumers](#running-consumers)
    * [Metrics](#metrics)
    * [Testing](#testing)
@@ -335,8 +336,53 @@ class MyConsumer < Deimos::Consumer
   def consume(payload, metadata)
     # Same method as Phobos consumers.
     # payload is an Avro-decoded hash.
-    # Metadata is a hash that contains information like :key and :topic. Both
-    # key (if configured) and payload will be Avro-decoded. 
+    # metadata is a hash that contains information like :key and :topic.
+    # In general, your key should be included in the payload itself. However,
+    # if you need to access it separately from the payload, you can use
+    # metadata[:key]
+  end
+end
+```
+
+### Batch Consumption
+
+Instead of consuming messages one at a time, consumers can receive a batch of
+messages as an array and then process them together. This can improve
+consumer throughput, depending on the use case. Batch consumers behave like
+other consumers in regards to key and payload decoding, etc.
+
+To enable batch consumption, create a listener in `phobos.yml` and ensure that
+the `delivery` property is set to `inline_batch`. For example:
+
+```yaml
+listeners:
+  - handler: Consumers::MyBatchConsumer
+    topic: my_batched_topic
+    group_id: my_group_id
+    delivery: inline_batch
+```
+
+Batch consumers must inherit from the Deimos::BatchConsumer class as in
+this sample:
+
+```ruby
+class MyBatchConsumer < Deimos::BatchConsumer
+
+  # See the Consumer sample in the previous section
+  schema 'MySchema'
+  namespace 'com.my-namespace'
+  key_config field: :my_id 
+
+  def consume_batch(payloads, metadata)
+    # payloads is an array of Avro-decoded hashes.
+    # metadata is a hash that contains information like :keys and :topic.
+    # Keys are automatically decoded and available as an array with
+    # the same cardinality as the payloads. If you need to iterate
+    # over payloads and keys together, you can use something like this:
+ 
+    payloads.zip(metadata[:keys]) do |_payload, _key|
+      # Do something 
+    end
   end
 end
 ```
@@ -454,7 +500,7 @@ class Widget < ActiveRecord::Base
 end
 ```
 
-#### Database Backend
+# Database Backend
 
 Deimos provides a way to allow Kafka messages to be created inside a
 database transaction, and send them asynchronously. This ensures that your
@@ -479,6 +525,7 @@ of immediately sending to Kafka. Now, you just need to call
 
     Deimos.start_db_backend!
     
+You can do this inside a thread or fork block.
 If using Rails, you can use a Rake task to do this:
 
     rails deimos:db_producer
@@ -495,6 +542,16 @@ If you want to force a message to send immediately, just call the `publish_list`
 method with `force_send: true`. You can also pass `force_send` into any of the
 other methods that publish events, like `send_event` in `ActiveRecordProducer`.
 
+A couple of gotchas when using this feature:
+* This may result in high throughput depending on your scale. If you're
+  using Rails < 5.1, you should add a migration to change the `id` column
+  to `BIGINT`. Rails >= 5.1 sets it to BIGINT by default.
+* This table is high throughput but should generally be empty. Make sure
+  you optimize/vacuum this table regularly to reclaim the disk space.
+* Currently, threads allow you to scale the *number* of topics but not
+  a single large topic with lots of messages. There is an [issue](https://github.com/flipp-oss/deimos/issues/23)
+  opened that would help with this case.
+
 For more information on how the database backend works and why it was 
 implemented, please see [Database Backends](docs/DATABASE_BACKEND.md).
 
@@ -562,7 +619,17 @@ The following metrics are reported:
     * `status:success`
     * `status:error`
     * `time:consume` (histogram)
+        * Amount of time spent executing handler for each message
+    * Batch Consumers - report counts by number of batches
+        * `status:batch_received`
+        * `status:batch_success`
+        * `status:batch_error`
+        * `time:consume_batch` (histogram)
+            * Amount of time spent executing handler for entire batch
     * `time:time_delayed` (histogram)
+        * Indicates the amount of time between the `timestamp` property of each
+        payload (if present) and the time that the consumer started processing 
+        the message.
 * `publish` - a count of the number of messages received. Tagged
   with `topic:{topic_name}`
 * `publish_error` - a count of the number of messages which failed
@@ -655,6 +722,15 @@ test_consume_message(MyConsumer,
 test_consume_invalid_message(MyConsumer, 
                             { 'some-invalid-payload' => 'some-value' })
                             
+# For batch consumers, there are similar methods such as:
+test_consume_batch(MyBatchConsumer,
+                   [{ 'some-payload' => 'some-value' },
+                    { 'some-payload' => 'some-other-value' }]) do |payloads, metadata|
+  # Expectations here
+end
+
+## Producing
+                            
 # A matcher which allows you to test that a message was sent on the given
 # topic, without having to know which class produced it.                         
 expect(topic_name).to have_sent(payload, key=nil)

data/deimos-ruby.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency('avro-patches', '~> 0.3')
   spec.add_runtime_dependency('avro_turf', '~> 0.8')
-  spec.add_runtime_dependency('phobos', '~> 1.8')
+  spec.add_runtime_dependency('phobos', '~> 1.8.2.pre.beta2')
   spec.add_runtime_dependency('ruby-kafka', '~> 0.7')
 
   spec.add_development_dependency('activerecord', '~> 5.2')

data/lib/deimos/base_consumer.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module Deimos
+  # Shared methods for Kafka Consumers
+  class BaseConsumer
+    include SharedConfig
+
+    class << self
+      # @return [AvroDataEncoder]
+      def decoder
+        @decoder ||= AvroDataDecoder.new(schema: config[:schema],
+                                         namespace: config[:namespace])
+      end
+
+      # @return [AvroDataEncoder]
+      def key_decoder
+        @key_decoder ||= AvroDataDecoder.new(schema: config[:key_schema],
+                                             namespace: config[:namespace])
+      end
+    end
+
+    # Helper method to decode an Avro-encoded key.
+    # @param key [String]
+    # @return [Object] the decoded key.
+    def decode_key(key)
+      return nil if key.nil?
+
+      config = self.class.config
+      if config[:encode_key] && config[:key_field].nil? &&
+         config[:key_schema].nil?
+        raise 'No key config given - if you are not decoding keys, please use '\
+          '`key_config plain: true`'
+      end
+
+      if config[:key_field]
+        self.class.decoder.decode_key(key, config[:key_field])
+      elsif config[:key_schema]
+        self.class.key_decoder.decode(key, schema: config[:key_schema])
+      else # no encoding
+        key
+      end
+    end
+
+  protected
+
+    # @param payload [Hash|String]
+    # @param metadata [Hash]
+    def _with_error_span(payload, metadata)
+      @span = Deimos.config.tracer&.start(
+        'deimos-consumer',
+        resource: self.class.name.gsub('::', '-')
+      )
+      yield
+    rescue StandardError => e
+      _handle_error(e, payload, metadata)
+    ensure
+      Deimos.config.tracer&.finish(@span)
+    end
+
+    def _report_time_delayed(payload, metadata)
+      return if payload.nil? || payload['timestamp'].blank?
+
+      begin
+        time_delayed = Time.now.in_time_zone - payload['timestamp'].to_datetime
+      rescue ArgumentError
+        Deimos.config.logger.info(
+          message: "Error parsing timestamp! #{payload['timestamp']}"
+        )
+        return
+      end
+      Deimos.config.metrics&.histogram('handler', time_delayed, tags: %W(
+                                         time:time_delayed
+                                         topic:#{metadata[:topic]}
+                                       ))
+    end
+
+    # @param exception [Throwable]
+    # @param _payload [Hash]
+    # @param _metadata [Hash]
+    def _handle_error(exception, _payload, _metadata)
+      Deimos.config.tracer&.set_error(@span, exception)
+
+      raise if Deimos.config.reraise_consumer_errors
+    end
+
+    # @param _time_taken [Float]
+    # @param _payload [Hash]
+    # @param _metadata [Hash]
+    def _handle_success(_time_taken, _payload, _metadata)
+      raise NotImplementedError
+    end
+  end
+end

data/lib/deimos/batch_consumer.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+require 'deimos/avro_data_decoder'
+require 'deimos/base_consumer'
+require 'phobos/batch_handler'
+
+module Deimos
+  # Class to consume batches of messages in a topic
+  # Note: According to the docs, instances of your handler will be created
+  # for every incoming batch of messages. This class should be lightweight.
+  class BatchConsumer < BaseConsumer
+    include Phobos::BatchHandler
+
+    # :nodoc:
+    def around_consume_batch(payloads, metadata)
+      _received_batch(payloads, metadata)
+      benchmark = Benchmark.measure do
+        _with_error_span(payloads, metadata) { yield }
+      end
+      _handle_success(benchmark.real, payloads, metadata)
+    end
+
+    # :nodoc:
+    def before_consume_batch(batch, metadata)
+      _with_error_span(batch, metadata) do
+        metadata[:keys] = batch.map do |message|
+          decode_key(message.key)
+        end
+
+        batch.map do |message|
+          self.class.decoder.decode(message.payload) if message.payload.present?
+        end
+      end
+    end
+
+    # Consume a batch of incoming messages.
+    # @param _payloads [Array<Phobos::BatchMessage>]
+    # @param _metadata [Hash]
+    def consume_batch(_payloads, _metadata)
+      raise NotImplementedError
+    end
+
+  protected
+
+    def _received_batch(payloads, metadata)
+      Deimos.config.logger.info(
+        message: 'Got Kafka batch event',
+        message_ids: _payload_identifiers(payloads, metadata),
+        metadata: metadata.except(:keys)
+      )
+      Deimos.config.logger.debug(
+        message: 'Kafka batch event payloads',
+        payloads: payloads
+      )
+      Deimos.config.metrics&.increment(
+        'handler',
+        tags: %W(
+          status:batch_received
+          topic:#{metadata[:topic]}
+        ))
+      Deimos.config.metrics&.increment(
+        'handler',
+        by: metadata['batch_size'],
+        tags: %W(
+          status:received
+          topic:#{metadata[:topic]}
+        ))
+      if payloads.present?
+        payloads.each { |payload| _report_time_delayed(payload, metadata) }
+      end
+    end
+
+    # @param exception [Throwable]
+    # @param payloads [Array<Hash>]
+    # @param metadata [Hash]
+    def _handle_error(exception, payloads, metadata)
+      Deimos.config.metrics&.increment(
+        'handler',
+        tags: %W(
+          status:batch_error
+          topic:#{metadata[:topic]}
+        ))
+      Deimos.config.logger.warn(
+        message: 'Error consuming message batch',
+        handler: self.class.name,
+        metadata: metadata.except(:keys),
+        message_ids: _payload_identifiers(payloads, metadata),
+        error_message: exception.message,
+        error: exception.backtrace
+      )
+      super
+    end
+
+    # @param time_taken [Float]
+    # @param payloads [Array<Hash>]
+    # @param metadata [Hash]
+    def _handle_success(time_taken, payloads, metadata)
+      Deimos.config.metrics&.histogram('handler', time_taken, tags: %W(
+                                         time:consume_batch
+                                         topic:#{metadata[:topic]}
+                                       ))
+      Deimos.config.metrics&.increment(
+        'handler',
+        tags: %W(
+          status:batch_success
+          topic:#{metadata[:topic]}
+        ))
+      Deimos.config.metrics&.increment(
+        'handler',
+        by: metadata['batch_size'],
+        tags: %W(
+          status:success
+          topic:#{metadata[:topic]}
+        ))
+      Deimos.config.logger.info(
+        message: 'Finished processing Kafka batch event',
+        message_ids: _payload_identifiers(payloads, metadata),
+        time_elapsed: time_taken,
+        metadata: metadata.except(:keys)
+      )
+    end
+
+    # Get payload identifiers (key and message_id if present) for logging.
+    # @param payloads [Array<Hash>]
+    # @param metadata [Hash]
+    # @return [Hash] the identifiers.
+    def _payload_identifiers(payloads, metadata)
+      message_ids = payloads&.map do |payload|
+        if payload.is_a?(Hash) && payload.key?('message_id')
+          payload['message_id']
+        end
+      end
+
+      # Payloads may be nil if preprocessing failed
+      messages = payloads || metadata[:keys] || []
+
+      messages.zip(metadata[:keys] || [], message_ids || []).map do |_, k, m_id|
+        ids = {}
+
+        ids[:key] = k if k.present?
+        ids[:message_id] = m_id if m_id.present?
+
+        ids
+      end
+    end
+  end
+end

data/lib/deimos/consumer.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'deimos/avro_data_decoder'
+require 'deimos/base_consumer'
 require 'deimos/shared_config'
 require 'phobos/handler'
 require 'active_support/all'
@@ -11,23 +12,8 @@ require 'ddtrace'
 # for every incoming message. This class should be lightweight.
 module Deimos
   # Parent consumer class.
-  class Consumer
+  class Consumer < BaseConsumer
     include Phobos::Handler
-    include SharedConfig
-
-    class << self
-      # @return [AvroDataEncoder]
-      def decoder
-        @decoder ||= AvroDataDecoder.new(schema: config[:schema],
-                                         namespace: config[:namespace])
-      end
-
-      # @return [AvroDataEncoder]
-      def key_decoder
-        @key_decoder ||= AvroDataDecoder.new(schema: config[:key_schema],
-                                             namespace: config[:namespace])
-      end
-    end
 
     # :nodoc:
     def around_consume(payload, metadata)
@@ -41,34 +27,11 @@ module Deimos
     # :nodoc:
     def before_consume(payload, metadata)
       _with_error_span(payload, metadata) do
-        if self.class.config[:key_schema] || self.class.config[:key_field]
-          metadata[:key] = decode_key(metadata[:key])
-        end
+        metadata[:key] = decode_key(metadata[:key])
         self.class.decoder.decode(payload) if payload.present?
       end
     end
 
-    # Helper method to decode an Avro-encoded key.
-    # @param key [String]
-    # @return [Object] the decoded key.
-    def decode_key(key)
-      return nil if key.nil?
-
-      config = self.class.config
-      if config[:encode_key] && config[:key_field].nil? &&
-         config[:key_schema].nil?
-        raise 'No key config given - if you are not decoding keys, please use `key_config plain: true`'
-      end
-
-      if config[:key_field]
-        self.class.decoder.decode_key(key, config[:key_field])
-      elsif config[:key_schema]
-        self.class.key_decoder.decode(key, schema: config[:key_schema])
-      else # no encoding
-        key
-      end
-    end
-
     # Consume incoming messages.
     # @param _payload [String]
     # @param _metadata [Hash]
@@ -78,20 +41,6 @@ module Deimos
 
   private
 
-    # @param payload [Hash|String]
-    # @param metadata [Hash]
-    def _with_error_span(payload, metadata)
-      @span = Deimos.config.tracer&.start(
-        'deimos-consumer',
-        resource: self.class.name.gsub('::', '-')
-      )
-      yield
-    rescue StandardError => e
-      _handle_error(e, payload, metadata)
-    ensure
-      Deimos.config.tracer&.finish(@span)
-    end
-
     def _received_message(payload, metadata)
       Deimos.config.logger.info(
         message: 'Got Kafka event',
@@ -102,13 +51,13 @@ module Deimos
                                          status:received
                                          topic:#{metadata[:topic]}
                                        ))
+      _report_time_delayed(payload, metadata)
     end
 
     # @param exception [Throwable]
     # @param payload [Hash]
     # @param metadata [Hash]
     def _handle_error(exception, payload, metadata)
-      Deimos.config.tracer&.set_error(@span, exception)
       Deimos.config.metrics&.increment(
         'handler',
         tags: %W(
@@ -124,7 +73,7 @@ module Deimos
         error_message: exception.message,
         error: exception.backtrace
       )
-      raise if Deimos.config.reraise_consumer_errors
+      super
     end
 
     # @param time_taken [Float]
@@ -145,20 +94,6 @@ module Deimos
         time_elapsed: time_taken,
         metadata: metadata
       )
-      return if payload.nil? || payload['timestamp'].blank?
-
-      begin
-        time_delayed = Time.now.in_time_zone - payload['timestamp'].to_datetime
-      rescue ArgumentError
-        Deimos.config.logger.info(
-          message: "Error parsing timestamp! #{payload['timestamp']}"
-        )
-        return
-      end
-      Deimos.config.metrics&.histogram('handler', time_delayed, tags: %W(
-                                         time:time_delayed
-                                         topic:#{metadata[:topic]}
-                                       ))
     end
   end
 end

data/lib/deimos/test_helpers.rb

@@ -91,7 +91,7 @@ module Deimos
       end
     end
 
-    # Stub all already-loaded producers and consumers fir unit testing purposes.
+    # Stub all already-loaded producers and consumers for unit testing purposes.
     def stub_producers_and_consumers!
       Deimos::TestHelpers.sent_messages.clear
 
@@ -106,10 +106,17 @@ module Deimos
       end
 
       Deimos::Consumer.descendants.each do |klass|
+        # TODO: remove this when ActiveRecordConsumer uses batching
         next if klass == Deimos::ActiveRecordConsumer # "abstract" class
 
         stub_consumer(klass)
       end
+
+      Deimos::BatchConsumer.descendants.each do |klass|
+        next if klass == Deimos::ActiveRecordConsumer # "abstract" class
+
+        stub_batch_consumer(klass)
+      end
     end
 
     # Stub a given producer class.
@@ -126,9 +133,7 @@ module Deimos
     # Stub a given consumer class.
     # @param klass [Class < Deimos::Consumer]
     def stub_consumer(klass)
-      allow(klass).to receive(:decoder) do
-        create_decoder(klass.config[:schema], klass.config[:namespace])
-      end
+      _stub_base_consumer(klass)
       klass.class_eval do
         alias_method(:old_consume, :consume) unless self.instance_methods.include?(:old_consume)
       end
@@ -138,6 +143,12 @@ module Deimos
       end
     end
 
+    # Stub a given batch consumer class.
+    # @param klass [Class < Deimos::BatchConsumer]
+    def stub_batch_consumer(klass)
+      _stub_base_consumer(klass)
+    end
+
     # get the difference of 2 hashes.
     # @param hash1 [Hash]
     # @param hash2 [Hash]
@@ -314,6 +325,102 @@ module Deimos
       }.to raise_error(Avro::SchemaValidator::ValidationError)
     end
 
+    # Test that a given handler will consume a given batch payload correctly,
+    # i.e. that the Avro schema is correct. If
+    # a block is given, that block will be executed when `consume` is called.
+    # Otherwise it will just confirm that `consume` is called at all.
+    # @param handler_class_or_topic [Class|String] Class which inherits from
+    # Deimos::Consumer or the topic as a string
+    # @param payloads [Array<Hash>] the payload to consume
+    def test_consume_batch(handler_class_or_topic,
+                           payloads,
+                           keys: [],
+                           partition_keys: [],
+                           call_original: false,
+                           skip_expectation: false,
+                           &block)
+      if call_original && block_given?
+        raise 'Cannot have both call_original and be given a block!'
+      end
+
+      topic_name = 'my-topic'
+      handler_class = if handler_class_or_topic.is_a?(String)
+                        _get_handler_class_from_topic(handler_class_or_topic)
+                      else
+                        handler_class_or_topic
+                      end
+      handler = handler_class.new
+      allow(handler_class).to receive(:new).and_return(handler)
+      listener = double('listener',
+                        handler_class: handler_class,
+                        encoding: nil)
+      batch_messages = payloads.zip(keys, partition_keys).map do |payload, key, partition_key|
+        key ||= _key_from_consumer(handler_class)
+
+        double('message',
+               'key' => key,
+               'partition_key' => partition_key,
+               'partition' => 1,
+               'offset' => 1,
+               'value' => payload)
+      end
+      batch = double('fetched_batch',
+                     'messages' => batch_messages,
+                     'topic' => topic_name,
+                     'partition' => 1,
+                     'offset_lag' => 0)
+      unless skip_expectation
+        expectation = expect(handler).to receive(:consume_batch).
+          with(payloads, anything, &block)
+        expectation.and_call_original if call_original
+      end
+      action = Phobos::Actions::ProcessBatchInline.new(
+        listener: listener,
+        batch: batch,
+        metadata: { topic: topic_name }
+      )
+      allow(action).to receive(:backoff_interval).and_return(0)
+      allow(action).to receive(:handle_error) { |e| raise e }
+      action.send(:execute)
+    end
+
+    # Check to see that a given message will fail due to Avro errors.
+    # @param handler_class [Class]
+    # @param payloads [Array<Hash>]
+    def test_consume_batch_invalid_message(handler_class, payloads)
+      topic_name = 'my-topic'
+      handler = handler_class.new
+      allow(handler_class).to receive(:new).and_return(handler)
+      listener = double('listener',
+                        handler_class: handler_class,
+                        encoding: nil)
+      batch_messages = payloads.map do |payload|
+        key ||= _key_from_consumer(handler_class)
+
+        double('message',
+               'key' => key,
+               'partition' => 1,
+               'offset' => 1,
+               'value' => payload)
+      end
+      batch = double('fetched_batch',
+                     'messages' => batch_messages,
+                     'topic' => topic_name,
+                     'partition' => 1,
+                     'offset_lag' => 0)
+
+      action = Phobos::Actions::ProcessBatchInline.new(
+        listener: listener,
+        batch: batch,
+        metadata: { topic: topic_name }
+      )
+      allow(action).to receive(:backoff_interval).and_return(0)
+      allow(action).to receive(:handle_error) { |e| raise e }
+
+      expect { action.send(:execute) }.
+        to raise_error(Avro::SchemaValidator::ValidationError)
+    end
+
     # @param schema1 [String|Hash] a file path, JSON string, or
     # hash representing a schema.
     # @param schema2 [String|Hash] a file path, JSON string, or
@@ -352,5 +459,13 @@ module Deimos
 
       handler.handler.constantize
     end
+
+    # Stub shared methods between consumers/batch consumers
+    # @param [Class < Deimos::BaseConsumer] klass Consumer class to stub
+    def _stub_base_consumer(klass)
+      allow(klass).to receive(:decoder) do
+        create_decoder(klass.config[:schema], klass.config[:namespace])
+      end
+    end
   end
 end

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.0.0'
+  VERSION = '1.1.0-beta1'
 end

data/lib/deimos.rb

@@ -10,6 +10,7 @@ require 'deimos/producer'
 require 'deimos/active_record_producer'
 require 'deimos/active_record_consumer'
 require 'deimos/consumer'
+require 'deimos/batch_consumer'
 require 'deimos/configuration'
 require 'deimos/instrumentation'
 require 'deimos/utils/lag_reporter'
@@ -58,6 +59,8 @@ module Deimos
         configure_loggers(phobos_config)
 
         Phobos.configure(phobos_config)
+
+        validate_consumers
       end
 
       validate_db_backend if self.config.publish_backend == :db
@@ -78,7 +81,7 @@ module Deimos
     # Start the DB producers to send Kafka messages.
     # @param thread_count [Integer] the number of threads to start.
     def start_db_backend!(thread_count: 1)
-      if self.config.publish_backend != :db # rubocop:disable Style/IfUnlessModifier
+      if self.config.publish_backend != :db
         raise('Publish backend is not set to :db, exiting')
       end
 
@@ -119,6 +122,28 @@ module Deimos
     def ssl_var_contents(filename)
       File.exist?(filename) ? File.read(filename) : filename
     end
+
+    # Validate that consumers are configured correctly, including their
+    # delivery mode.
+    def validate_consumers
+      Phobos.config.listeners.each do |listener|
+        handler_class = listener.handler.constantize
+        delivery = listener.delivery
+
+        # Validate that Deimos consumers use proper delivery configs
+        if handler_class < Deimos::BatchConsumer
+          unless delivery == 'inline_batch'
+            raise "BatchConsumer #{listener.handler} must have delivery set to"\
+              ' `inline_batch`'
+          end
+        elsif handler_class < Deimos::Consumer
+          if delivery.present? && !%w(message batch).include?(delivery)
+            raise "Non-batch Consumer #{listener.handler} must have delivery"\
+              ' set to `message` or `batch`'
+          end
+        end
+      end
+    end
   end
 end
 

data/spec/batch_consumer_spec.rb

@@ -0,0 +1,247 @@
+# frozen_string_literal: true
+
+require 'deimos/batch_consumer'
+
+# :nodoc:
+module ConsumerTest
+  describe Deimos::BatchConsumer do
+
+    prepend_before(:each) do
+      # :nodoc:
+      consumer_class = Class.new(Deimos::BatchConsumer) do
+        schema 'MySchema'
+        namespace 'com.my-namespace'
+        key_config field: 'test_id'
+
+        # :nodoc:
+        def consume_batch(_payloads, _metadata)
+          raise 'This should not be called unless call_original is set'
+        end
+      end
+      stub_const('ConsumerTest::MyBatchConsumer', consumer_class)
+    end
+
+    let(:batch) do
+      [
+        { 'test_id' => 'foo', 'some_int' => 123 },
+        { 'test_id' => 'bar', 'some_int' => 456 }
+      ]
+    end
+
+    let(:invalid_payloads) do
+      batch.concat([{ 'invalid' => 'key' }])
+    end
+
+    it 'should consume a batch of messages' do
+      test_consume_batch(MyBatchConsumer, batch) do |received, _metadata|
+        expect(received).to eq(batch)
+      end
+    end
+
+    it 'should consume a message on a topic' do
+      test_consume_batch('my_batch_consume_topic', batch) do |received, _metadata|
+        expect(received).to eq(batch)
+      end
+    end
+
+    it 'should fail on an invalid message in the batch' do
+      test_consume_batch_invalid_message(MyBatchConsumer, batch.concat(invalid_payloads))
+    end
+
+    describe 'when reraising errors is disabled' do
+      before(:each) do
+        Deimos.configure { |config| config.reraise_consumer_errors = false }
+      end
+
+      it 'should not fail when before_consume_batch fails' do
+        expect {
+          test_consume_batch(
+            MyBatchConsumer,
+            batch,
+            skip_expectation: true
+          ) { raise 'OH NOES' }
+        }.not_to raise_error
+      end
+
+      it 'should not fail when consume_batch fails' do
+        expect {
+          test_consume_batch(
+            MyBatchConsumer,
+            invalid_payloads,
+            skip_expectation: true
+          )
+        }.not_to raise_error
+      end
+    end
+
+    describe 'decoding' do
+      let(:keys) do
+        batch.map { |v| v.slice('test_id') }
+      end
+
+      it 'should decode payloads for all messages in the batch' do
+        expect_any_instance_of(Deimos::AvroDataDecoder).
+          to receive(:decode).with(batch[0])
+        expect_any_instance_of(Deimos::AvroDataDecoder).
+          to receive(:decode).with(batch[1])
+
+        test_consume_batch('my_batch_consume_topic', batch) do |received, _metadata|
+          # Mock decoder simply returns the payload
+          expect(received).to eq(batch)
+        end
+      end
+
+      it 'should decode keys for all messages in the batch' do
+        expect_any_instance_of(ConsumerTest::MyBatchConsumer).
+          to receive(:decode_key).with(keys[0]).and_call_original
+        expect_any_instance_of(ConsumerTest::MyBatchConsumer).
+          to receive(:decode_key).with(keys[1]).and_call_original
+
+        test_consume_batch('my_batch_consume_topic', batch, keys: keys) do |_received, metadata|
+          # Mock decode_key extracts the value of the first field as the key
+          expect(metadata[:keys]).to eq(%w(foo bar))
+        end
+      end
+
+      it 'should decode plain keys for all messages in the batch' do
+        consumer_class = Class.new(Deimos::BatchConsumer) do
+          schema 'MySchema'
+          namespace 'com.my-namespace'
+          key_config plain: true
+        end
+        stub_const('ConsumerTest::MyBatchConsumer', consumer_class)
+        stub_batch_consumer(consumer_class)
+
+        test_consume_batch('my_batch_consume_topic', batch, keys: [1, 2]) do |_received, metadata|
+          expect(metadata[:keys]).to eq([1, 2])
+        end
+      end
+    end
+
+    describe 'timestamps' do
+      before(:each) do
+        # :nodoc:
+        consumer_class = Class.new(Deimos::BatchConsumer) do
+          schema 'MySchemaWithDateTimes'
+          namespace 'com.my-namespace'
+          key_config plain: true
+
+          # :nodoc:
+          def consume_batch(_payloads, _metadata)
+            raise 'This should not be called unless call_original is set'
+          end
+        end
+        stub_const('ConsumerTest::MyBatchConsumer', consumer_class)
+        stub_batch_consumer(consumer_class)
+        allow(Deimos.config.metrics).to receive(:histogram)
+      end
+
+      let(:batch_with_time) do
+        [
+          {
+            'test_id' => 'foo',
+            'some_int' => 123,
+            'updated_at' => Time.now.to_i,
+            'timestamp' => 2.minutes.ago.to_s
+          },
+          {
+            'test_id' => 'bar',
+            'some_int' => 456,
+            'updated_at' => Time.now.to_i,
+            'timestamp' => 3.minutes.ago.to_s
+          }
+        ]
+      end
+
+      let(:invalid_times) do
+        [
+          {
+            'test_id' => 'baz',
+            'some_int' => 123,
+            'updated_at' => Time.now.to_i,
+            'timestamp' => 'yesterday morning'
+          },
+          {
+            'test_id' => 'ok',
+            'some_int' => 456,
+            'updated_at' => Time.now.to_i,
+            'timestamp' => ''
+          },
+          {
+            'test_id' => 'hello',
+            'some_int' => 456,
+            'updated_at' => Time.now.to_i,
+            'timestamp' => '1234567890'
+          }
+        ]
+      end
+
+      it 'should consume a batch' do
+        expect(Deimos.config.metrics).
+          to receive(:histogram).with('handler',
+                                      a_kind_of(Numeric),
+                                      tags: %w(time:time_delayed topic:my-topic)).twice
+
+        test_consume_batch('my_batch_consume_topic', batch_with_time) do |received, _metadata|
+          expect(received).to eq(batch_with_time)
+        end
+      end
+
+      it 'should fail nicely and ignore timestamps with the wrong format' do
+        batch = invalid_times.concat(batch_with_time)
+
+        expect(Deimos.config.metrics).
+          to receive(:histogram).with('handler',
+                                      a_kind_of(Numeric),
+                                      tags: %w(time:time_delayed topic:my-topic)).twice
+
+        test_consume_batch('my_batch_consume_topic', batch) do |received, _metadata|
+          expect(received).to eq(batch)
+        end
+      end
+    end
+
+    describe 'logging' do
+      before(:each) do
+        # :nodoc:
+        consumer_class = Class.new(Deimos::BatchConsumer) do
+          schema 'MySchemaWithUniqueId'
+          namespace 'com.my-namespace'
+          key_config plain: true
+
+          # :nodoc:
+          def consume_batch(_payloads, _metadata)
+            raise 'This should not be called unless call_original is set'
+          end
+        end
+        stub_const('ConsumerTest::MyBatchConsumer', consumer_class)
+        stub_batch_consumer(consumer_class)
+        allow(Deimos.config.metrics).to receive(:histogram)
+      end
+
+      it 'should log message identifiers' do
+        batch_with_message_id = [
+          { 'id' => 1, 'test_id' => 'foo', 'some_int' => 5,
+            'timestamp' => 3.minutes.ago.to_s, 'message_id' => 'one' },
+          { 'id' => 2, 'test_id' => 'bar', 'some_int' => 6,
+            'timestamp' => 2.minutes.ago.to_s, 'message_id' => 'two' }
+        ]
+
+        allow(Deimos.config.logger).
+          to receive(:info)
+
+        expect(Deimos.config.logger).
+          to receive(:info).
+          with(hash_including(
+                 message_ids: [
+                   { key: 1, message_id: 'one' },
+                   { key: 2, message_id: 'two' }
+                 ]
+               )).
+          twice
+
+        test_consume_batch('my_batch_consume_topic', batch_with_message_id, keys: [1, 2])
+      end
+    end
+  end
+end

data/spec/consumer_spec.rb

@@ -96,7 +96,7 @@ module ConsumerTest
         expect { MyConsumer.new.decode_key(123) }.to raise_error(NoMethodError)
       end
 
-      it 'should not encode if plain is set' do
+      it 'should not decode if plain is set' do
         consumer_class = Class.new(Deimos::Consumer) do
           schema 'MySchema'
           namespace 'com.my-namespace'

data/spec/deimos_spec.rb

@@ -2,58 +2,66 @@
 
 describe Deimos do
 
+  let(:phobos_configuration) do
+    { 'logger' =>
+        { 'file' => 'log/phobos.log',
+          'stdout_json' => false,
+          'level' => 'debug',
+          'ruby_kafka' =>
+              { 'level' => 'debug' } },
+      'kafka' =>
+        { 'client_id' => 'phobos',
+          'connect_timeout' => 15,
+          'socket_timeout' => 15,
+          'seed_brokers' => 'my_seed_broker.com',
+          'ssl_ca_cert' => 'my_ssl_ca_cert',
+          'ssl_client_cert' => 'my_ssl_client_cert',
+          'ssl_client_cert_key' => 'my_ssl_client_cert_key' },
+      'producer' =>
+        { 'ack_timeout' => 5,
+          'required_acks' => :all,
+          'max_retries' => 2,
+          'retry_backoff' => 1,
+          'max_buffer_size' => 10_000,
+          'max_buffer_bytesize' => 10_000_000,
+          'compression_codec' => nil,
+          'compression_threshold' => 1,
+          'max_queue_size' => 10_000,
+          'delivery_threshold' => 0,
+          'delivery_interval' => 0 },
+      'consumer' =>
+        { 'session_timeout' => 300,
+          'offset_commit_interval' => 10,
+          'offset_commit_threshold' => 0,
+          'heartbeat_interval' => 10 },
+      'backoff' =>
+        { 'min_ms' => 1000,
+          'max_ms' => 60_000 },
+      'listeners' => [
+        { 'handler' => 'ConsumerTest::MyConsumer',
+          'topic' => 'my_consume_topic',
+          'group_id' => 'my_group_id',
+          'max_bytes_per_partition' => 524_288 },
+        { 'handler' => 'ConsumerTest::MyBatchConsumer',
+          'topic' => 'my_batch_consume_topic',
+          'group_id' => 'my_batch_group_id',
+          'delivery' => 'inline_batch' }
+      ],
+      'custom_logger' => nil,
+      'custom_kafka_logger' => nil }
+  end
+
+  let(:config_path) { File.join(File.dirname(__FILE__), 'phobos.yml') }
+
   it 'should have a version number' do
     expect(Deimos::VERSION).not_to be_nil
   end
 
   specify 'configure' do
-    phobos_configuration = { 'logger' =>
-  { 'file' => 'log/phobos.log',
-    'stdout_json' => false,
-    'level' => 'debug',
-    'ruby_kafka' =>
-     { 'level' => 'debug' } },
-                             'kafka' =>
-  { 'client_id' => 'phobos',
-    'connect_timeout' => 15,
-    'socket_timeout' => 15,
-    'seed_brokers' => 'my_seed_broker.com',
-    'ssl_ca_cert' => 'my_ssl_ca_cert',
-    'ssl_client_cert' => 'my_ssl_client_cert',
-    'ssl_client_cert_key' => 'my_ssl_client_cert_key' },
-                             'producer' =>
-  { 'ack_timeout' => 5,
-    'required_acks' => :all,
-    'max_retries' => 2,
-    'retry_backoff' => 1,
-    'max_buffer_size' => 10_000,
-    'max_buffer_bytesize' => 10_000_000,
-    'compression_codec' => nil,
-    'compression_threshold' => 1,
-    'max_queue_size' => 10_000,
-    'delivery_threshold' => 0,
-    'delivery_interval' => 0 },
-                             'consumer' =>
-  { 'session_timeout' => 300,
-    'offset_commit_interval' => 10,
-    'offset_commit_threshold' => 0,
-    'heartbeat_interval' => 10 },
-                             'backoff' =>
-                              { 'min_ms' => 1000,
-                                'max_ms' => 60_000 },
-                             'listeners' => [
-                               { 'handler' => 'ConsumerTest::MyConsumer',
-                                 'topic' => 'my_consume_topic',
-                                 'group_id' => 'my_group_id',
-                                 'max_bytes_per_partition' => 524_288 }
-                             ],
-                             'custom_logger' => nil,
-                             'custom_kafka_logger' => nil }
-
     expect(Phobos).to receive(:configure).with(phobos_configuration)
     allow(described_class).to receive(:ssl_var_contents) { |key| key }
     described_class.configure do |config|
-      config.phobos_config_file = File.join(File.dirname(__FILE__), 'phobos.yml')
+      config.phobos_config_file = config_path
       config.seed_broker = 'my_seed_broker.com'
       config.ssl_enabled = true
       config.ssl_ca_cert = 'my_ssl_ca_cert'
@@ -118,5 +126,61 @@ describe Deimos do
         to raise_error('Thread count is not given or set to zero, exiting')
     end
 
+    describe 'delivery configuration' do
+      before(:each) do
+        described_class.config = nil
+        allow(YAML).to receive(:load).and_return(phobos_configuration)
+      end
+
+      it 'should not raise an error with properly configured handlers' do
+        # Add explicit consumers
+        phobos_configuration['listeners'] << { 'handler' => 'ConsumerTest::MyConsumer',
+                                               'delivery' => 'message' }
+        phobos_configuration['listeners'] << { 'handler' => 'ConsumerTest::MyConsumer',
+                                               'delivery' => 'batch' }
+
+        expect {
+          described_class.configure { |c| c.phobos_config_file = config_path }
+        }.not_to raise_error
+      end
+
+      it 'should raise an error if BatchConsumers do not have inline_batch delivery' do
+        phobos_configuration['listeners'] = [{ 'handler' => 'ConsumerTest::MyBatchConsumer',
+                                               'delivery' => 'message' }]
+
+        expect {
+          described_class.configure { |c| c.phobos_config_file = config_path }
+        }.to raise_error('BatchConsumer ConsumerTest::MyBatchConsumer must have delivery set to `inline_batch`')
+      end
+
+      it 'should raise an error if Consumers do not have message or batch delivery' do
+        phobos_configuration['listeners'] = [{ 'handler' => 'ConsumerTest::MyConsumer',
+                                               'delivery' => 'inline_batch' }]
+
+        expect {
+          described_class.configure { |c| c.phobos_config_file = config_path }
+        }.to raise_error('Non-batch Consumer ConsumerTest::MyConsumer must have delivery set to `message` or `batch`')
+      end
+
+      it 'should treat nil as `batch`' do
+        phobos_configuration['listeners'] = [{ 'handler' => 'ConsumerTest::MyConsumer' }]
+
+        expect {
+          described_class.configure { |c| c.phobos_config_file = config_path }
+        }.not_to raise_error
+      end
+
+      it 'should ignore non-Deimos listeners' do
+        consumer_class = Class.new { include Phobos::Handler }
+        stub_const('ConsumerTest::MyOtherConsumer', consumer_class)
+        phobos_configuration['listeners'] = [{ 'handler' => 'ConsumerTest::MyOtherConsumer',
+                                               'topic' => 'my_consume_topic',
+                                               'group_id' => 'my_group_id' }]
+
+        expect {
+          described_class.configure { |c| c.phobos_config_file = config_path }
+        }.not_to raise_error
+      end
+    end
   end
 end

data/spec/handlers/my_batch_consumer.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module ConsumerTest
+  class MyBatchConsumer < Deimos::BatchConsumer; end
+end

data/spec/handlers/my_consumer.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module ConsumerTest
+  class MyConsumer < Deimos::Consumer; end
+end

data/spec/phobos.yml

@@ -71,3 +71,7 @@ listeners:
     topic: my_consume_topic
     group_id: my_group_id
     max_bytes_per_partition: 524288 # 512 KB
+  - handler: ConsumerTest::MyBatchConsumer
+    topic: my_batch_consume_topic
+    group_id: my_batch_group_id
+    delivery: inline_batch

data/spec/spec_helper.rb

@@ -8,6 +8,8 @@ require 'deimos/tracing/mock'
 require 'deimos/test_helpers'
 require 'active_support/testing/time_helpers'
 require 'activerecord-import'
+require 'handlers/my_batch_consumer'
+require 'handlers/my_consumer'
 
 # Helpers for Executor/DbProducer
 module TestRunners

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0.pre.beta1
 platform: ruby
 authors:
 - Daniel Orner
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-03 00:00:00.000000000 Z
+date: 2019-09-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro-patches
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: 1.8.2.pre.beta2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: 1.8.2.pre.beta2
 - !ruby/object:Gem::Dependency
   name: ruby-kafka
   requirement: !ruby/object:Gem::Requirement
@@ -328,6 +328,8 @@ files:
 - lib/deimos/backends/db.rb
 - lib/deimos/backends/kafka.rb
 - lib/deimos/backends/kafka_async.rb
+- lib/deimos/base_consumer.rb
+- lib/deimos/batch_consumer.rb
 - lib/deimos/configuration.rb
 - lib/deimos/consumer.rb
 - lib/deimos/instrumentation.rb
@@ -369,8 +371,11 @@ files:
 - spec/backends/db_spec.rb
 - spec/backends/kafka_async_spec.rb
 - spec/backends/kafka_spec.rb
+- spec/batch_consumer_spec.rb
 - spec/consumer_spec.rb
 - spec/deimos_spec.rb
+- spec/handlers/my_batch_consumer.rb
+- spec/handlers/my_consumer.rb
 - spec/kafka_source_spec.rb
 - spec/kafka_topic_info_spec.rb
 - spec/phobos.bad_db.yml
@@ -411,9 +416,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.0.2
 signing_key: 
@@ -427,8 +432,11 @@ test_files:
 - spec/backends/db_spec.rb
 - spec/backends/kafka_async_spec.rb
 - spec/backends/kafka_spec.rb
+- spec/batch_consumer_spec.rb
 - spec/consumer_spec.rb
 - spec/deimos_spec.rb
+- spec/handlers/my_batch_consumer.rb
+- spec/handlers/my_consumer.rb
 - spec/kafka_source_spec.rb
 - spec/kafka_topic_info_spec.rb
 - spec/phobos.bad_db.yml