deimos-ruby 1.1.0.pre.beta2 → 1.2.0.pre.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 66e879a3dd314b2291d62b45044cc909173a4a91f72b3f749114c525f53a8eaf
-  data.tar.gz: e5b4c2c22d3392f97991675825ed4aa0ede4f78164aaf1385c5850f5c77403d0
+  metadata.gz: 5d27374b4f6388c2eae44b4b9654b10461c8fd7d044f5a67a63d7b1866b8658f
+  data.tar.gz: 1b058d23b6e5bbab471b57f74d0f85c3cd3c78131c9b43a32494f1b95b3c9110
 SHA512:
-  metadata.gz: 5cb509aacf5c75c7b9be9ffd9fc6e9d2e52f5a7351dfeefd8bf7e6a5a76b795985ab08bbfe5330f2a1917859bd6f82d3a5df750c972e30b39db3df7cd47289fd
-  data.tar.gz: c8c7f1380c021ca4e991b507cc859e848dc3eb07d6af5fb2fad76bfb8c4560cd08e9aa9337450a1f597dcbbb620db3546125127ee11c137ef164ce3621de50b2
+  metadata.gz: fb7e58230650beaec9d777a8ee8e33a9210b08ef05d59f64e6e3a9c5f8cfb095af72f15a443063998d3c6fd579263f1b25d0409b371200fbe9d95bee43ab0300
+  data.tar.gz: a6198b80dd040db5f2b633fc5e8c83da727f6d9e0d9915684df00f9f1e55f73fec99ed8ff482ac5f37378d1ebae18ebf7d9d24b6b89051625274b2f70b9cfbc4

data/CHANGELOG.md

@@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+# [1.2.0-beta1] - 2019-09-12
+- Added `fatal_error` to both global config and consumer classes.
+- Changed `pending_db_messages_max_wait` metric to send per topic.
+- Added config to compact messages in the DB producer.
+- Added config to log messages in the DB producer.
+- Added config to provide a separate logger to the DB producer.
+
 # [1.1.0-beta2] - 2019-09-11
 - Fixed bug where ActiveRecordConsumer was not using `unscoped` to update
   via primary key and causing duplicate record errors.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    deimos-ruby (1.1.0.pre.beta2)
+    deimos-ruby (1.2.0.pre.beta1)
       avro-patches (~> 0.3)
       avro_turf (~> 0.8)
       phobos (~> 1.8.2.pre.beta2)

data/README.md

@@ -2,7 +2,7 @@
   <img src="support/deimos-with-name.png" title="Deimos logo"/>
   <br/>
   <img src="https://img.shields.io/circleci/build/github/flipp-oss/deimos.svg" alt="CircleCI"/>
-  <a href="https://badge.fury.io/rb/deimos"><img src="https://badge.fury.io/rb/deimos.svg" alt="Gem Version" height="18"></a>
+  <a href="https://badge.fury.io/rb/deimos-ruby"><img src="https://badge.fury.io/rb/deimos-ruby.svg" alt="Gem Version" height="18"></a>
   <img src="https://img.shields.io/codeclimate/maintainability/flipp-oss/deimos.svg"/>
 </p>
 
@@ -46,7 +46,7 @@ Or install it yourself as:
 
 # Versioning
 
-We use version of semver for this gem. Any change in previous behavior 
+We use a version of semver for this gem. Any change in previous behavior 
 (something works differently or something old no longer works)
 is denoted with a bump in the minor version (0.4 -> 0.5). Patch versions 
 are for bugfixes or new functionality which does not affect existing code. You
@@ -98,17 +98,38 @@ Deimos.configure do |config|
   # be able to proceed past it and will be stuck forever until you fix
   # your code.
   config.reraise_consumer_errors = true
+
+  # Another way to handle errors is to set reraise_consumer_errors to false
+  # but to set a global "fatal error" block that determines when to reraise:
+  config.fatal_error do |exception, payload, metadata|
+    exception.is_a?(BadError)
+  end
+  # Another example would be to check the database connection and fail
+  # if the DB is down entirely.
   
   # Set to true to send consumer lag metrics
   config.report_lag = %w(production staging).include?(Rails.env)
   
   # Change the default backend. See Backends, below.
   config.backend = :db
-  
-  # If the DB backend is being used, specify the number of threads to create
-  # to process the DB messages.
-  config.num_producer_threads = 1
-  
+
+  # Database Backend producer configuration
+ 
+  # Logger for DB producer
+  config.db_producer.logger = Logger.new('/db_producer.log')
+
+  # List of topics to print full messages for, or :all to print all
+  # topics. This can introduce slowdown since it needs to decode
+  # each message using the schema registry. 
+  config.db_producer.log_topics = ['topic1', 'topic2']
+
+  # List of topics to compact before sending, i.e. only send the
+  # last message with any given key in a batch. This is an optimization
+  # which mirrors what Kafka itself will do with compaction turned on
+  # but only within a single batch.  You can also specify :all to
+  # compact all topics.
+  config.db_producer.compact_topics = ['topic1', 'topic2']
+
   # Configure the metrics provider (see below).
   config.metrics = Deimos::Metrics::Mock.new({ tags: %w(env:prod my_tag:another_1) })
 
@@ -333,6 +354,13 @@ class MyConsumer < Deimos::Consumer
   # `schema` and `namespace`, above, for this to work.
   key_config field: :my_id 
 
+  # Optionally overload this to consider a particular exception
+  # "fatal" only for this consumer. This is considered in addition
+  # to the global `fatal_error` configuration block. 
+  def fatal_error?(exception, payload, metadata)
+    exception.is_a?(MyBadError)
+  end
+
   def consume(payload, metadata)
     # Same method as Phobos consumers.
     # payload is an Avro-decoded hash.
@@ -344,6 +372,20 @@ class MyConsumer < Deimos::Consumer
 end
 ```
 
+### Fatal Errors
+
+The recommended configuration is for consumers *not* to raise errors
+they encounter while consuming messages. Errors can be come from 
+a variety of sources and it's possible that the message itself (or
+what downstream systems are doing with it) is causing it. If you do
+not continue on past this message, your consumer will essentially be
+stuck forever unless you take manual action to skip the offset.
+
+Use `config.reraise_consumer_errors = false` to swallow errors. You
+can use instrumentation to handle errors you receive. You can also
+specify "fatal errors" either via global configuration (`config.fatal_error`)
+or via overriding a method on an individual consumer (`def fatal_error`).
+
 ### Batch Consumption
 
 Instead of consuming messages one at a time, consumers can receive a batch of
@@ -636,7 +678,9 @@ The following metrics are reported:
   to publish. Tagged with `topic:{topic_name}`
 * `pending_db_messages_max_wait` - the number of seconds which the
   oldest KafkaMessage in the database has been waiting for, for use
-  with the database backend.
+  with the database backend. Tagged with the topic that is waiting.
+  Will send a value of 0 with no topics tagged if there are no messages
+  waiting.
 
 ### Configuring Metrics Providers
 

data/lib/deimos.rb

@@ -90,7 +90,8 @@ module Deimos
       end
 
       producers = (1..thread_count).map do
-        Deimos::Utils::DbProducer.new(self.config.logger)
+        Deimos::Utils::DbProducer.
+          new(self.config.db_producer.logger || self.config.logger)
       end
       executor = Deimos::Utils::Executor.new(producers,
                                              sleep_seconds: 5,

data/lib/deimos/base_consumer.rb

@@ -74,13 +74,25 @@ module Deimos
                                        ))
     end
 
-    # @param exception [Throwable]
-    # @param _payload [Hash]
-    # @param _metadata [Hash]
-    def _handle_error(exception, _payload, _metadata)
+    # Overrideable method to determine if a given error should be considered
+    # "fatal" and always be reraised.
+    # @param error [Exception]
+    # @param payload [Hash]
+    # @param metadata [Hash]
+    # @return [Boolean]
+    def fatal_error?(_error, _payload, _metadata)
+      false
+    end
+
+    # @param exception [Exception]
+    # @param payload [Hash]
+    # @param metadata [Hash]
+    def _handle_error(exception, payload, metadata)
       Deimos.config.tracer&.set_error(@span, exception)
 
-      raise if Deimos.config.reraise_consumer_errors
+      raise if Deimos.config.reraise_consumer_errors ||
+               Deimos.config.fatal_error_block.call(exception, payload, metadata) ||
+               fatal_error?(exception, payload, metadata)
     end
 
     # @param _time_taken [Float]

data/lib/deimos/configuration.rb

@@ -72,10 +72,26 @@ module Deimos
     # @return [Tracing::Provider]
     attr_accessor :tracer
 
+    # @return [Deimos::DbProducerConfiguration]
+    attr_accessor :db_producer
+
+    # For internal purposes only
+    # @return [Block]
+    attr_accessor :fatal_error_block
+
     # :nodoc:
     def initialize
       @phobos_config_file = 'config/phobos.yml'
       @publish_backend = :kafka_async
+      @db_producer = DbProducerConfiguration.new
+      fatal_error { false }
+    end
+
+    # Block taking an exception, payload and metadata and returning
+    # true if this should be considered a fatal error and false otherwise.
+    # Not needed if reraise_consumer_errors is set to true.
+    def fatal_error(&block)
+      @fatal_error_block = block
     end
 
     # @param other_config [Configuration]
@@ -87,4 +103,22 @@ module Deimos
       other_config.logger != self.logger
     end
   end
+
+  # Sub-class for DB producer configs.
+  class DbProducerConfiguration
+    # @return [Logger]
+    attr_accessor :logger
+    # @return [Symbol|Array<String>] A list of topics to log all messages, or
+    # :all to log all topics.
+    attr_accessor :log_topics
+    # @return [Symbol|Array<String>] A list of topics to compact messages for
+    # before sending, or :all to compact all keyed messages.
+    attr_accessor :compact_topics
+
+    # :nodoc:
+    def initialize
+      @log_topics = []
+      @compact_topics = []
+    end
+  end
 end

data/lib/deimos/kafka_message.rb

@@ -14,9 +14,11 @@ module Deimos
       write_attribute(:message, mess ? mess.to_s : nil)
     end
 
+    # Get a decoder to decode a set of messages on the given topic.
+    # @param topic [String]
     # @return [Deimos::Consumer]
-    def decoder
-      producer = Deimos::Producer.descendants.find { |c| c.topic == self.topic }
+    def self.decoder(topic)
+      producer = Deimos::Producer.descendants.find { |c| c.topic == topic }
       return nil unless producer
 
       consumer = Class.new(Deimos::Consumer)
@@ -24,17 +26,19 @@ module Deimos
       consumer
     end
 
-    # Decode the message. This assumes for now that we have access to a producer
-    # in the codebase which can decode it.
-    # @param decoder [Deimos::Consumer]
-    # @return [Hash]
-    def decoded_message(decoder=self.decoder)
-      return { key: self.key, message: self.message } unless decoder
-
-      {
-        key: self.key.present? ? decoder.new.decode_key(self.key) : nil,
-        payload: decoder.decoder.decode(self.message)
-      }
+    # Decoded payloads for a list of messages.
+    # @param messages [Array<Deimos::KafkaMessage>]
+    # @return [Array<Hash>]
+    def self.decoded(messages=[])
+      return [] if messages.empty?
+
+      decoder = self.decoder(messages.first.topic)&.new
+      messages.map do |m|
+        {
+          key: m.key.present? ? decoder&.decode_key(m.key) || m.key : nil,
+          payload: decoder&.decoder&.decode(self.message) || self.message
+        }
+      end
     end
 
     # @return [Hash]

data/lib/deimos/utils/db_producer.rb

@@ -16,6 +16,11 @@ module Deimos
         @logger.push_tags("DbProducer #{@id}") if @logger.respond_to?(:push_tags)
       end
 
+      # @return [Deimos::DbProducerConfig]
+      def config
+        Deimos.config.db_producer
+      end
+
       # Start the poll.
       def start
         @logger.info('Starting...')
@@ -60,44 +65,72 @@ module Deimos
           return
         end
         @current_topic = topic
-        messages = retrieve_messages
 
-        while messages.any?
-          @logger.debug do
-            decoder = messages.first.decoder
-            "DB producer: Topic #{topic} Producing messages: #{messages.map { |m| m.decoded_message(decoder) }}"
-          end
-          Deimos.instrument('db_producer.produce', topic: topic, messages: messages) do
-            begin
-              produce_messages(messages.map(&:phobos_message))
-            rescue Kafka::BufferOverflow, Kafka::MessageSizeTooLarge, Kafka::RecordListTooLarge
-              messages.each(&:delete)
-              raise
-            end
-          end
-          messages.first.class.where(id: messages.map(&:id)).delete_all
-          break if messages.size < BATCH_SIZE
+        loop { break unless process_topic_batch }
 
-          KafkaTopicInfo.heartbeat(@current_topic, @id) # keep alive
-          send_pending_metrics
-          messages = retrieve_messages
-        end
         KafkaTopicInfo.clear_lock(@current_topic, @id)
       rescue StandardError => e
         @logger.error("Error processing messages for topic #{@current_topic}: #{e.class.name}: #{e.message} #{e.backtrace.join("\n")}")
         KafkaTopicInfo.register_error(@current_topic, @id)
       end
 
+      # Process a single batch in a topic.
+      def process_topic_batch
+        messages = retrieve_messages
+        return false if messages.empty?
+
+        batch_size = messages.size
+        compacted_messages = compact_messages(messages)
+        log_messages(compacted_messages)
+        Deimos.instrument('db_producer.produce', topic: @current_topic, messages: compacted_messages) do
+          begin
+            produce_messages(compacted_messages.map(&:phobos_message))
+          rescue Kafka::BufferOverflow, Kafka::MessageSizeTooLarge, Kafka::RecordListTooLarge
+            Deimos::KafkaMessage.where(id: messages.map(&:id)).delete_all
+            @logger.error('Message batch too large, deleting...')
+            @logger.error(Deimos::KafkaMessage.decoded(messages))
+            raise
+          end
+        end
+        Deimos::KafkaMessage.where(id: messages.map(&:id)).delete_all
+        return false if batch_size < BATCH_SIZE
+
+        KafkaTopicInfo.heartbeat(@current_topic, @id) # keep alive
+        send_pending_metrics
+        true
+      end
+
       # @return [Array<Deimos::KafkaMessage>]
       def retrieve_messages
         KafkaMessage.where(topic: @current_topic).order(:id).limit(BATCH_SIZE)
       end
 
+      # @param messages [Array<Deimos::KafkaMessage>]
+      def log_messages(messages)
+        return if config.log_topics != :all && !config.log_topics.include?(@current_topic)
+
+        @logger.debug do
+          decoded_messages = Deimos::KafkaMessage.decoded(messages)
+          "DB producer: Topic #{@current_topic} Producing messages: #{decoded_messages}}"
+        end
+      end
+
       # Send metrics to Datadog.
       def send_pending_metrics
-        first_message = KafkaMessage.first
-        time_diff = first_message ? Time.zone.now - KafkaMessage.first.created_at : 0
-        Deimos.config.metrics&.gauge('pending_db_messages_max_wait', time_diff)
+        metrics = Deimos.config.metrics
+        return unless metrics
+
+        messages = Deimos::KafkaMessage.
+          select('count(*) as num_messages, min(created_at) as earliest, topic').
+          group(:topic)
+        if messages.none?
+          metrics.gauge('pending_db_messages_max_wait', 0)
+        end
+        messages.each do |record|
+          time_diff = Time.zone.now - record.earliest
+          metrics.gauge('pending_db_messages_max_wait', time_diff,
+                        tags: ["topic:#{record.topic}"])
+        end
       end
 
       # Shut down the sync producer if we have to. Phobos will automatically
@@ -140,6 +173,18 @@ module Deimos
           retry
         end
       end
+
+      # @param batch [Array<Deimos::KafkaMessage>]
+      # @return [Array<Deimos::KafkaMessage>]
+      def compact_messages(batch)
+        return batch unless batch.first&.key.present?
+
+        topic = batch.first.topic
+        return batch if config.compact_topics != :all &&
+                        !config.compact_topics.include?(topic)
+
+        batch.reverse.uniq!(&:key).reverse!
+      end
     end
   end
 end

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.1.0-beta2'
+  VERSION = '1.2.0-beta1'
 end

data/spec/consumer_spec.rb

@@ -11,6 +11,11 @@ module ConsumerTest
         namespace 'com.my-namespace'
         key_config field: 'test_id'
 
+        # :nodoc:
+        def fatal_error?(_exception, payload, _metadata)
+          payload == 'fatal'
+        end
+
         # :nodoc:
         def consume(_payload, _metadata)
           raise 'This should not be called unless call_original is set'
@@ -39,6 +44,19 @@ module ConsumerTest
       test_consume_invalid_message(MyConsumer, 'invalid' => 'key')
     end
 
+    it 'should fail if reraise is false but fatal_error is true' do
+      Deimos.configure { |config| config.reraise_consumer_errors = false }
+      test_consume_invalid_message(MyConsumer, 'fatal')
+    end
+
+    it 'should fail if fatal_error is true globally' do
+      Deimos.configure do |config|
+        config.fatal_error { true }
+        config.reraise_consumer_errors = false
+      end
+      test_consume_invalid_message(MyConsumer, 'invalid' => 'key')
+    end
+
     it 'should fail on message with extra fields' do
       test_consume_invalid_message(MyConsumer,
                                    'test_id' => 'foo',

data/spec/deimos_spec.rb

@@ -84,7 +84,7 @@ describe Deimos do
       allow(described_class).to receive(:run_db_backend)
     end
 
-    it 'should start if backend is db and num_producer_threads is > 0' do
+    it 'should start if backend is db and thread_count is > 0' do
       signal_handler = instance_double(Deimos::Utils::SignalHandler)
       allow(signal_handler).to receive(:run!)
       expect(Deimos::Utils::Executor).to receive(:new).
@@ -108,7 +108,7 @@ describe Deimos do
         to raise_error('Publish backend is not set to :db, exiting')
     end
 
-    it 'should not start if num_producer_threads is nil' do
+    it 'should not start if thread_count is nil' do
       expect(Deimos::Utils::SignalHandler).not_to receive(:new)
       described_class.configure do |config|
         config.publish_backend = :db
@@ -117,7 +117,7 @@ describe Deimos do
         to raise_error('Thread count is not given or set to zero, exiting')
     end
 
-    it 'should not start if num_producer_threads is 0' do
+    it 'should not start if thread_count is 0' do
       expect(Deimos::Utils::SignalHandler).not_to receive(:new)
       described_class.configure do |config|
         config.publish_backend = :db

data/spec/utils/db_producer_spec.rb

@@ -93,6 +93,64 @@ each_db_config(Deimos::Utils::DbProducer) do
       expect(phobos_producer).to have_received(:publish_list).with(['A']).once
 
     end
+
+    describe '#compact_messages' do
+      let(:batch) do
+        [
+          {
+            key: 1,
+            topic: 'my-topic',
+            message: 'AAA'
+          },
+          {
+            key: 2,
+            topic: 'my-topic',
+            message: 'BBB'
+          },
+          {
+            key: 1,
+            topic: 'my-topic',
+            message: 'CCC'
+          }
+        ].map { |h| Deimos::KafkaMessage.create!(h) }
+      end
+
+      let(:deduped_batch) { batch[1..2] }
+
+      it 'should dedupe messages when :all is set' do
+        Deimos.configure { |c| c.db_producer.compact_topics = :all }
+        expect(producer.compact_messages(batch)).to eq(deduped_batch)
+      end
+
+      it 'should dedupe messages when topic is included' do
+        Deimos.configure { |c| c.db_producer.compact_topics = %w(my-topic my-topic2) }
+        expect(producer.compact_messages(batch)).to eq(deduped_batch)
+      end
+
+      it 'should not dedupe messages when topic is not included' do
+        Deimos.configure { |c| c.db_producer.compact_topics = %w(my-topic3 my-topic2) }
+        expect(producer.compact_messages(batch)).to eq(batch)
+      end
+
+      it 'should not dedupe messages without keys' do
+        unkeyed_batch = [
+          {
+            key: nil,
+            topic: 'my-topic',
+            message: 'AAA'
+          },
+          {
+            key: nil,
+            topic: 'my-topic',
+            message: 'BBB'
+          }
+        ].map { |h| Deimos::KafkaMessage.create!(h) }
+        Deimos.configure { |c| c.db_producer.compact_topics = :all }
+        expect(producer.compact_messages(unkeyed_batch)).to eq(unkeyed_batch)
+        Deimos.configure { |c| c.db_producer.compact_topics = [] }
+      end
+
+    end
   end
 
   describe '#process_topic' do
@@ -119,6 +177,7 @@ each_db_config(Deimos::Utils::DbProducer) do
         with('my-topic', 'abc').and_return(true)
       expect(producer).to receive(:retrieve_messages).ordered.
         and_return(messages[0..1])
+      expect(producer).to receive(:send_pending_metrics).twice
       expect(producer).to receive(:produce_messages).ordered.with([
                                                                     {
                                                                       payload: 'mess1',
@@ -193,16 +252,70 @@ each_db_config(Deimos::Utils::DbProducer) do
       expect(Deimos::KafkaTopicInfo).to receive(:register_error)
 
       expect(Deimos::KafkaMessage.count).to eq(4)
-      Deimos.subscribe('db_producer.produce') do |event|
+      subscriber = Deimos.subscribe('db_producer.produce') do |event|
         expect(event.payload[:exception_object].message).to eq('OH NOES')
         expect(event.payload[:messages]).to eq(messages)
       end
       producer.process_topic('my-topic')
+      # don't delete for regular errors
+      expect(Deimos::KafkaMessage.count).to eq(4)
+      Deimos.unsubscribe(subscriber)
+    end
+
+    it 'should delete messages on buffer overflow' do
+      messages = (1..4).map do |i|
+        Deimos::KafkaMessage.create!(
+          id: i,
+          topic: 'my-topic',
+          message: "mess#{i}",
+          partition_key: "key#{i}"
+        )
+      end
+
+      expect(Deimos::KafkaTopicInfo).to receive(:lock).
+        with('my-topic', 'abc').and_return(true)
+      expect(producer).to receive(:produce_messages).and_raise(Kafka::BufferOverflow)
+      expect(producer).to receive(:retrieve_messages).and_return(messages)
+      expect(Deimos::KafkaTopicInfo).to receive(:register_error)
+
+      expect(Deimos::KafkaMessage.count).to eq(4)
+      producer.process_topic('my-topic')
       expect(Deimos::KafkaMessage.count).to eq(0)
     end
 
   end
 
+  describe '#send_pending_metrics' do
+    it 'should use the first created_at for each topic' do |example|
+      # sqlite date-time strings don't work correctly
+      next if example.metadata[:db_config][:adapter] == 'sqlite3'
+
+      freeze_time do
+        (1..2).each do |i|
+          Deimos::KafkaMessage.create!(topic: "topic#{i}", message: nil,
+                                       created_at: (3 + i).minutes.ago)
+          Deimos::KafkaMessage.create!(topic: "topic#{i}", message: nil,
+                                       created_at: (2 + i).minutes.ago)
+          Deimos::KafkaMessage.create!(topic: "topic#{i}", message: nil,
+                                       created_at: (1 + i).minute.ago)
+        end
+        allow(Deimos.config.metrics).to receive(:gauge)
+        producer.send_pending_metrics
+        expect(Deimos.config.metrics).to have_received(:gauge).twice
+        expect(Deimos.config.metrics).to have_received(:gauge).
+          with('pending_db_messages_max_wait', 4.minutes.to_i, tags: ['topic:topic1'])
+        expect(Deimos.config.metrics).to have_received(:gauge).
+          with('pending_db_messages_max_wait', 5.minutes.to_i, tags: ['topic:topic2'])
+      end
+    end
+
+    it 'should send 0 if no messages' do
+      expect(Deimos.config.metrics).to receive(:gauge).
+        with('pending_db_messages_max_wait', 0)
+      producer.send_pending_metrics
+    end
+  end
+
   example 'Full integration test' do
     (1..4).each do |i|
       (1..2).each do |j|

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.1.0.pre.beta2
+  version: 1.2.0.pre.beta1
 platform: ruby
 authors:
 - Daniel Orner
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-11 00:00:00.000000000 Z
+date: 2019-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro-patches