deimos-ruby 1.0.0.pre.beta25 → 1.0.0.pre.beta26

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cf690c099639314ff468feb6b9f925ca74655f7738e0c3162c4c879254dc707b
-  data.tar.gz: ecad3f7d4c68a824df0a2e28431885d95b896814db40d77383c6e7a162327dd4
+  metadata.gz: c9dbc69d5cb3ef5b93c1d926443dc2be92e295329f49edd0dec7a9e812a4e7a5
+  data.tar.gz: e336c6f5faf5f95c86e4d243ad0c537ba57f6db755097c83195b90bf41a1415e
 SHA512:
-  metadata.gz: c077533375e21f16c3ceb9f81b5fcc5bb8c1a970c35efbf1fa95dbcef28d7f4a18eebafda136f16d98dbfae8b9c920755ac75b453a38d504e489c87bcbf9e2a1
-  data.tar.gz: 425e6caeff23223e41da507a5631a13c6cf2f4d7d3f32b8338c37f2454bb0886702069de05dcd359aa5661ebb62e474db9fd9cb54c0ae1504cfcea3c6ec220ac
+  metadata.gz: 57fea10ed43861b17d8fd31ba1a5d322e12db96ec7de3fff6ea28e314efe10686bc6c7fd0da2f11bdf2bfa8c7510a60bfa17c9bd836a32e3e3ed5e190faa9050
+  data.tar.gz: 3ffd651469987c6a337decd275c78171870ab604599f4c7ca0347ebd8e9493b8fcda7ee4b3fb0012b0cdfbc13ceb7bdd2af51392707012d4559fed05842d7239

data/CHANGELOG.md

@@ -7,10 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
-## 1.0.0-beta25 - 2019-08-28
+## [1.0.0-beta26] - 2019-08-29
+- Recover from Kafka::MessageSizeTooLarge in the DB producer.
+- Shut down sync producers correctly when persistent_connections is true.
+- Notify when messages fail to produce in the DB producer.
+- Delete messages on failure and rely on notification.
+
+## [1.0.0-beta25] - 2019-08-28
 - Fix bug where crashing would cause producers to stay disabled
 
-## 1.0.0-beta24 - 2019-08-26
+## [1.0.0-beta24] - 2019-08-26
 - Reconnect DB backend if database goes away.
 - Sleep only 5 seconds between attempts instead of using exponential backoff.
 - Fix for null payload being Avro-encoded.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    deimos-ruby (1.0.0.pre.beta25)
+    deimos-ruby (1.0.0.pre.beta26)
       avro-patches (~> 0.3)
       avro_turf (~> 0.8)
       phobos (~> 1.8)

data/README.md

@@ -197,7 +197,8 @@ You can listen to these notifications e.g. as follows:
   end
 ``` 
 
-The following events are also produced:
+The following events are produced (in addition to the ones already
+produced by Phobos and RubyKafka):
 
 * `produce_error` - sent when an error occurs when producing a message.
   * producer - the class that produced the message
@@ -208,6 +209,14 @@ The following events are also produced:
   * producer - the class that produced the message
   * topic
   * payloads - the unencoded payloads
+* `db_producer.produce` - sent when the DB producer sends messages for the
+   DB backend. Messages that are too large will be caught with this
+   notification - they will be deleted from the table and this notification
+   will be fired with an exception object.
+   * topic
+   * exception_object
+   * messages - the batch of messages (in the form of `Deimos::KafkaMessage`s)
+     that failed - this should have only a single message in the batch.
   
 Similarly: 
 ```ruby

data/lib/deimos/backends/kafka.rb

@@ -6,6 +6,12 @@ module Deimos
     class Kafka < Deimos::PublishBackend
       include Phobos::Producer
 
+      # Shut down the producer if necessary.
+      def self.shutdown_producer
+        producer.sync_producer_shutdown if producer.respond_to?(:sync_producer_shutdown)
+        producer.kafka_client&.close
+      end
+
       # :nodoc:
       def self.execute(producer_class:, messages:)
         Deimos.instrument(

data/lib/deimos/backends/kafka_async.rb

@@ -6,6 +6,12 @@ module Deimos
     class KafkaAsync < Deimos::PublishBackend
       include Phobos::Producer
 
+      # Shut down the producer cleanly.
+      def self.shutdown_producer
+        producer.async_producer_shutdown
+        producer.kafka_client&.close
+      end
+
       # :nodoc:
       def self.execute(producer_class:, messages:)
         Deimos.instrument(

data/lib/deimos/kafka_message.rb

@@ -14,6 +14,29 @@ module Deimos
       write_attribute(:message, mess ? mess.to_s : nil)
     end
 
+    # @return [Deimos::Consumer]
+    def decoder
+      producer = Deimos::Producer.descendants.find { |c| c.topic == self.topic }
+      return nil unless producer
+
+      consumer = Class.new(Deimos::Consumer)
+      consumer.config.merge!(producer.config)
+      consumer
+    end
+
+    # Decode the message. This assumes for now that we have access to a producer
+    # in the codebase which can decode it.
+    # @param decoder [Deimos::Consumer]
+    # @return [Hash]
+    def decoded_message(decoder=self.decoder)
+      return { key: self.key, message: self.message } unless decoder
+
+      {
+        key: self.key.present? ? decoder.new.decode_key(self.key) : nil,
+        payload: decoder.decoder.decode(self.message)
+      }
+    end
+
     # @return [Hash]
     def phobos_message
       {

data/lib/deimos/utils/db_producer.rb

@@ -64,22 +64,17 @@ module Deimos
 
         while messages.any?
           @logger.debug do
-            producer = Deimos::Producer.descendants.find { |c| c.topic == topic }
-            decoded_messages = if producer
-                                 consumer = Class.new(Deimos::Consumer)
-                                 consumer.config.merge!(producer.config)
-                                 messages.map do |message|
-                                   {
-                                     key: message[:key].present? ? consumer.new.decode_key(message[:key]) : nil,
-                                     message: consumer.decoder.decode(message[:payload])
-                                   }
-                                 end
-                               else
-                                 messages
-                               end
-            "DB producer: Topic #{topic} Producing messages: #{decoded_messages}"
+            decoder = messages.first.decoder
+            "DB producer: Topic #{topic} Producing messages: #{messages.map { |m| m.decoded_message(decoder) }}"
+          end
+          Deimos.instrument('db_producer.produce', topic: topic, messages: messages) do
+            begin
+              produce_messages(messages.map(&:phobos_message))
+            rescue Kafka::BufferOverflow, Kafka::MessageSizeTooLarge, Kafka::RecordListTooLarge
+              messages.each(&:delete)
+              raise
+            end
           end
-          produce_messages(messages.map(&:phobos_message))
           messages.first.class.where(id: messages.map(&:id)).delete_all
           break if messages.size < BATCH_SIZE
 
@@ -105,6 +100,15 @@ module Deimos
         Deimos.config.metrics&.gauge('pending_db_messages_max_wait', time_diff)
       end
 
+      # Shut down the sync producer if we have to. Phobos will automatically
+      # create a new one. We should call this if the producer can be in a bad
+      # state and e.g. we need to clear the buffer.
+      def shutdown_producer
+        if self.class.producer.respond_to?(:sync_producer_shutdown) # Phobos 1.8.3
+          self.class.producer.sync_producer_shutdown
+        end
+      end
+
       # @param batch [Array<Hash>]
       def produce_messages(batch)
         batch_size = batch.size
@@ -119,18 +123,20 @@ module Deimos
             )
             @logger.info("Sent #{group.size} messages to #{@current_topic}")
           end
-        rescue Kafka::BufferOverflow
-          raise if batch_size == 1
+        rescue Kafka::BufferOverflow, Kafka::MessageSizeTooLarge,
+               Kafka::RecordListTooLarge => e
+          if batch_size == 1
+            shutdown_producer
+            raise
+          end
 
-          @logger.error("Buffer overflow when publishing #{batch.size} in groups of #{batch_size}, retrying...")
+          @logger.error("Got error #{e.class.name} when publishing #{batch.size} in groups of #{batch_size}, retrying...")
           if batch_size < 10
             batch_size = 1
           else
             batch_size /= 10
           end
-          if self.class.producer.respond_to?(:sync_producer_shutdown) # Phobos 1.8.3
-            self.class.producer.sync_producer_shutdown
-          end
+          shutdown_producer
           retry
         end
       end

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.0.0-beta25'
+  VERSION = '1.0.0-beta26'
 end

data/lib/deimos.rb

@@ -124,11 +124,11 @@ end
 
 at_exit do
   begin
-    Deimos::Backends::KafkaAsync.producer.async_producer_shutdown
-    Deimos::Backends::KafkaAsync.producer.kafka_client&.close
+    Deimos::Backends::KafkaAsync.shutdown_producer
+    Deimos::Backends::Kafka.shutdown_producer
   rescue StandardError => e
     Deimos.config.logger.error(
-      "Error closing async producer on shutdown: #{e.message} #{e.backtrace.join("\n")}"
+      "Error closing producer on shutdown: #{e.message} #{e.backtrace.join("\n")}"
     )
   end
 end

data/spec/producer_spec.rb

@@ -148,6 +148,15 @@ module ProducerTest
       expect(MyProducer.topic).to have_sent(anything)
     end
 
+    it 'should send messages after a crash' do
+      expect {
+        Deimos.disable_producers do
+          raise 'OH NOES'
+        end
+      }      .to raise_error('OH NOES')
+      expect(Deimos).not_to be_producers_disabled
+    end
+
     it 'should produce to a prefixed topic' do
       Deimos.configure { |c| c.producer_topic_prefix = 'prefix.' }
       payload = { 'test_id' => 'foo', 'some_int' => 123 }

data/spec/utils/db_producer_spec.rb

@@ -176,6 +176,31 @@ each_db_config(Deimos::Utils::DbProducer) do
       producer.process_topic('my-topic')
     end
 
+    it 'should notify on error' do
+      messages = (1..4).map do |i|
+        Deimos::KafkaMessage.create!(
+          id: i,
+          topic: 'my-topic',
+          message: "mess#{i}",
+          partition_key: "key#{i}"
+        )
+      end
+
+      expect(Deimos::KafkaTopicInfo).to receive(:lock).
+        with('my-topic', 'abc').and_return(true)
+      expect(producer).to receive(:produce_messages).and_raise('OH NOES')
+      expect(producer).to receive(:retrieve_messages).and_return(messages)
+      expect(Deimos::KafkaTopicInfo).to receive(:register_error)
+
+      expect(Deimos::KafkaMessage.count).to eq(4)
+      Deimos.subscribe('db_producer.produce') do |event|
+        expect(event.payload[:exception_object].message).to eq('OH NOES')
+        expect(event.payload[:messages]).to eq(messages)
+      end
+      producer.process_topic('my-topic')
+      expect(Deimos::KafkaMessage.count).to eq(0)
+    end
+
   end
 
   example 'Full integration test' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.0.0.pre.beta25
+  version: 1.0.0.pre.beta26
 platform: ruby
 authors:
 - Daniel Orner
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-08-28 00:00:00.000000000 Z
+date: 2019-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro-patches