deimos-kafka 1.0.0.pre.beta19 → 1.0.0.pre.beta20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b20ab1bd5462b9ebdd61c00c0e8b70e30e547a42860ece6509b2981f582547ef
-  data.tar.gz: 25d2eb38bed18e090b5729ca7f217cf963fc9396f6ebf141546077597d421053
+  metadata.gz: c85c35c068b88ef2a45ae788e6361ab63ee71fbb27fa4b58f598330bef855de7
+  data.tar.gz: 70d2b1d9e0967d614c1ced30db4222dbeb56256b94ef5cfc8684b3589eb62a0a
 SHA512:
-  metadata.gz: 9be9ce52caf3014dd496bfcbb8a2626f394b76d5af6bf7b5a4ce6dcd3ad5e2bc6af55c225e15d81c672ed653b2dd025d651035b48ffac175367683529d62e3a3
-  data.tar.gz: 222d68c723d352559f79b4ba328dc5aad1ed85a8191dc08266ab7c3113a5853fe7125b2bfba549c4bab081a2082a3b8206f36699461d678eb2eb9d492a0ab8ab
+  metadata.gz: 6c7e5148a6129c9d418a59d9bd0ece9727003aa3df1d37feb05ca21f00d526e357905c20ad44e1b8776af8df4d3dece4f43330fecaa0fb1a2456d0648643d3f9
+  data.tar.gz: 3ddae38b0b3b3616f448f785432feb373d55b991b064c01b5e742ac0454e3f7ea5b6f12d9e16a41cdd542c0ebf9afa08165e7d234a0363a690e6d5a232dff9cd

data/CHANGELOG.md

@@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.0.0-beta20] - 2019-08-07
+- Catch buffer overflows when producing via the DB producer and split the
+  batch up.
+
 ## [1.0.0-beta19] - 2019-08-06
 - Fix for DB producer crashing on error in Rails 3.
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    deimos-kafka (1.0.0.pre.beta19)
+    deimos-kafka (1.0.0.pre.beta20)
       avro-patches (~> 0.3)
       avro_turf (~> 0.8)
       phobos (~> 1.8)
@@ -127,7 +127,7 @@ GEM
       unicode-display_width (>= 1.4.0, < 1.7)
     rubocop-rspec (1.32.0)
       rubocop (>= 0.60.0)
-    ruby-kafka (0.7.9)
+    ruby-kafka (0.7.10)
       digest-crc
     ruby-progressbar (1.10.1)
     ruby_dep (1.5.0)

data/lib/deimos/kafka_topic_info.rb

@@ -65,8 +65,7 @@ module Deimos
         attr_hash = { locked_by: nil,
                       locked_at: Time.zone.now,
                       error: true,
-                      retries: record.retries + 1
-        }
+                      retries: record.retries + 1 }
         if Rails::VERSION::MAJOR >= 4
           record.update!(attr_hash)
         else

data/lib/deimos/utils/db_producer.rb

@@ -81,14 +81,29 @@ module Deimos
 
       # @param batch [Array<Hash>]
       def produce_messages(batch)
-        @logger.debug("Publishing #{batch.size} messages to #{@current_topic}")
-        producer.publish_list(batch)
-        Deimos.config.metrics&.increment(
-          'publish',
-          tags: %W(status:success topic:#{@current_topic}),
-          by: batch.size
-        )
-        @logger.info("Sent #{batch.size} messages to #{@current_topic}")
+        batch_size = batch.size
+        begin
+          batch.in_groups_of(batch_size, false).each do |group|
+            @logger.debug("Publishing #{group.size} messages to #{@current_topic}")
+            producer.publish_list(group)
+            Deimos.config.metrics&.increment(
+              'publish',
+              tags: %W(status:success topic:#{@current_topic}),
+              by: group.size
+            )
+            @logger.info("Sent #{group.size} messages to #{@current_topic}")
+          end
+        rescue Kafka::BufferOverflow
+          raise if batch_size == 1
+
+          @logger.error("Buffer overflow when publishing #{batch.size} in groups of #{batch_size}, retrying...")
+          if batch_size < 10
+            batch_size = 1
+          else
+            batch_size /= 10
+          end
+          retry
+        end
       end
     end
   end

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.0.0-beta19'
+  VERSION = '1.0.0-beta20'
 end

data/spec/utils/db_producer_spec.rb

@@ -4,12 +4,16 @@ each_db_config(Deimos::Utils::DbProducer) do
   let(:producer) do
     producer = described_class.new
     allow(producer).to receive(:sleep)
-    phobos_producer = instance_double(Phobos::Producer::PublicAPI)
-    allow(phobos_producer).to receive(:publish_list)
     allow(producer).to receive(:producer).and_return(phobos_producer)
     producer
   end
 
+  let(:phobos_producer) do
+    pp = instance_double(Phobos::Producer::PublicAPI)
+    allow(pp).to receive(:publish_list)
+    pp
+  end
+
   before(:each) do
     stub_const('Deimos::Utils::DbProducer::BATCH_SIZE', 2)
   end
@@ -44,6 +48,49 @@ each_db_config(Deimos::Utils::DbProducer) do
     expect(messages).to all(be_a_kind_of(Deimos::KafkaMessage))
   end
 
+  describe '#produce_messages' do
+
+    it 'should produce normally' do
+      batch = ['A'] * 1000
+      expect(phobos_producer).to receive(:publish_list).with(batch).once
+      expect(Deimos.config.metrics).to receive(:increment).with('publish',
+                                                                tags: %w(status:success topic:),
+                                                                by: 1000).once
+      producer.produce_messages(batch)
+    end
+
+    it 'should split the batch size on buffer overflow' do
+      count = 0
+      allow(phobos_producer).to receive(:publish_list) do
+        count += 1
+        raise Kafka::BufferOverflow if count < 3
+      end
+      allow(Deimos.config.metrics).to receive(:increment)
+      batch = ['A'] * 1000
+      producer.produce_messages(batch)
+      expect(phobos_producer).to have_received(:publish_list).with(batch)
+      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 100)
+      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 10).exactly(100).times
+      expect(Deimos.config.metrics).to have_received(:increment).with('publish',
+                                                                      tags: %w(status:success topic:),
+                                                                      by: 10).exactly(100).times
+    end
+
+    it "should raise an error if it can't split any more" do
+      allow(phobos_producer).to receive(:publish_list) do
+        raise Kafka::BufferOverflow
+      end
+      expect(Deimos.config.metrics).not_to receive(:increment)
+      batch = ['A'] * 1000
+      expect { producer.produce_messages(batch) }.to raise_error(Kafka::BufferOverflow)
+      expect(phobos_producer).to have_received(:publish_list).with(batch)
+      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 100).once
+      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 10).once
+      expect(phobos_producer).to have_received(:publish_list).with(['A']).once
+
+    end
+  end
+
   describe '#process_topic' do
     before(:each) do
       producer.id = 'abc'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-kafka
 version: !ruby/object:Gem::Version
-  version: 1.0.0.pre.beta19
+  version: 1.0.0.pre.beta20
 platform: ruby
 authors:
 - Daniel Orner
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-08-06 00:00:00.000000000 Z
+date: 2019-08-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro-patches