deimos-ruby 1.22.3 → 1.22.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 014d4e4a2682f57a22a065431a0de6ae0ca76fd64cb8e754f391f8f40edd8e3f
-  data.tar.gz: b7bec2abc1e5394ec435dc0a0ef86dbb9b26e379cca0db63f448a89c5b4717ad
+  metadata.gz: 997678ce4ed796037a0f554a7d0897a9ab58f810f437573da7684a6c23fa5d40
+  data.tar.gz: e13fc14fb0bf985a02c38ff628a527198aa1e3bbf1eaab52510ea63832a03f37
 SHA512:
-  metadata.gz: 6ac915ad65b58c77d08bf61c70f978f1ffd37ef58ae39d0e03156b778d5a36ba00de227b855b5ccbd0251d6eda1f4ea7e288b7328961d6fd790cc6a4423b9210
-  data.tar.gz: ca1d551a581a1a968746858a3d99aacc9ace8e7f342cfe4526cc9d27aa158360c2f858b1bf9cc1018c1848b26689b239a0b74f0ccbc2d00b085776ed34a52525
+  metadata.gz: 90062c59b953e4fff9b1f5ddb0c5107e08f98028a7e221c78637bd9cb57c8d13c081a7417dedad57274e4b2df470a58d31ed5a1a1fd985f853ecca9ab53527fc
+  data.tar.gz: fa07d49754b91fdfc48fd6edb46a1906f057ddbd9d0b11d0a66d14382ae1907a0ff7fbd2526e4653e6fc74ed53e78db78203fc043621111193fca370e50da777

data/CHANGELOG.md

@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+# 1.22.5 - 2023-07-18
+- Fix: Fixed buffer overflow crash with DB producer.
+
+# 1.22.4 - 2023-07-05
+- Feature: Add support for message headers.
+
 # 1.22.3 - 2023-06-13
 
 - Fix: Don't update last_sent to current time on every poll.

data/README.md

@@ -123,6 +123,7 @@ class MyProducer < Deimos::Producer
       }
       # You can also publish an array with self.publish_list(payloads)
       # You may specify the topic here with self.publish(payload, topic: 'my-topic')
+      # You may also specify the headers here with self.publish(payload, headers: { 'foo' => 'bar' })
       self.publish(payload)
     end
     
@@ -1171,13 +1172,14 @@ end
                             
 # A matcher which allows you to test that a message was sent on the given
 # topic, without having to know which class produced it.                         
-expect(topic_name).to have_sent(payload, key=nil)
+expect(topic_name).to have_sent(payload, key=nil, partition_key=nil, headers=nil)
 
 # Inspect sent messages
 message = Deimos::Backends::Test.sent_messages[0]
 expect(message).to eq({
   message: {'some-key' => 'some-value'},
   topic: 'my-topic',
+  headers: { 'foo' => 'bar' },
   key: 'my-id'
 })
 ```

data/lib/deimos/backends/base.rb

@@ -37,6 +37,10 @@ module Deimos
             log_message.merge!(
               payloads_count: messages.count
             )
+          when :headers
+            log_message.merge!(
+              payload_headers: messages.map(&:headers)
+            )
           else
             log_message.merge!(
               payloads: messages.map do |message|

data/lib/deimos/kafka_message.rb

@@ -39,11 +39,12 @@ module Deimos
     def self.decoded(messages=[])
       return [] if messages.empty?
 
-      decoder = self.decoder(messages.first.topic)&.new
+      decoder_class = self.decoder(messages.first.topic)
+      decoder = decoder_class&.new
       messages.map do |m|
         {
           key: m.key.present? ? decoder&.decode_key(m.key) || m.key : nil,
-          payload: decoder&.decoder&.decode(m.message) || m.message
+          payload: decoder_class&.decoder&.decode(m.message) || m.message
         }
       end
     end

data/lib/deimos/message.rb

@@ -7,6 +7,8 @@ module Deimos
     attr_accessor :payload
     # @return [Hash, String, Integer]
     attr_accessor :key
+    # @return [Hash]
+    attr_accessor :headers
     # @return [Integer]
     attr_accessor :partition_key
     # @return [String]
@@ -23,11 +25,12 @@ module Deimos
     # @param topic [String]
     # @param key [String, Integer, Hash]
     # @param partition_key [Integer]
-    def initialize(payload, producer, topic: nil, key: nil, partition_key: nil)
+    def initialize(payload, producer, topic: nil, key: nil, headers: nil, partition_key: nil)
       @payload = payload&.with_indifferent_access
       @producer_name = producer&.name
       @topic = topic
       @key = key
+      @headers = headers&.with_indifferent_access
       @partition_key = partition_key
     end
 
@@ -59,13 +62,14 @@ module Deimos
       {
         topic: @topic,
         key: @encoded_key,
+        headers: @headers,
         partition_key: @partition_key || @encoded_key,
         payload: @encoded_payload,
         metadata: {
           decoded_payload: @payload,
           producer_name: @producer_name
         }
-      }
+      }.delete_if { |k, v| k == :headers && v.nil? }
     end
 
     # @return [Hash]
@@ -73,13 +77,14 @@ module Deimos
       {
         topic: @topic,
         key: @key,
+        headers: @headers,
         partition_key: @partition_key || @key,
         payload: @payload,
         metadata: {
           decoded_payload: @payload,
           producer_name: @producer_name
         }
-      }
+      }.delete_if { |k, v| k == :headers && v.nil? }
     end
 
     # @param other [Message]

data/lib/deimos/producer.rb

@@ -95,9 +95,10 @@ module Deimos
       # Publish the payload to the topic.
       # @param payload [Hash, SchemaClass::Record] with an optional payload_key hash key.
       # @param topic [String] if specifying the topic
+      # @param headers [Hash] if specifying headers
       # @return [void]
-      def publish(payload, topic: self.topic)
-        publish_list([payload], topic: topic)
+      def publish(payload, topic: self.topic, headers: nil)
+        publish_list([payload], topic: topic, headers: headers)
       end
 
       # Publish a list of messages.
@@ -107,8 +108,9 @@ module Deimos
       # @param force_send [Boolean] if true, ignore the configured backend
       # and send immediately to Kafka.
       # @param topic [String] if specifying the topic
+      # @param headers [Hash] if specifying headers
       # @return [void]
-      def publish_list(payloads, sync: nil, force_send: false, topic: self.topic)
+      def publish_list(payloads, sync: nil, force_send: false, topic: self.topic, headers: nil)
         return if Deimos.config.kafka.seed_brokers.blank? ||
                   Deimos.config.producers.disabled ||
                   Deimos.producers_disabled?(self)
@@ -122,7 +124,7 @@ module Deimos
           topic: topic,
           payloads: payloads
         ) do
-          messages = Array(payloads).map { |p| Deimos::Message.new(p.to_h, self) }
+          messages = Array(payloads).map { |p| Deimos::Message.new(p.to_h, self, headers: headers) }
           messages.each { |m| _process_message(m, topic) }
           messages.in_groups_of(MAX_BATCH_SIZE, false) do |batch|
             self.produce_batch(backend_class, batch)

data/lib/deimos/test_helpers.rb

@@ -133,7 +133,7 @@ module Deimos
       str + "\nAll Messages received:\n#{message_string}"
     end
 
-    RSpec::Matchers.define :have_sent do |msg, key=nil, partition_key=nil|
+    RSpec::Matchers.define :have_sent do |msg, key=nil, partition_key=nil, headers=nil|
       message = if msg.respond_to?(:with_indifferent_access)
                   msg.with_indifferent_access
                 else
@@ -147,7 +147,14 @@ module Deimos
                             m[:payload]&.with_indifferent_access) &&
             topic == m[:topic] &&
             (key.present? ? key == m[:key] : true) &&
-            (partition_key.present? ? partition_key == m[:partition_key] : true)
+            (partition_key.present? ? partition_key == m[:partition_key] : true) &&
+            if headers.present?
+              hash_matcher.send(:match,
+                                headers&.with_indifferent_access,
+                                m[:headers]&.with_indifferent_access)
+            else
+              true
+            end
         end
       end
 

data/lib/deimos/utils/db_producer.rb

@@ -97,11 +97,16 @@ module Deimos
         Deimos.instrument('db_producer.produce', topic: @current_topic, messages: compacted_messages) do
           begin
             produce_messages(compacted_messages.map(&:phobos_message))
-          rescue Kafka::BufferOverflow, Kafka::MessageSizeTooLarge, Kafka::RecordListTooLarge
+          rescue Kafka::BufferOverflow, Kafka::MessageSizeTooLarge, Kafka::RecordListTooLarge => e
             delete_messages(messages)
             @logger.error('Message batch too large, deleting...')
-            @logger.error(Deimos::KafkaMessage.decoded(messages))
-            raise
+            begin
+              @logger.error(Deimos::KafkaMessage.decoded(messages))
+            rescue StandardError => logging_exception # rubocop:disable Naming/RescuedExceptionsVariableName
+              @logger.error("Large message details logging failure: #{logging_exception.message}")
+            ensure
+              raise e
+            end
           end
         end
         delete_messages(messages)

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.22.3'
+  VERSION = '1.22.5'
 end

data/lib/deimos.rb

@@ -23,7 +23,6 @@ require 'deimos/utils/schema_class'
 require 'deimos/schema_class/enum'
 require 'deimos/schema_class/record'
 
-require 'deimos/monkey_patches/phobos_producer'
 require 'deimos/monkey_patches/phobos_cli'
 
 require 'deimos/railtie' if defined?(Rails)

data/spec/message_spec.rb

@@ -16,4 +16,24 @@ RSpec.describe(Deimos::Message) do
     expect { described_class.new({ a: 1, b: 2 }, nil, key: { c: 3, d: 4 }) }.
       not_to raise_exception
   end
+
+  describe 'headers' do
+    it 'returns nil when not set' do
+      expect(described_class.new({ v: 'val1' }, nil, key: 'key1')).
+        to have_attributes(headers: nil)
+    end
+
+    it 'can set and get headers' do
+      expect(described_class.new({ v: 'val1' }, nil, key: 'key1', headers: { a: 1 })).
+        to have_attributes(headers: { a: 1 })
+    end
+
+    it 'includes headers when converting to Hash' do
+      expect(described_class.new({ v: 'val1' }, nil, key: 'key1', headers: { a: 1 }).to_h).
+        to include(headers: { a: 1 })
+
+      expect(described_class.new({ v: 'val1' }, nil, key: 'key1', headers: { a: 1 }).encoded_hash).
+        to include(headers: { a: 1 })
+    end
+  end
 end

data/spec/producer_spec.rb

@@ -110,18 +110,20 @@ module ProducerTest
       expect('my-topic').not_to have_sent('test_id' => 'foo2', 'some_int' => 123)
     end
 
-    it 'should allow setting the topic from publish_list' do
+    it 'should allow setting the topic and headers from publish_list' do
       expect(described_class).to receive(:produce_batch).once.with(
         Deimos::Backends::Test,
         [
           Deimos::Message.new({ 'test_id' => 'foo', 'some_int' => 123 },
                               MyProducer,
                               topic: 'a-new-topic',
+                              headers: { 'foo' => 'bar' },
                               partition_key: 'foo',
                               key: 'foo'),
           Deimos::Message.new({ 'test_id' => 'bar', 'some_int' => 124 },
                               MyProducer,
                               topic: 'a-new-topic',
+                              headers: { 'foo' => 'bar' },
                               partition_key: 'bar',
                               key: 'bar')
         ]
@@ -130,9 +132,10 @@ module ProducerTest
       MyProducer.publish_list(
         [{ 'test_id' => 'foo', 'some_int' => 123 },
          { 'test_id' => 'bar', 'some_int' => 124 }],
-        topic: 'a-new-topic'
+        topic: 'a-new-topic',
+        headers: { 'foo' => 'bar' }
       )
-      expect('a-new-topic').to have_sent('test_id' => 'foo', 'some_int' => 123)
+      expect('a-new-topic').to have_sent({ 'test_id' => 'foo', 'some_int' => 123 }, nil, nil, { 'foo' => 'bar' })
       expect('my-topic').not_to have_sent('test_id' => 'foo', 'some_int' => 123)
       expect('my-topic').not_to have_sent('test_id' => 'foo2', 'some_int' => 123)
     end

data/spec/utils/db_producer_spec.rb

@@ -2,12 +2,13 @@
 
 each_db_config(Deimos::Utils::DbProducer) do
   let(:producer) do
-    producer = described_class.new
+    producer = described_class.new(logger)
     allow(producer).to receive(:sleep)
     allow(producer).to receive(:producer).and_return(phobos_producer)
     producer
   end
 
+  let(:logger) { nil }
   let(:phobos_producer) do
     pp = instance_double(Phobos::Producer::PublicAPI)
     allow(pp).to receive(:publish_list)
@@ -308,35 +309,6 @@ each_db_config(Deimos::Utils::DbProducer) do
       Deimos.unsubscribe(subscriber)
     end
 
-    it 'should delete messages on buffer overflow' do
-      messages = (1..4).map do |i|
-        Deimos::KafkaMessage.create!(
-          id: i,
-          topic: 'my-topic',
-          message: "mess#{i}",
-          partition_key: "key#{i}"
-        )
-      end
-      (5..8).each do |i|
-        Deimos::KafkaMessage.create!(
-          id: i,
-          topic: 'my-topic2',
-          message: "mess#{i}",
-          partition_key: "key#{i}"
-        )
-      end
-
-      expect(Deimos::KafkaTopicInfo).to receive(:lock).
-        with('my-topic', 'abc').and_return(true)
-      expect(producer).to receive(:produce_messages).and_raise(Kafka::BufferOverflow)
-      expect(producer).to receive(:retrieve_messages).and_return(messages)
-      expect(Deimos::KafkaTopicInfo).to receive(:register_error)
-
-      expect(Deimos::KafkaMessage.count).to eq(8)
-      producer.process_topic('my-topic')
-      expect(Deimos::KafkaMessage.count).to eq(4)
-    end
-
     it 'should retry deletes and not re-publish' do
       messages = (1..4).map do |i|
         Deimos::KafkaMessage.create!(
@@ -388,6 +360,102 @@ each_db_config(Deimos::Utils::DbProducer) do
       expect { producer.delete_messages(messages) }.to raise_exception('OH NOES')
     end
 
+    context 'with buffer overflow exception' do
+      let(:messages) do
+        (1..4).map do |i|
+          Deimos::KafkaMessage.create!(
+            id: i,
+            key: i,
+            topic: 'my-topic',
+            message: { message: "mess#{i}" },
+            partition_key: "key#{i}"
+          )
+        end
+      end
+      let(:logger) do
+        logger = instance_double(Logger)
+        allow(logger).to receive(:error)
+        logger
+      end
+      let(:message_producer) do
+        Deimos.config.schema.backend = :mock
+        Deimos::ActiveRecordProducer.topic('my-topic')
+        Deimos::ActiveRecordProducer.key_config
+        Deimos::ActiveRecordProducer
+      end
+
+      around(:each) do |example|
+        config = Deimos::ActiveRecordProducer.config.clone
+        backend = Deimos.config.schema.backend
+
+        example.run
+      ensure
+        Deimos::ActiveRecordProducer.instance_variable_set(:@config, config)
+        Deimos.config.schema.backend = backend
+      end
+
+      before(:each) do
+        message_producer
+        (5..8).each do |i|
+          Deimos::KafkaMessage.create!(
+            id: i,
+            topic: 'my-topic2',
+            message: "mess#{i}",
+            partition_key: "key#{i}"
+          )
+        end
+        allow(Deimos::KafkaTopicInfo).to receive(:lock).
+          with('my-topic', 'abc').and_return(true)
+        allow(producer).to receive(:produce_messages).and_raise(Kafka::BufferOverflow)
+        allow(producer).to receive(:retrieve_messages).and_return(messages)
+        allow(Deimos::KafkaTopicInfo).to receive(:register_error)
+      end
+
+      it 'should delete messages on buffer overflow' do
+        expect(Deimos::KafkaMessage.count).to eq(8)
+        producer.process_topic('my-topic')
+        expect(Deimos::KafkaMessage.count).to eq(4)
+      end
+
+      it 'should notify on buffer overflow' do
+        subscriber = Deimos.subscribe('db_producer.produce') do |event|
+          expect(event.payload[:exception_object].message).to eq('Kafka::BufferOverflow')
+          expect(event.payload[:messages]).to eq(messages)
+        end
+        producer.process_topic('my-topic')
+        Deimos.unsubscribe(subscriber)
+        expect(logger).to have_received(:error).with('Message batch too large, deleting...')
+        expect(logger).to have_received(:error).with(
+          [
+            { key: '1', payload: 'payload-decoded' },
+            { key: '2', payload: 'payload-decoded' },
+            { key: '3', payload: 'payload-decoded' },
+            { key: '4', payload: 'payload-decoded' }
+          ]
+        )
+      end
+
+      context 'with exception on error logging attempt' do
+        let(:message_producer) do
+          Deimos::ActiveRecordProducer.topic('my-topic')
+          Deimos::ActiveRecordProducer
+        end
+
+        it 'should notify on buffer overflow disregarding decoding exception' do
+          subscriber = Deimos.subscribe('db_producer.produce') do |event|
+            expect(event.payload[:exception_object].message).to eq('Kafka::BufferOverflow')
+            expect(event.payload[:messages]).to eq(messages)
+          end
+          producer.process_topic('my-topic')
+          Deimos.unsubscribe(subscriber)
+          expect(logger).to have_received(:error).with('Message batch too large, deleting...')
+          expect(logger).to have_received(:error).with(
+            'Large message details logging failure: '\
+            'No key config given - if you are not decoding keys, please use `key_config plain: true`'
+          )
+        end
+      end
+    end
   end
 
   describe '#send_pending_metrics' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.22.3
+  version: 1.22.5
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-06-13 00:00:00.000000000 Z
+date: 2023-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf
@@ -459,7 +459,6 @@ files:
 - lib/deimos/metrics/mock.rb
 - lib/deimos/metrics/provider.rb
 - lib/deimos/monkey_patches/phobos_cli.rb
-- lib/deimos/monkey_patches/phobos_producer.rb
 - lib/deimos/poll_info.rb
 - lib/deimos/producer.rb
 - lib/deimos/railtie.rb

data/lib/deimos/monkey_patches/phobos_producer.rb

@@ -1,52 +0,0 @@
-# frozen_string_literal: true
-
-require 'phobos/producer'
-
-#@!visibility private
-module Phobos
-  module Producer
-    # :nodoc:
-    class PublicAPI
-      # :nodoc:
-      def publish(topic, payload, key=nil, partition_key=nil)
-        class_producer.publish(topic, payload, key, partition_key)
-      end
-
-      # :nodoc:
-      def async_publish(topic, payload, key=nil, partition_key=nil)
-        class_producer.async_publish(topic, payload, key, partition_key)
-      end
-    end
-
-    # :nodoc:
-    module ClassMethods
-      # :nodoc:
-      class PublicAPI
-        # :nodoc:
-        def publish(topic, payload, key=nil, partition_key=nil)
-          publish_list([{ topic: topic, payload: payload, key: key,
-                          partition_key: partition_key }])
-        end
-
-        # :nodoc:
-        def async_publish(topic, payload, key=nil, partition_key=nil)
-          async_publish_list([{ topic: topic, payload: payload, key: key,
-                                partition_key: partition_key }])
-        end
-
-      private
-
-        # :nodoc:
-        def produce_messages(producer, messages)
-          messages.each do |message|
-            partition_key = message[:partition_key] || message[:key]
-            producer.produce(message[:payload],
-                             topic: message[:topic],
-                             key: message[:key],
-                             partition_key: partition_key)
-          end
-        end
-      end
-    end
-  end
-end