deimos-ruby 1.8.1.pre.beta5 → 1.8.2.pre.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 516d09f0dc29c3388d1885c67d7faf3c2f240eede7278ad1ac76dfa46727e4e7
-  data.tar.gz: c9fcf141d5dfca3de041d82079f773c53b6a2f663fe7a8b3aa5fbee28964fb19
+  metadata.gz: ce4c0e112957a2a8983372b2eda22b139e1eadac18945d4cbd9945c31e181b9d
+  data.tar.gz: 43df5dcfd68305868cc2ac19f33c070bfc2d6b3c3679fc85116d81939e3a5ce2
 SHA512:
-  metadata.gz: f4b5feee05510a8e6c9d85c9af85aff48f01c43419b5e5600a853d60a62d4b06f8430bb91b11047f6fd2579f23a89e8b9cbafe41a40ed84f7448a54b47115587
-  data.tar.gz: e4c1a0b181f4e5289f995056338b78cab4a8da3cb50863e70ba9f78e8526e470a52a035eaa3a34a0261b1ad14dbff41d2e25a3ffff5e75218408e31aa94a9ed0
+  metadata.gz: ffdbf97e2ad36eafd643cf958b44990a16315da4d346e8b5aaa8dacbdf8d0caecfd493063305b888e240091ab26f5a39bf044b8c3919419f2a6c35c460d774f8
+  data.tar.gz: cb200f4a2d47519f2cb3f0f577a0295690842eb7a8367867749a3cc75236d7205eddb413cc22c28efba7b44b6c8d8a4a98744cd4d3adacedfbc2d64d690397d7

data/CHANGELOG.md

@@ -7,6 +7,32 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+## 1.8.2-beta1 - 2020-09-09
+
+### Features :star:
+
+- Added the ability to specify the topic for `publish`
+and `publish_list` in a producer
+
+## 1.8.1-beta9 - 2020-08-27
+
+### Fixes :wrench:
+- Moved the TestHelpers hook to `before(:suite)` to allow for
+  overriding e.g. in integration tests.
+
+## 1.8.1-beta7 - 2020-08-25
+
+### Fixes :wrench:
+- Fix for crash when sending pending metrics with DB producer.
+- Fix for compacting messages if all the keys are already unique
+  (fixes [#75](https://github.com/flipp-oss/deimos/issues/75))
+
+## 1.8.1-beta6 - 2020-08-13
+
+### Fixes :wrench:
+
+- Fix for consuming nil payloads with Ruby 2.3.
+
 ## 1.8.1-beta5 - 2020-08-13
 
 ### Fixes :wrench:

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    deimos-ruby (1.8.1.pre.beta4)
+    deimos-ruby (1.8.2.pre.beta1)
       avro_turf (~> 0.11)
       phobos (~> 1.9)
       ruby-kafka (~> 0.7)

data/README.md

@@ -108,6 +108,7 @@ class MyProducer < Deimos::Producer
         'some-key2' => an_object.bar
       }
       # You can also publish an array with self.publish_list(payloads)
+      # You may specify the topic here with self.publish(payload, topic: 'my-topic')
       self.publish(payload)
     end
     
@@ -921,6 +922,17 @@ Deimos::TestHelpers.schemas_compatible?(schema1, schema2)
 
 ### Integration Test Helpers
 
+When running integration tests, you'll want to override the default test helper settings:
+
+```ruby
+config.before(:each, :my_integration_metadata) do
+  Deimos.configure do
+    producers.backend :kafka
+    schema.backend :avro_schema_registry
+  end
+end
+```
+
 You can use the `InlineConsumer` class to help with integration testing,
 with a full external Kafka running.
 

data/lib/deimos/consume/message_consumption.rb

@@ -10,7 +10,7 @@ module Deimos
 
       # :nodoc:
       def around_consume(payload, metadata)
-        decoded_payload = payload.dup
+        decoded_payload = payload.nil? ? nil : payload.dup
         new_metadata = metadata.dup
         benchmark = Benchmark.measure do
           _with_span do

data/lib/deimos/producer.rb

@@ -87,8 +87,9 @@ module Deimos
 
       # Publish the payload to the topic.
       # @param payload [Hash] with an optional payload_key hash key.
-      def publish(payload)
-        publish_list([payload])
+      # @param topic [String] if specifying the topic
+      def publish(payload, topic: self.topic)
+        publish_list([payload], topic: topic)
       end
 
       # Publish a list of messages.
@@ -97,7 +98,8 @@ module Deimos
       # whether to publish synchronously.
       # @param force_send [Boolean] if true, ignore the configured backend
       # and send immediately to Kafka.
-      def publish_list(payloads, sync: nil, force_send: false)
+      # @param topic [String] if specifying the topic
+      def publish_list(payloads, sync: nil, force_send: false, topic: self.topic)
         return if Deimos.config.kafka.seed_brokers.blank? ||
                   Deimos.config.producers.disabled ||
                   Deimos.producers_disabled?(self)
@@ -110,7 +112,7 @@ module Deimos
           payloads: payloads
         ) do
           messages = Array(payloads).map { |p| Deimos::Message.new(p, self) }
-          messages.each(&method(:_process_message))
+          messages.each { |m| _process_message(m, topic) }
           messages.in_groups_of(MAX_BATCH_SIZE, false) do |batch|
             self.produce_batch(backend_class, batch)
           end
@@ -163,7 +165,8 @@ module Deimos
     private
 
       # @param message [Message]
-      def _process_message(message)
+      # @param topic [String]
+      def _process_message(message, topic)
         # this violates the Law of Demeter but it has to happen in a very
         # specific order and requires a bunch of methods on the producer
         # to work correctly.
@@ -175,7 +178,7 @@ module Deimos
         message.payload = nil if message.payload.blank?
         message.coerce_fields(encoder)
         message.encoded_key = _encode_key(message.key)
-        message.topic = self.topic
+        message.topic = topic
         message.encoded_payload = if message.payload.nil?
                                     nil
                                   else

data/lib/deimos/test_helpers.rb

@@ -30,18 +30,19 @@ module Deimos
             d_config.consumers.reraise_errors = true
             d_config.kafka.seed_brokers ||= ['test_broker']
             d_config.schema.backend = Deimos.schema_backend_class.mock_backend
+            d_config.producers.backend = :test
           end
         end
-      end
 
-      before(:each) do
-        client = double('client').as_null_object
-        allow(client).to receive(:time) do |*_args, &block|
-          block.call
+        config.before(:each) do
+          client = double('client').as_null_object
+          allow(client).to receive(:time) do |*_args, &block|
+            block.call
+          end
+          Deimos::Backends::Test.sent_messages.clear
         end
-        Deimos.configure { |c| c.producers.backend = :test }
-        Deimos::Backends::Test.sent_messages.clear
       end
+
     end
 
     # @deprecated

data/lib/deimos/utils/db_producer.rb

@@ -161,7 +161,7 @@ module Deimos
           # the oldest message, or the last time we processed, whichever comes
           # last.
           if message_record
-            record_earliest = record.earliest
+            record_earliest = message_record.earliest
             # SQLite gives a string here
             if record_earliest.is_a?(String)
               record_earliest = Time.zone.parse(record_earliest)
@@ -231,7 +231,7 @@ module Deimos
         return batch if config.compact_topics != :all &&
                         !config.compact_topics.include?(topic)
 
-        batch.reverse.uniq!(&:key).reverse!
+        batch.reverse.uniq(&:key).reverse!
       end
     end
   end

data/lib/deimos/utils/inline_consumer.rb

@@ -19,7 +19,7 @@ module Deimos
           offset = last_offset - num_messages
           if offset.positive?
             Deimos.config.logger.info("Seeking to #{offset}")
-            @consumer.seek(topic, 0, offset)
+            @consumer.seek(topic, 1, offset)
           end
         rescue StandardError => e
           "Could not seek to offset: #{e.message}"

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.8.1-beta5'
+  VERSION = '1.8.2-beta1'
 end

data/spec/consumer_spec.rb

@@ -32,6 +32,12 @@ module ConsumerTest
                            end
     end
 
+    it 'should consume a nil message' do
+      test_consume_message(MyConsumer, nil) do |payload, _metadata|
+        expect(payload).to be_nil
+      end
+    end
+
     it 'should consume a message idempotently' do
       # testing for a crash and re-consuming the same message/metadata
       key = { 'test_id' => 'foo' }

data/spec/producer_spec.rb

@@ -102,6 +102,33 @@ module ProducerTest
       expect('my-topic').not_to have_sent('test_id' => 'foo2', 'some_int' => 123)
     end
 
+    it 'should allow setting the topic from publish_list' do
+      expect(described_class).to receive(:produce_batch).once.with(
+        Deimos::Backends::Test,
+        [
+          Deimos::Message.new({ 'test_id' => 'foo', 'some_int' => 123 },
+                              MyProducer,
+                              topic: 'a-new-topic',
+                              partition_key: 'foo',
+                              key: 'foo'),
+          Deimos::Message.new({ 'test_id' => 'bar', 'some_int' => 124 },
+                              MyProducer,
+                              topic: 'a-new-topic',
+                              partition_key: 'bar',
+                              key: 'bar')
+        ]
+      ).and_call_original
+
+      MyProducer.publish_list(
+        [{ 'test_id' => 'foo', 'some_int' => 123 },
+         { 'test_id' => 'bar', 'some_int' => 124 }],
+        topic: 'a-new-topic'
+      )
+      expect('a-new-topic').to have_sent('test_id' => 'foo', 'some_int' => 123)
+      expect('my-topic').not_to have_sent('test_id' => 'foo', 'some_int' => 123)
+      expect('my-topic').not_to have_sent('test_id' => 'foo2', 'some_int' => 123)
+    end
+
     it 'should add a message ID' do
       payload = { 'test_id' => 'foo',
                   'some_int' => 123,

data/spec/spec_helper.rb

@@ -183,6 +183,7 @@ RSpec.configure do |config|
   config.before(:each) do
     Deimos.config.reset!
     Deimos.configure do |deimos_config|
+      deimos_config.producers.backend = :test
       deimos_config.phobos_config_file = File.join(File.dirname(__FILE__), 'phobos.yml')
       deimos_config.schema.path = File.join(File.expand_path(__dir__), 'schemas')
       deimos_config.consumers.reraise_errors = true

data/spec/utils/db_producer_spec.rb

@@ -155,6 +155,10 @@ each_db_config(Deimos::Utils::DbProducer) do
         Deimos.configure { |c| c.db_producer.compact_topics = [] }
       end
 
+      it 'should compact messages when all messages are unique' do
+        Deimos.configure { |c| c.db_producer.compact_topics = %w(my-topic my-topic2) }
+        expect(producer.compact_messages(deduped_batch)).to eq(deduped_batch)
+      end
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.8.1.pre.beta5
+  version: 1.8.2.pre.beta1
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-08-13 00:00:00.000000000 Z
+date: 2020-09-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf