deimos-ruby 1.10.1 → 1.11.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 810ce13be8ab8ba5dc64d586ed863a8603fdcc96961ed1779d1af791589161e8
-  data.tar.gz: c6be10e665fee7e50583c927253877d5e70d1ad3905560601e2cde844debf1cb
+  metadata.gz: a5424d471910dffc66acead04734420173b69b41dae286fac03edb60825936fe
+  data.tar.gz: fe424e2929984d971cb1f48cd88e6a1d78c83783c0b81fd8b8376571ebcf5634
 SHA512:
-  metadata.gz: 42517bc99f57cef76b0f03957f59550aae1d3234b3f2680958b1d8fd6cb8bd431a43459a5b8542c30658146adfd12fe5eecbc608dd9b878e18166edb84f9ee80
-  data.tar.gz: d5c68af5593e66ac67125108a2f583af7f8003a40515816b6797a17c200a5cffd8aef77d56b400f5c44780c42b78ed0a430037dd2b7033776374aa6a1a5cfb97
+  metadata.gz: 9a5f53681cc8fdc79c39c2b42ae11c40f44246ffdf7bcfaeff6dac7b5b1e0cd21de4bc4a931615c1c4c944c0516dcc663a4638ad90d8e8ddf2a4ae9669b4fadc
+  data.tar.gz: fc21c854ebebce35ab5ecf7a113ca24d7ba4c1a0a880dc225d53b3fdf99823f9a1b06a039159a1a0fe0b7acedd45eec29f3b84af09a847efb329b03eef9f46bd

data/CHANGELOG.md

@@ -7,6 +7,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+## 1.11.2 - 2021-08-27
+
+- ### Fixes :wrench:
+- Fixed ActiveRecord batch consumption on MySQL.
+
+- ### Roadmap :car:
+- TestHelper does not automatically reset Deimos config before each test. [#120](https://github.com/flipp-oss/deimos/pull/120).
+  **Please note that this is a breaking change**
+
+
+## 1.10.2 - 2021-07-20
+
+- ### Fixes :wrench:
+
+- Fixed issue where producers would stay in an error state after e.g. authorization failures for one topic that wouldn't apply to other topics.
+
 ## 1.10.1 - 2021-06-21
 
 - ### Fixes :wrench:

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    deimos-ruby (1.10.1)
+    deimos-ruby (1.11.2)
       avro_turf (~> 0.11)
       fig_tree (~> 0.0.2)
       phobos (>= 1.9, < 3.0)

data/README.md

@@ -29,6 +29,7 @@ Built on Phobos and hence Ruby-Kafka.
    * [Running Consumers](#running-consumers)
    * [Metrics](#metrics)
    * [Testing](#testing)
+        * [Test Helpers](#test-helpers)
         * [Integration Test Helpers](#integration-test-helpers)
    * [Utilities](#utilities)
    * [Contributing](#contributing) 
@@ -879,8 +880,7 @@ Also see [deimos.rb](lib/deimos.rb) under `Configure tracing` to see how the tra
 
 # Testing
 
-Deimos comes with a test helper class which sets the various backends
-to mock versions, and provides useful methods for testing consumers.
+Deimos comes with a test helper class which provides useful methods for testing consumers.
 
 In `spec_helper.rb`:
 ```ruby
@@ -889,7 +889,56 @@ RSpec.configure do |config|
 end
 ```
 
-In your test, you now have the following methods available:
+## Test Configuration
+
+```ruby
+# The following can be added to a rpsec file so that each unit 
+# test can have the same settings every time it is run
+around(:each) do |example|
+  Deimos::TestHelpers.unit_test!
+  example.run
+  Deimos.config.reset!
+end
+
+# Similarly you can use the Kafka test helper
+around(:each) do |example|
+  Deimos::TestHelpers.kafka_test!
+  example.run
+  Deimos.config.reset!
+end
+
+# Kakfa test helper using schema registry
+around(:each) do |example|
+  Deimos::TestHelpers.full_integration_test!
+  example.run
+  Deimos.config.reset!
+end
+```
+
+With the help of these helper methods, rspec examples can be written without having to tinker with Deimos settings.
+This also prevents Deimos setting changes from leaking in to other examples.
+
+This does not take away the ability to configure Deimos manually in individual examples. Deimos can still be configured like so:
+```ruby
+    it 'should not fail this random test' do
+      
+      Deimos.configure do |config|
+        config.consumers.fatal_error = proc { true }
+        config.consumers.reraise_errors = false
+      end
+      ...
+      expect(some_object).to be_truthy
+      ...
+      Deimos.config.reset!
+    end
+```
+If you are using one of the test helpers in an `around(:each)` block and want to override few settings for one example, 
+you can do it like in the example shown above. These settings would only apply to that specific example and the Deimos config should
+reset once the example has finished running.
+
+## Test Usage
+
+In your tests, you now have the following methods available:
 ```ruby
 # Pass a consumer class (not instance) to validate a payload against it.
 # This will fail if the payload does not match the schema the consumer
@@ -937,6 +986,8 @@ expect(message).to eq({
 })
 ```
 
+### Test Utilities
+
 There is also a helper method that will let you test if an existing schema
 would be compatible with a new version of it. You can use this in your 
 Ruby console but it would likely not be part of your RSpec test:
@@ -947,19 +998,6 @@ require 'deimos/test_helpers'
 Deimos::TestHelpers.schemas_compatible?(schema1, schema2)
 ```
 
-### Integration Test Helpers
-
-When running integration tests, you'll want to override the default test helper settings:
-
-```ruby
-config.before(:each, :my_integration_metadata) do
-  Deimos.configure do
-    producers.backend :kafka
-    schema.backend :avro_schema_registry
-  end
-end
-```
-
 You can use the `InlineConsumer` class to help with integration testing,
 with a full external Kafka running.
 

data/lib/deimos/active_record_consume/batch_consumption.rb

@@ -44,14 +44,12 @@ module Deimos
       # @param key [String] The encoded key.
       # @return [Hash] The key attributes.
       def record_key(key)
-        decoded_key = decode_key(key)
-
-        if decoded_key.nil?
+        if key.nil?
           {}
-        elsif decoded_key.is_a?(Hash)
-          @key_converter.convert(decoded_key)
+        elsif key.is_a?(Hash)
+          @key_converter.convert(key)
         else
-          { @klass.primary_key => decoded_key }
+          { @klass.primary_key => key }
         end
       end
 
@@ -102,6 +100,10 @@ module Deimos
 
         options = if key_cols.empty?
                     {} # Can't upsert with no key, just do regular insert
+                  elsif ActiveRecord::Base.connection.adapter_name =~ /mysql/i
+                    {
+                      on_duplicate_key_update: :all
+                    }
                   else
                     {
                       on_duplicate_key_update: {

data/lib/deimos/test_helpers.rb

@@ -18,22 +18,39 @@ module Deimos
       def sent_messages
         Deimos::Backends::Test.sent_messages
       end
+
+      # Set the config to the right settings for a unit test
+      def unit_test!
+        Deimos.configure do |deimos_config|
+          deimos_config.logger = Logger.new(STDOUT)
+          deimos_config.consumers.reraise_errors = true
+          deimos_config.kafka.seed_brokers ||= ['test_broker']
+          deimos_config.schema.backend = Deimos.schema_backend_class.mock_backend
+          deimos_config.producers.backend = :test
+        end
+      end
+
+      # Kafka test config with avro schema registry
+      def full_integration_test!
+        Deimos.configure do |deimos_config|
+          deimos_config.producers.backend = :kafka
+          deimos_config.schema.backend = :avro_schema_registry
+        end
+      end
+
+      # Set the config to the right settings for a kafka test
+      def kafka_test!
+        Deimos.configure do |deimos_config|
+          deimos_config.producers.backend = :kafka
+          deimos_config.schema.backend = :avro_validation
+        end
+      end
     end
 
     included do
 
       RSpec.configure do |config|
 
-        config.before(:suite) do
-          Deimos.configure do |d_config|
-            d_config.logger = Logger.new(STDOUT)
-            d_config.consumers.reraise_errors = true
-            d_config.kafka.seed_brokers ||= ['test_broker']
-            d_config.schema.backend = Deimos.schema_backend_class.mock_backend
-            d_config.producers.backend = :test
-          end
-        end
-
         config.prepend_before(:each) do
           client = double('client').as_null_object
           allow(client).to receive(:time) do |*_args, &block|

data/lib/deimos/utils/db_producer.rb

@@ -60,7 +60,7 @@ module Deimos
       end
 
       # @param topic [String]
-      # @return [String] the topic that was locked, or nil if none were.
+      # @return [String, nil] the topic that was locked, or nil if none were.
       def process_topic(topic)
         # If the topic is already locked, another producer is currently
         # working on it. Move on to the next one.
@@ -76,6 +76,7 @@ module Deimos
       rescue StandardError => e
         @logger.error("Error processing messages for topic #{@current_topic}: #{e.class.name}: #{e.message} #{e.backtrace.join("\n")}")
         KafkaTopicInfo.register_error(@current_topic, @id)
+        shutdown_producer
       end
 
       # Process a single batch in a topic.

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.10.1'
+  VERSION = '1.11.2'
 end

data/spec/utils/db_producer_spec.rb

@@ -262,11 +262,13 @@ each_db_config(Deimos::Utils::DbProducer) do
     end
 
     it 'should register an error if it gets an error' do
+      allow(producer).to receive(:shutdown_producer)
       expect(producer).to receive(:retrieve_messages).and_raise('OH NOES')
       expect(Deimos::KafkaTopicInfo).to receive(:register_error).
         with('my-topic', 'abc')
       expect(producer).not_to receive(:produce_messages)
       producer.process_topic('my-topic')
+      expect(producer).to have_received(:shutdown_producer)
     end
 
     it 'should move on if it gets a partial batch' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.10.1
+  version: 1.11.2
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-21 00:00:00.000000000 Z
+date: 2021-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf