deimos-ruby 1.10.0 → 1.11.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a614febc556860a51e870e89beb90fde63201fffae3ba7ca78c53d423a65a7b6
-  data.tar.gz: e4676b33fbaf9c105ffd12b89a0090a86b8befd7cf0e09b39078a70e213f72dc
+  metadata.gz: 4c2fe121d207349905ce908ee4bdb3a2d420a545135c25b47aa4c99e5b5153ef
+  data.tar.gz: e28688edca132bd6d67ad4414d5e3616eb2f217274a6c27dd701816520241a76
 SHA512:
-  metadata.gz: 0d418e1d0b27008392da829980c6bdf754111a1f103965dce6882f97d3a02f0220d58756f6f1f0682df12944eac1a012516b0a3612c257342509b6e0b4c3da57
-  data.tar.gz: 1be3de2310297dc6b75a8f98b6c3f84483c6c401d6395dc6752cc57c342f8935911ba1ad11e8eb69e6bfcb19720e8540985c42976033401c39e137a3ab8fbc40
+  metadata.gz: 7ab87d73ff46dcbf415f65579f3a13202f9e7d0e61918672ef0f9e100babbeb4a27bfd1dce791bf5f36d38cea775881bd38ef424df8317cfaf440ce5b44414ed
+  data.tar.gz: 252d9a2a0e65a78c3faa5b420a1504230cd7d621714387930b369183724c33936240ec534181df024e6d4b5688ba8bca6bae2e0ca1572487dd1aadaf1960b97d

data/CHANGELOG.md

@@ -7,6 +7,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+## 1.11.1 - 2021-08-27
+
+- ### Fixes :wrench:
+- Fixed issue where ActiveRecord batch consumption would try to decode keys twice.
+
+- ### Roadmap :car:
+- TestHelper does not automatically reset Deimos config before each test. [#120](https://github.com/flipp-oss/deimos/pull/120).
+  **Please note that this is a breaking change**
+
+
+## 1.10.2 - 2021-07-20
+
+- ### Fixes :wrench:
+
+- Fixed issue where producers would stay in an error state after e.g. authorization failures for one topic that wouldn't apply to other topics.
+
+## 1.10.1 - 2021-06-21
+
+- ### Fixes :wrench:
+
+- Fixed crash when trying to decode a nil payload (e.g. during instrumentation of `send_produce_error`.)
+
 ## 1.10.0 - 2021-03-22
 
 - ### Roadmap :car:

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    deimos-ruby (1.9.2)
+    deimos-ruby (1.11.0)
       avro_turf (~> 0.11)
       fig_tree (~> 0.0.2)
       phobos (>= 1.9, < 3.0)
@@ -92,7 +92,7 @@ GEM
       rake (>= 12.0.0, < 14.0.0)
     dogstatsd-ruby (4.8.3)
     erubi (1.10.0)
-    excon (0.79.0)
+    excon (0.81.0)
     exponential-backoff (0.0.4)
     ffi (1.15.0)
     fig_tree (0.0.2)
@@ -135,16 +135,18 @@ GEM
     marcel (0.3.3)
       mimemagic (~> 0.3.2)
     method_source (1.0.0)
-    mimemagic (0.3.5)
+    mimemagic (0.3.10)
+      nokogiri (~> 1)
+      rake
     mini_mime (1.0.2)
-    mini_portile2 (2.5.0)
+    mini_portile2 (2.5.1)
     minitest (5.14.4)
     msgpack (1.4.2)
     multi_json (1.15.0)
     mysql2 (0.5.3)
     nenv (0.3.0)
     nio4r (2.5.7)
-    nokogiri (1.11.1)
+    nokogiri (1.11.5)
       mini_portile2 (~> 2.5.0)
       racc (~> 1.4)
     notiffany (0.1.3)
@@ -201,7 +203,7 @@ GEM
     rb-inotify (0.10.1)
       ffi (~> 1.0)
     regexp_parser (2.1.1)
-    rexml (3.2.4)
+    rexml (3.2.5)
     rspec (3.10.0)
       rspec-core (~> 3.10.0)
       rspec-expectations (~> 3.10.0)
@@ -287,4 +289,4 @@ DEPENDENCIES
   sqlite3 (~> 1.3)
 
 BUNDLED WITH
-   2.2.5
+   2.2.17

data/README.md

@@ -29,6 +29,7 @@ Built on Phobos and hence Ruby-Kafka.
    * [Running Consumers](#running-consumers)
    * [Metrics](#metrics)
    * [Testing](#testing)
+        * [Test Helpers](#test-helpers)
         * [Integration Test Helpers](#integration-test-helpers)
    * [Utilities](#utilities)
    * [Contributing](#contributing) 
@@ -879,8 +880,7 @@ Also see [deimos.rb](lib/deimos.rb) under `Configure tracing` to see how the tra
 
 # Testing
 
-Deimos comes with a test helper class which sets the various backends
-to mock versions, and provides useful methods for testing consumers.
+Deimos comes with a test helper class which provides useful methods for testing consumers.
 
 In `spec_helper.rb`:
 ```ruby
@@ -889,7 +889,56 @@ RSpec.configure do |config|
 end
 ```
 
-In your test, you now have the following methods available:
+## Test Configuration
+
+```ruby
+# The following can be added to a rpsec file so that each unit 
+# test can have the same settings every time it is run
+around(:each) do |example|
+  Deimos::TestHelpers.unit_test!
+  example.run
+  Deimos.config.reset!
+end
+
+# Similarly you can use the Kafka test helper
+around(:each) do |example|
+  Deimos::TestHelpers.kafka_test!
+  example.run
+  Deimos.config.reset!
+end
+
+# Kakfa test helper using schema registry
+around(:each) do |example|
+  Deimos::TestHelpers.full_integration_test!
+  example.run
+  Deimos.config.reset!
+end
+```
+
+With the help of these helper methods, rspec examples can be written without having to tinker with Deimos settings.
+This also prevents Deimos setting changes from leaking in to other examples.
+
+This does not take away the ability to configure Deimos manually in individual examples. Deimos can still be configured like so:
+```ruby
+    it 'should not fail this random test' do
+      
+      Deimos.configure do |config|
+        config.consumers.fatal_error = proc { true }
+        config.consumers.reraise_errors = false
+      end
+      ...
+      expect(some_object).to be_truthy
+      ...
+      Deimos.config.reset!
+    end
+```
+If you are using one of the test helpers in an `around(:each)` block and want to override few settings for one example, 
+you can do it like in the example shown above. These settings would only apply to that specific example and the Deimos config should
+reset once the example has finished running.
+
+## Test Usage
+
+In your tests, you now have the following methods available:
 ```ruby
 # Pass a consumer class (not instance) to validate a payload against it.
 # This will fail if the payload does not match the schema the consumer
@@ -937,6 +986,8 @@ expect(message).to eq({
 })
 ```
 
+### Test Utilities
+
 There is also a helper method that will let you test if an existing schema
 would be compatible with a new version of it. You can use this in your 
 Ruby console but it would likely not be part of your RSpec test:
@@ -947,19 +998,6 @@ require 'deimos/test_helpers'
 Deimos::TestHelpers.schemas_compatible?(schema1, schema2)
 ```
 
-### Integration Test Helpers
-
-When running integration tests, you'll want to override the default test helper settings:
-
-```ruby
-config.before(:each, :my_integration_metadata) do
-  Deimos.configure do
-    producers.backend :kafka
-    schema.backend :avro_schema_registry
-  end
-end
-```
-
 You can use the `InlineConsumer` class to help with integration testing,
 with a full external Kafka running.
 

data/lib/deimos/active_record_consume/batch_consumption.rb

@@ -44,14 +44,12 @@ module Deimos
       # @param key [String] The encoded key.
       # @return [Hash] The key attributes.
       def record_key(key)
-        decoded_key = decode_key(key)
-
-        if decoded_key.nil?
+        if key.nil?
           {}
-        elsif decoded_key.is_a?(Hash)
-          @key_converter.convert(decoded_key)
+        elsif key.is_a?(Hash)
+          @key_converter.convert(key)
         else
-          { @klass.primary_key => decoded_key }
+          { @klass.primary_key => key }
         end
       end
 

data/lib/deimos/schema_backends/base.rb

@@ -40,8 +40,9 @@ module Deimos
       # Decode a payload with a schema. Public method.
       # @param payload [String]
       # @param schema [Symbol|String]
-      # @return [Hash]
+      # @return [Hash,nil]
       def decode(payload, schema: nil)
+        return nil if payload.nil?
         decode_payload(payload, schema: schema || @schema)
       end
 

data/lib/deimos/test_helpers.rb

@@ -18,22 +18,39 @@ module Deimos
       def sent_messages
         Deimos::Backends::Test.sent_messages
       end
+
+      # Set the config to the right settings for a unit test
+      def unit_test!
+        Deimos.configure do |deimos_config|
+          deimos_config.logger = Logger.new(STDOUT)
+          deimos_config.consumers.reraise_errors = true
+          deimos_config.kafka.seed_brokers ||= ['test_broker']
+          deimos_config.schema.backend = Deimos.schema_backend_class.mock_backend
+          deimos_config.producers.backend = :test
+        end
+      end
+
+      # Kafka test config with avro schema registry
+      def full_integration_test!
+        Deimos.configure do |deimos_config|
+          deimos_config.producers.backend = :kafka
+          deimos_config.schema.backend = :avro_schema_registry
+        end
+      end
+
+      # Set the config to the right settings for a kafka test
+      def kafka_test!
+        Deimos.configure do |deimos_config|
+          deimos_config.producers.backend = :kafka
+          deimos_config.schema.backend = :avro_validation
+        end
+      end
     end
 
     included do
 
       RSpec.configure do |config|
 
-        config.before(:suite) do
-          Deimos.configure do |d_config|
-            d_config.logger = Logger.new(STDOUT)
-            d_config.consumers.reraise_errors = true
-            d_config.kafka.seed_brokers ||= ['test_broker']
-            d_config.schema.backend = Deimos.schema_backend_class.mock_backend
-            d_config.producers.backend = :test
-          end
-        end
-
         config.prepend_before(:each) do
           client = double('client').as_null_object
           allow(client).to receive(:time) do |*_args, &block|

data/lib/deimos/utils/db_producer.rb

@@ -60,7 +60,7 @@ module Deimos
       end
 
       # @param topic [String]
-      # @return [String] the topic that was locked, or nil if none were.
+      # @return [String, nil] the topic that was locked, or nil if none were.
       def process_topic(topic)
         # If the topic is already locked, another producer is currently
         # working on it. Move on to the next one.
@@ -76,6 +76,7 @@ module Deimos
       rescue StandardError => e
         @logger.error("Error processing messages for topic #{@current_topic}: #{e.class.name}: #{e.message} #{e.backtrace.join("\n")}")
         KafkaTopicInfo.register_error(@current_topic, @id)
+        shutdown_producer
       end
 
       # Process a single batch in a topic.

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.10.0'
+  VERSION = '1.11.1'
 end

data/spec/schema_backends/base_spec.rb

@@ -26,4 +26,8 @@ describe Deimos::SchemaBackends::Base do
     backend.decode(payload, schema: 'schema2')
   end
 
+  it 'should return nil if passed nil' do
+    expect(backend.decode(nil, schema: 'schema2')).to be_nil
+  end
+
 end

data/spec/utils/db_producer_spec.rb

@@ -262,11 +262,13 @@ each_db_config(Deimos::Utils::DbProducer) do
     end
 
     it 'should register an error if it gets an error' do
+      allow(producer).to receive(:shutdown_producer)
       expect(producer).to receive(:retrieve_messages).and_raise('OH NOES')
       expect(Deimos::KafkaTopicInfo).to receive(:register_error).
         with('my-topic', 'abc')
       expect(producer).not_to receive(:produce_messages)
       producer.process_topic('my-topic')
+      expect(producer).to have_received(:shutdown_producer)
     end
 
     it 'should move on if it gets a partial batch' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.10.0
+  version: 1.11.1
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-03-22 00:00:00.000000000 Z
+date: 2021-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf