deimos-ruby 1.9.2 → 1.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 32eb1834570b6a1ea7d61274bf35be7365f16f1bb818c5c1148459bbc5a0b87a
-  data.tar.gz: eb1ea3e922b947c631f05c0e1a41ce65a3c7a366b42e9b5884de499fff14817b
+  metadata.gz: ac454bfdafdeec22f07c9b00386dfcdd687ed1320288384f53f4541eb6037589
+  data.tar.gz: a012d3b33de492bf84d17c74da545dbd0180377fde07b0e52f8002806e3cc489
 SHA512:
-  metadata.gz: f9d27f061f4ddcc3db165c08aaa09c02d80471cfa8b815cb2fc91909a6992615a49697870f517e70fae958d8bd8fa022b2369310f839c5bcba60a415c9a95750
-  data.tar.gz: '0578c18592f11f646ae89b44a12c712ef86403b282ce9624c7e3127f5d54ba9f20faa634fd463e03b1287dd84d4bfddbecc2ee20e503a76b2f83a700c491b72e'
+  metadata.gz: 6bad789e44e21108e652119640e641f4efa217398d8db89b1b734373f5bdc23cbdd60605828e4f6df9e366bac948512c9ff041222db8fd858605fb5a82e7fea9
+  data.tar.gz: 218fb6e258b3a7859644ece11feff5e27738cbe85c20830f704128c5e9fa3d50e764b4d178fc58ca31b35de90839cb1411d1cf98bcfe73c601162c8cb5017d83

data/CHANGELOG.md

@@ -7,6 +7,38 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+## 1.11.0 - 2021-08-27
+
+- ### Fixes :wrench:
+- Fixed issue where ActiveRecord batch consumption could fail when decoding keys.
+
+- ### Roadmap :car:
+- TestHelper does not automatically reset Deimos config before each test. [#120](https://github.com/flipp-oss/deimos/pull/120).
+  **Please note that this is a breaking change**
+
+
+## 1.10.2 - 2021-07-20
+
+- ### Fixes :wrench:
+
+- Fixed issue where producers would stay in an error state after e.g. authorization failures for one topic that wouldn't apply to other topics.
+
+## 1.10.1 - 2021-06-21
+
+- ### Fixes :wrench:
+
+- Fixed crash when trying to decode a nil payload (e.g. during instrumentation of `send_produce_error`.)
+
+## 1.10.0 - 2021-03-22
+
+- ### Roadmap :car:
+
+- Extracted the configuration piece into a separate gem, [fig_tree](https://www.github.com/flipp-oss/fig_tree).
+- Added a `save_record` method to ActiveRecordConsumer in case calling code wants to work with the record before saving.
+
+- ### Fixes :wrench:
+
+- Fixed a regression where the default values for consumer / Phobos listener configs were not correct (they were all nil). This is technically a breaking change, but it puts the configs back the way they were at version 1.4 and matches the documentation.
 
 ## 1.9.2 - 2021-01-29
 

data/Gemfile.lock

@@ -1,71 +1,72 @@
 PATH
   remote: .
   specs:
-    deimos-ruby (1.9.1)
+    deimos-ruby (1.10.2)
       avro_turf (~> 0.11)
+      fig_tree (~> 0.0.2)
       phobos (>= 1.9, < 3.0)
       ruby-kafka (< 2)
-      sigurd (= 0.0.1)
+      sigurd (~> 0.0.1)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (6.1.1)
-      actionpack (= 6.1.1)
-      activesupport (= 6.1.1)
+    actioncable (6.1.3)
+      actionpack (= 6.1.3)
+      activesupport (= 6.1.3)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailbox (6.1.1)
-      actionpack (= 6.1.1)
-      activejob (= 6.1.1)
-      activerecord (= 6.1.1)
-      activestorage (= 6.1.1)
-      activesupport (= 6.1.1)
+    actionmailbox (6.1.3)
+      actionpack (= 6.1.3)
+      activejob (= 6.1.3)
+      activerecord (= 6.1.3)
+      activestorage (= 6.1.3)
+      activesupport (= 6.1.3)
       mail (>= 2.7.1)
-    actionmailer (6.1.1)
-      actionpack (= 6.1.1)
-      actionview (= 6.1.1)
-      activejob (= 6.1.1)
-      activesupport (= 6.1.1)
+    actionmailer (6.1.3)
+      actionpack (= 6.1.3)
+      actionview (= 6.1.3)
+      activejob (= 6.1.3)
+      activesupport (= 6.1.3)
       mail (~> 2.5, >= 2.5.4)
       rails-dom-testing (~> 2.0)
-    actionpack (6.1.1)
-      actionview (= 6.1.1)
-      activesupport (= 6.1.1)
+    actionpack (6.1.3)
+      actionview (= 6.1.3)
+      activesupport (= 6.1.3)
       rack (~> 2.0, >= 2.0.9)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actiontext (6.1.1)
-      actionpack (= 6.1.1)
-      activerecord (= 6.1.1)
-      activestorage (= 6.1.1)
-      activesupport (= 6.1.1)
+    actiontext (6.1.3)
+      actionpack (= 6.1.3)
+      activerecord (= 6.1.3)
+      activestorage (= 6.1.3)
+      activesupport (= 6.1.3)
       nokogiri (>= 1.8.5)
-    actionview (6.1.1)
-      activesupport (= 6.1.1)
+    actionview (6.1.3)
+      activesupport (= 6.1.3)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.1, >= 1.2.0)
-    activejob (6.1.1)
-      activesupport (= 6.1.1)
+    activejob (6.1.3)
+      activesupport (= 6.1.3)
       globalid (>= 0.3.6)
-    activemodel (6.1.1)
-      activesupport (= 6.1.1)
-    activerecord (6.1.1)
-      activemodel (= 6.1.1)
-      activesupport (= 6.1.1)
-    activerecord-import (1.0.7)
+    activemodel (6.1.3)
+      activesupport (= 6.1.3)
+    activerecord (6.1.3)
+      activemodel (= 6.1.3)
+      activesupport (= 6.1.3)
+    activerecord-import (1.0.8)
       activerecord (>= 3.2)
-    activestorage (6.1.1)
-      actionpack (= 6.1.1)
-      activejob (= 6.1.1)
-      activerecord (= 6.1.1)
-      activesupport (= 6.1.1)
+    activestorage (6.1.3)
+      actionpack (= 6.1.3)
+      activejob (= 6.1.3)
+      activerecord (= 6.1.3)
+      activesupport (= 6.1.3)
       marcel (~> 0.3.1)
       mimemagic (~> 0.3.2)
-    activesupport (6.1.1)
+    activesupport (6.1.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -83,17 +84,19 @@ GEM
     concurrent-ruby-ext (1.1.8)
       concurrent-ruby (= 1.1.8)
     crass (1.0.6)
-    database_cleaner (1.8.5)
-    ddtrace (0.45.0)
+    database_cleaner (1.99.0)
+    ddtrace (0.46.0)
       msgpack
     diff-lcs (1.4.4)
     digest-crc (0.6.3)
       rake (>= 12.0.0, < 14.0.0)
-    dogstatsd-ruby (4.8.2)
+    dogstatsd-ruby (4.8.3)
     erubi (1.10.0)
-    excon (0.78.1)
+    excon (0.81.0)
     exponential-backoff (0.0.4)
-    ffi (1.14.2)
+    ffi (1.15.0)
+    fig_tree (0.0.2)
+      activesupport (>= 3.0.0)
     formatador (0.2.5)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
@@ -114,7 +117,7 @@ GEM
     guard-rubocop (1.4.0)
       guard (~> 2.0)
       rubocop (< 2.0)
-    i18n (1.8.7)
+    i18n (1.8.9)
       concurrent-ruby (~> 1.0)
     listen (3.4.1)
       rb-fsevent (~> 0.10, >= 0.10.3)
@@ -132,16 +135,18 @@ GEM
     marcel (0.3.3)
       mimemagic (~> 0.3.2)
     method_source (1.0.0)
-    mimemagic (0.3.5)
+    mimemagic (0.3.10)
+      nokogiri (~> 1)
+      rake
     mini_mime (1.0.2)
-    mini_portile2 (2.5.0)
-    minitest (5.14.3)
-    msgpack (1.4.1)
+    mini_portile2 (2.5.1)
+    minitest (5.14.4)
+    msgpack (1.4.2)
     multi_json (1.15.0)
     mysql2 (0.5.3)
     nenv (0.3.0)
-    nio4r (2.5.4)
-    nokogiri (1.11.1)
+    nio4r (2.5.7)
+    nokogiri (1.11.5)
       mini_portile2 (~> 2.5.0)
       racc (~> 1.4)
     notiffany (0.1.3)
@@ -159,36 +164,36 @@ GEM
       logging
       ruby-kafka
       thor
-    pry (0.13.1)
+    pry (0.14.0)
       coderay (~> 1.1)
       method_source (~> 1.0)
     racc (1.5.2)
     rack (2.2.3)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
-    rails (6.1.1)
-      actioncable (= 6.1.1)
-      actionmailbox (= 6.1.1)
-      actionmailer (= 6.1.1)
-      actionpack (= 6.1.1)
-      actiontext (= 6.1.1)
-      actionview (= 6.1.1)
-      activejob (= 6.1.1)
-      activemodel (= 6.1.1)
-      activerecord (= 6.1.1)
-      activestorage (= 6.1.1)
-      activesupport (= 6.1.1)
+    rails (6.1.3)
+      actioncable (= 6.1.3)
+      actionmailbox (= 6.1.3)
+      actionmailer (= 6.1.3)
+      actionpack (= 6.1.3)
+      actiontext (= 6.1.3)
+      actionview (= 6.1.3)
+      activejob (= 6.1.3)
+      activemodel (= 6.1.3)
+      activerecord (= 6.1.3)
+      activestorage (= 6.1.3)
+      activesupport (= 6.1.3)
       bundler (>= 1.15.0)
-      railties (= 6.1.1)
+      railties (= 6.1.3)
       sprockets-rails (>= 2.0.0)
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
     rails-html-sanitizer (1.3.0)
       loofah (~> 2.3)
-    railties (6.1.1)
-      actionpack (= 6.1.1)
-      activesupport (= 6.1.1)
+    railties (6.1.3)
+      actionpack (= 6.1.3)
+      activesupport (= 6.1.3)
       method_source
       rake (>= 0.8.7)
       thor (~> 1.0)
@@ -197,8 +202,8 @@ GEM
     rb-fsevent (0.10.4)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
-    regexp_parser (2.0.3)
-    rexml (3.2.4)
+    regexp_parser (2.1.1)
+    rexml (3.2.5)
     rspec (3.10.0)
       rspec-core (~> 3.10.0)
       rspec-expectations (~> 3.10.0)
@@ -208,10 +213,10 @@ GEM
     rspec-expectations (3.10.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.1)
+    rspec-mocks (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
-    rspec-rails (4.0.2)
+    rspec-rails (4.1.1)
       actionpack (>= 4.2)
       activesupport (>= 4.2)
       railties (>= 4.2)
@@ -219,7 +224,7 @@ GEM
       rspec-expectations (~> 3.10)
       rspec-mocks (~> 3.10)
       rspec-support (~> 3.10)
-    rspec-support (3.10.1)
+    rspec-support (3.10.2)
     rspec_junit_formatter (0.4.1)
       rspec-core (>= 2, < 4, != 2.12.0)
     rubocop (0.88.0)
@@ -239,7 +244,7 @@ GEM
       digest-crc
     ruby-progressbar (1.11.0)
     shellany (0.0.1)
-    sigurd (0.0.1)
+    sigurd (0.0.3)
       concurrent-ruby (~> 1)
       exponential-backoff
     sprockets (4.0.2)
@@ -284,4 +289,4 @@ DEPENDENCIES
   sqlite3 (~> 1.3)
 
 BUNDLED WITH
-   2.2.5
+   2.2.17

data/README.md

@@ -29,6 +29,7 @@ Built on Phobos and hence Ruby-Kafka.
    * [Running Consumers](#running-consumers)
    * [Metrics](#metrics)
    * [Testing](#testing)
+        * [Test Helpers](#test-helpers)
         * [Integration Test Helpers](#integration-test-helpers)
    * [Utilities](#utilities)
    * [Contributing](#contributing) 
@@ -946,6 +947,56 @@ require 'deimos/test_helpers'
 # Can pass a file path, a string or a hash into this:
 Deimos::TestHelpers.schemas_compatible?(schema1, schema2)
 ```
+### Test Helpers
+
+There are helper methods available to configure Deimos for different types of testing scenarios. 
+Currently there are helpers defined for unit tests and for testing Kafka related code. You can use it as follows:
+
+```ruby
+# The following can be added to a rpsec file so that each unit 
+# test can have the same settings every time it is run
+around(:each) do |example|
+  Deimos::TestHelpers.unit_test!
+  example.run
+  Deimos.config.reset!
+end
+
+# Similarly you can use the Kafka test helper
+around(:each) do |example|
+  Deimos::TestHelpers.kafka_test!
+  example.run
+  Deimos.config.reset!
+end
+
+# Kakfa test helper using schema registry
+around(:each) do |example|
+  Deimos::TestHelpers.full_integration_test!
+  example.run
+  Deimos.config.reset!
+end
+```
+
+With the help of these helper methods, rspec examples can be written without having to tinker with Deimos settings.
+This also prevents Deimos setting changes from leaking in to other examples.
+
+This does not take away the ability to configure Deimos manually in individual examples. Deimos can still be configured like so:
+```ruby
+    it 'should not fail this random test' do
+      
+      Deimos.configure do |config|
+        config.consumers.fatal_error = proc { true }
+        config.consumers.reraise_errors = false
+      end
+      ...
+      expect(some_object).to be_truthy
+      ...
+      Deimos.config.reset!
+    end
+```
+If you are using one of the test helpers in an `around(:each)` block and want to override few settings for one example, 
+you can do it like in the example shown above. These settings would only apply to that specific example and the Deimos conifg should
+reset once the example has finished running.
+
 
 ### Integration Test Helpers
 

data/deimos-ruby.gemspec

@@ -21,7 +21,8 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency('avro_turf', '~> 0.11')
   spec.add_runtime_dependency('phobos', '>= 1.9', '< 3.0')
   spec.add_runtime_dependency('ruby-kafka', '< 2')
-  spec.add_runtime_dependency('sigurd', '0.0.1')
+  spec.add_runtime_dependency('sigurd', '~> 0.0.1')
+  spec.add_runtime_dependency('fig_tree', '~> 0.0.2')
 
   spec.add_development_dependency('activerecord-import')
   spec.add_development_dependency('avro', '~> 1.9')

data/docs/ARCHITECTURE.md

@@ -74,11 +74,7 @@ must define methods such as:
   
 ### Configuration
 
-Deimos has its own `Configurable` module that makes heavy use of `method_missing`
-to provide a very succinct but powerful configuration format (including
-default values, procs, print out as hash, reset, etc.). It also
-allows for multiple blocks to define different objects of the same time
-(like producers, consumers, pollers etc.).
+Deimos uses the [https://www.github.com/flipp_oss/fig_tree](fig_tree) gem for configuration.
 
 The configuration definition for Deimos is in `config/configuration.rb`. In
 addition, there are methods in `config/phobos_config.rb` which translate to/from

data/docs/CONFIGURATION.md

@@ -91,6 +91,7 @@ delivery|`:batch`|The delivery mode for the consumer. Possible values: `:message
 session_timeout|300|Number of seconds after which, if a client hasn't contacted the Kafka cluster, it will be kicked out of the group.
 offset_commit_interval|10|Interval between offset commits, in seconds.
 offset_commit_threshold|0|Number of messages that can be processed before their offsets are committed. If zero, offset commits are not triggered by message processing
+offset_retention_time|nil|The time period that committed offsets will be retained, in seconds. Defaults to the broker setting.
 heartbeat_interval|10|Interval between heartbeats; must be less than the session window.
 backoff|`(1000..60_000)`|Range representing the minimum and maximum number of milliseconds to back off after a consumer error.
 

data/lib/deimos/active_record_consume/message_consumption.rb

@@ -58,6 +58,11 @@ module Deimos
         attrs.each do |k, v|
           record.send("#{k}=", v)
         end
+        save_record(record)
+      end
+
+      # @param record [ActiveRecord::Base]
+      def save_record(record)
         record.created_at ||= Time.zone.now if record.respond_to?(:created_at)
         record.updated_at = Time.zone.now if record.respond_to?(:updated_at)
         record.save!

data/lib/deimos/config/configuration.rb

@@ -1,17 +1,17 @@
 # frozen_string_literal: true
 
+require 'fig_tree'
 require_relative 'phobos_config'
-require_relative 'configurable'
 require_relative '../metrics/mock'
 require_relative '../tracing/mock'
 require 'active_support/core_ext/numeric'
 
 # :nodoc:
 module Deimos
-  include Configurable
+  include FigTree
 
   # :nodoc:
-  class Configurable::ConfigStruct
+  class FigTree::ConfigStruct
     include Deimos::PhobosConfig
   end
 
@@ -60,7 +60,7 @@ module Deimos
     end
   end
 
-  # @param kafka_config [Configurable::ConfigStruct]
+  # @param kafka_config [FigTree::ConfigStruct]
   def self.configure_producer_or_consumer(kafka_config)
     klass = kafka_config.class_name.constantize
     klass.class_eval do
@@ -327,19 +327,19 @@ module Deimos
       # These are the phobos "listener" configs. See CONFIGURATION.md for more
       # info.
       setting :group_id
-      setting :max_concurrency
-      setting :start_from_beginning
+      setting :max_concurrency, 1
+      setting :start_from_beginning, true
       setting :max_bytes_per_partition, 500.kilobytes
-      setting :min_bytes
-      setting :max_wait_time
+      setting :min_bytes, 1
+      setting :max_wait_time, 5
       setting :force_encoding
-      setting :delivery
+      setting :delivery, :batch
       setting :backoff
-      setting :session_timeout
-      setting :offset_commit_interval
-      setting :offset_commit_threshold
+      setting :session_timeout, 300
+      setting :offset_commit_interval, 10
+      setting :offset_commit_threshold, 0
       setting :offset_retention_time
-      setting :heartbeat_interval
+      setting :heartbeat_interval, 10
     end
 
     setting_object :db_poller do

data/lib/deimos/schema_backends/avro_schema_registry.rb

@@ -10,7 +10,7 @@ module Deimos
     class AvroSchemaRegistry < AvroBase
       # @override
       def decode_payload(payload, schema:)
-        avro_turf_messaging.decode(payload, schema_name: schema)
+        avro_turf_messaging.decode(payload.to_s, schema_name: schema)
       end
 
       # @override

data/lib/deimos/schema_backends/base.rb

@@ -40,8 +40,9 @@ module Deimos
       # Decode a payload with a schema. Public method.
       # @param payload [String]
       # @param schema [Symbol|String]
-      # @return [Hash]
+      # @return [Hash,nil]
       def decode(payload, schema: nil)
+        return nil if payload.nil?
         decode_payload(payload, schema: schema || @schema)
       end
 

data/lib/deimos/test_helpers.rb

@@ -18,22 +18,39 @@ module Deimos
       def sent_messages
         Deimos::Backends::Test.sent_messages
       end
+
+      # Set the config to the right settings for a unit test
+      def unit_test!
+        Deimos.configure do |deimos_config|
+          deimos_config.logger = Logger.new(STDOUT)
+          deimos_config.consumers.reraise_errors = true
+          deimos_config.kafka.seed_brokers ||= ['test_broker']
+          deimos_config.schema.backend = Deimos.schema_backend_class.mock_backend
+          deimos_config.producers.backend = :test
+        end
+      end
+
+      # Kafka test config with avro schema registry
+      def full_integration_test!
+        Deimos.configure do |deimos_config|
+          deimos_config.producers.backend = :kafka
+          deimos_config.schema.backend = :avro_schema_registry
+        end
+      end
+
+      # Set the config to the right settings for a kafka test
+      def kafka_test!
+        Deimos.configure do |deimos_config|
+          deimos_config.producers.backend = :kafka
+          deimos_config.schema.backend = :avro_validation
+        end
+      end
     end
 
     included do
 
       RSpec.configure do |config|
 
-        config.before(:suite) do
-          Deimos.configure do |d_config|
-            d_config.logger = Logger.new(STDOUT)
-            d_config.consumers.reraise_errors = true
-            d_config.kafka.seed_brokers ||= ['test_broker']
-            d_config.schema.backend = Deimos.schema_backend_class.mock_backend
-            d_config.producers.backend = :test
-          end
-        end
-
         config.prepend_before(:each) do
           client = double('client').as_null_object
           allow(client).to receive(:time) do |*_args, &block|

data/lib/deimos/utils/db_producer.rb

@@ -60,7 +60,7 @@ module Deimos
       end
 
       # @param topic [String]
-      # @return [String] the topic that was locked, or nil if none were.
+      # @return [String, nil] the topic that was locked, or nil if none were.
       def process_topic(topic)
         # If the topic is already locked, another producer is currently
         # working on it. Move on to the next one.
@@ -76,6 +76,7 @@ module Deimos
       rescue StandardError => e
         @logger.error("Error processing messages for topic #{@current_topic}: #{e.class.name}: #{e.message} #{e.backtrace.join("\n")}")
         KafkaTopicInfo.register_error(@current_topic, @id)
+        shutdown_producer
       end
 
       # Process a single batch in a topic.

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.9.2'
+  VERSION = '1.11.0'
 end

data/spec/config/configuration_spec.rb

@@ -74,34 +74,34 @@ describe Deimos, 'configuration' do
         {
           topic: 'my_consume_topic',
           group_id: 'my_group_id',
-          max_concurrency: nil,
-          start_from_beginning: nil,
+          max_concurrency: 1,
+          start_from_beginning: true,
           max_bytes_per_partition: 524_288,
-          min_bytes: nil,
-          max_wait_time: nil,
+          min_bytes: 1,
+          max_wait_time: 5,
           force_encoding: nil,
-          delivery: nil,
-          session_timeout: nil,
-          offset_commit_interval: nil,
-          offset_commit_threshold: nil,
+          delivery: 'batch',
+          session_timeout: 300,
+          offset_commit_interval: 10,
+          offset_commit_threshold: 0,
           offset_retention_time: nil,
-          heartbeat_interval: nil,
+          heartbeat_interval: 10,
           handler: 'ConsumerTest::MyConsumer'
         }, {
           topic: 'my_batch_consume_topic',
           group_id: 'my_batch_group_id',
-          max_concurrency: nil,
-          start_from_beginning: nil,
+          max_concurrency: 1,
+          start_from_beginning: true,
           max_bytes_per_partition: 500.kilobytes,
-          min_bytes: nil,
-          max_wait_time: nil,
+          min_bytes: 1,
+          max_wait_time: 5,
           force_encoding: nil,
           delivery: 'inline_batch',
-          session_timeout: nil,
-          offset_commit_interval: nil,
-          offset_commit_threshold: nil,
+          session_timeout: 300,
+          offset_commit_interval: 10,
+          offset_commit_threshold: 0,
           offset_retention_time: nil,
-          heartbeat_interval: nil,
+          heartbeat_interval: 10,
           handler: 'ConsumerTest::MyBatchConsumer'
         }
       ],

data/spec/schema_backends/base_spec.rb

@@ -26,4 +26,8 @@ describe Deimos::SchemaBackends::Base do
     backend.decode(payload, schema: 'schema2')
   end
 
+  it 'should return nil if passed nil' do
+    expect(backend.decode(nil, schema: 'schema2')).to be_nil
+  end
+
 end

data/spec/utils/db_producer_spec.rb

@@ -262,11 +262,13 @@ each_db_config(Deimos::Utils::DbProducer) do
     end
 
     it 'should register an error if it gets an error' do
+      allow(producer).to receive(:shutdown_producer)
       expect(producer).to receive(:retrieve_messages).and_raise('OH NOES')
       expect(Deimos::KafkaTopicInfo).to receive(:register_error).
         with('my-topic', 'abc')
       expect(producer).not_to receive(:produce_messages)
       producer.process_topic('my-topic')
+      expect(producer).to have_received(:shutdown_producer)
     end
 
     it 'should move on if it gets a partial batch' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.9.2
+  version: 1.11.0
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-01-29 00:00:00.000000000 Z
+date: 2021-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf
@@ -62,16 +62,30 @@ dependencies:
   name: sigurd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.0.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.0.1
+- !ruby/object:Gem::Dependency
+  name: fig_tree
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.2
 - !ruby/object:Gem::Dependency
   name: activerecord-import
   requirement: !ruby/object:Gem::Requirement
@@ -370,7 +384,6 @@ files:
 - lib/deimos/backends/kafka_async.rb
 - lib/deimos/backends/test.rb
 - lib/deimos/batch_consumer.rb
-- lib/deimos/config/configurable.rb
 - lib/deimos/config/configuration.rb
 - lib/deimos/config/phobos_config.rb
 - lib/deimos/consume/batch_consumption.rb
@@ -428,7 +441,6 @@ files:
 - spec/backends/kafka_async_spec.rb
 - spec/backends/kafka_spec.rb
 - spec/batch_consumer_spec.rb
-- spec/config/configurable_spec.rb
 - spec/config/configuration_spec.rb
 - spec/consumer_spec.rb
 - spec/deimos_spec.rb
@@ -512,7 +524,6 @@ test_files:
 - spec/backends/kafka_async_spec.rb
 - spec/backends/kafka_spec.rb
 - spec/batch_consumer_spec.rb
-- spec/config/configurable_spec.rb
 - spec/config/configuration_spec.rb
 - spec/consumer_spec.rb
 - spec/deimos_spec.rb

data/lib/deimos/config/configurable.rb

@@ -1,278 +0,0 @@
-# frozen_string_literal: true
-
-require 'active_support/concern'
-require 'active_support/callbacks'
-
-module Deimos
-  # Module to allow configuration. Loosely based off of the dry-configuration
-  # gem but with several advantages:
-  # - Works with Ruby 2.3.
-  # - More succinct syntax using method_missing so you do not need to write
-  #   "config.whatever" and can just write "whatever".
-  # - Use nested blocks:
-  #   Deimos.configure do
-  #     config.kafka.ssl do
-  #       enabled true
-  #       ca_cert_file 'my_file'
-  #     end
-  #     config.kafka do
-  #       ssl do
-  #         enabled true
-  #       end
-  #     end
-  #   end
-  # - Allows for arrays of configurations:
-  #   Deimos.configure do |config|
-  #     config.producer do
-  #       class_name 'MyProducer'
-  #       topic 'MyTopic'
-  #     end
-  #   end
-  # - Allows to call `configure` multiple times without crashing.
-  # - Allows to lazy-set default values by passing a proc as a default:
-  #  Deimos.define_settings do |config|
-  #   setting :my_val, default_proc: proc { MyDefault.calculated_value }
-  #  end
-  # - Support for setting up and automatically calling deprecated configurations.
-  # - Support for configuration callbacks.
-  module Configurable
-    extend ActiveSupport::Concern
-
-    ConfigValue = Struct.new(:value, :default_value, :default_proc, :deprecation) do
-
-      # Reset value back to default.
-      def reset!
-        if self.value.is_a?(ConfigStruct)
-          self.value.reset!
-        else
-          self.value = self.default_value
-        end
-      end
-
-      # :nodoc:
-      def clone_and_reset
-        setting = ConfigValue.new(self.value, self.default_value,
-                                  self.default_proc, self.deprecation)
-        setting.reset!
-        setting
-      end
-
-    end
-
-    # Class that defines and keeps the configuration values.
-    class ConfigStruct
-      include ActiveSupport::Callbacks
-
-      define_callbacks :configure
-
-      # @param name [String]
-      def initialize(name)
-        @name = name
-        @settings = {}
-        @setting_objects = {}
-        @setting_templates = {}
-      end
-
-      # Reset config back to default values.
-      def reset!
-        @setting_objects = @setting_templates.map { |k, _| [k, []] }.to_h
-        @settings.values.each(&:reset!)
-      end
-
-      # Mark a configuration as deprecated and replaced with the new config.
-      # @param old_config [String]
-      # @param new_config [String]
-      def deprecate(old_config, new_config)
-        @settings[old_config.to_sym] ||= ConfigValue.new
-        @settings[old_config.to_sym].deprecation = new_config
-      end
-
-      # :nodoc:
-      def inspect
-        "#{@name}: #{@settings.inspect} #{@setting_objects.inspect}"
-      end
-
-      # @return [Hash]
-      def to_h
-        @settings.map { |k, v| [k, v.value] }.to_h
-      end
-
-      # :nodoc:
-      def clone_and_reset
-        new_config = self.clone
-        new_config.setting_objects = new_config.setting_objects.clone
-        new_config.settings = new_config.settings.map { |k, v| [k, v.clone_and_reset] }.to_h
-        new_config
-      end
-
-      # Define a setting template for an array of objects via a block:
-      #   setting_object :producer do
-      #     setting :topic
-      #     setting :class_name
-      #   end
-      # This will create the `producer` method to define these values as well
-      # as the `producer_objects` method to retrieve them.
-      # @param name [Symbol]
-      def setting_object(name, &block)
-        new_config = ConfigStruct.new("#{@name}.#{name}")
-        @setting_objects[name] = []
-        @setting_templates[name] = new_config
-        new_config.instance_eval(&block)
-      end
-
-      # Define a setting with the given name.
-      # @param name [Symbol]
-      # @param default_value [Object]
-      # @param default_proc [Proc]
-      def setting(name, default_value=nil, default_proc: nil, &block)
-        if block_given?
-          # Create a nested setting
-          setting_config = @settings[name]&.value || ConfigStruct.new("#{@name}.#{name}")
-          setting = ConfigValue.new
-          setting.value = setting_config
-          @settings[name] = setting
-          setting_config.instance_eval(&block)
-        else
-          setting = ConfigValue.new
-          setting.default_proc = default_proc
-          setting.default_value = default_value
-          setting.reset!
-          @settings[name] = setting
-        end
-      end
-
-      # :nodoc:
-      def respond_to_missing?(method, include_all=true)
-        method = method.to_s.sub(/=$/, '')
-        method.ends_with?('objects') ||
-          @setting_templates.key?(method.to_sym) ||
-          @settings.key?(method.to_sym) ||
-          super
-      end
-
-      # :nodoc:
-      def method_missing(method, *args, &block)
-        config_key = method.to_s.sub(/=$/, '').to_sym
-
-        # Return the list of setting objects with the given name
-        if config_key.to_s.end_with?('objects')
-          return _setting_object_method(config_key)
-        end
-
-        # Define a new setting object with the given name
-        if @setting_templates.key?(config_key) && block_given?
-          return _new_setting_object_method(config_key, &block)
-        end
-
-        setting = @settings[config_key]
-
-        if setting&.deprecation
-          return _deprecated_config_method(method, *args)
-        end
-
-        return super unless setting
-
-        if block_given?
-          return _block_config_method(config_key, &block)
-        end
-
-        _default_config_method(config_key, *args)
-      end
-
-    protected
-
-      # Only for the clone method
-      attr_accessor :settings, :setting_objects
-
-    private
-
-      def _deprecated_config_method(method, *args)
-        config_key = method.to_s.sub(/=$/, '').to_sym
-        new_config = @settings[config_key].deprecation
-        equals = method.to_s.end_with?('=') ? '=' : ''
-        ActiveSupport::Deprecation.warn("config.#{config_key}#{equals} is deprecated - use config.#{new_config}#{equals}")
-        obj = self
-        messages = new_config.split('.')
-        messages[0..-2].each do |message|
-          obj = obj.send(message)
-        end
-        if args.length.positive?
-          obj.send(messages[-1], args[0])
-        else
-          obj.send(messages[-1])
-        end
-      end
-
-      # Get or set a value.
-      def _default_config_method(config_key, *args)
-        if args.length.positive?
-          # Set the value
-          @settings[config_key].value = args[0]
-        else
-          # Get the value
-          setting = @settings[config_key]
-          if setting.default_proc && setting.value.nil?
-            setting.value = setting.default_proc.call
-          end
-          setting.value
-        end
-      end
-
-      # Define a new setting object and use the passed block to define values.
-      def _new_setting_object_method(config_key, &block)
-        new_config = @setting_templates[config_key].clone_and_reset
-        new_config.instance_eval(&block)
-        @setting_objects[config_key] << new_config
-      end
-
-      # Return a setting object.
-      def _setting_object_method(config_key)
-        key = config_key.to_s.sub(/_objects$/, '').to_sym
-        @setting_objects[key]
-      end
-
-      # Define new values inside a block.
-      def _block_config_method(config_key, &block)
-        unless @settings[config_key].value.is_a?(ConfigStruct)
-          raise "Block called for #{config_key} but it is not a nested config!"
-        end
-
-        @settings[config_key].value.instance_eval(&block)
-      end
-    end
-
-    # :nodoc:
-    module ClassMethods
-      # Define and redefine settings.
-      def define_settings(&block)
-        config.instance_eval(&block)
-      end
-
-      # Configure the settings with values.
-      def configure(&block)
-        if defined?(Rake) && defined?(Rake.application)
-          tasks = Rake.application.top_level_tasks
-          if tasks.any? { |t| %w(assets webpacker yarn).include?(t.split(':').first) }
-            puts 'Skipping Deimos configuration since we are in JS/CSS compilation'
-            return
-          end
-        end
-        config.run_callbacks(:configure) do
-          config.instance_eval(&block)
-        end
-      end
-
-      # @return [ConfigStruct]
-      def config
-        @config ||= ConfigStruct.new('config')
-      end
-
-      # Pass a block to run after configuration is done.
-      def after_configure(&block)
-        mod = self
-        config.class.set_callback(:configure, :after,
-                                  proc { mod.instance_eval(&block) })
-      end
-    end
-  end
-end

data/spec/config/configurable_spec.rb

@@ -1,136 +0,0 @@
-# frozen_string_literal: true
-
-# :nodoc:
-class MyConfig
-  include Deimos::Configurable
-
-  define_settings do
-    setting :set1
-    setting :set2, 'hi mom'
-    setting :group do
-      setting :set3, default_proc: proc { false }
-      setting :set5, (proc { 5 })
-    end
-
-    setting_object :listy do
-      setting :list1, 10
-      setting :list2, 5
-    end
-  end
-end
-
-describe Deimos::Configurable do
-  it 'should configure correctly with default values' do
-    expect(MyConfig.config.set1).to be_nil
-    expect(MyConfig.config.set2).to eq('hi mom')
-    expect(MyConfig.config.group.set3).to eq(false)
-    expect(MyConfig.config.listy_objects).to be_empty
-    expect { MyConfig.config.blah }.to raise_error(NameError)
-    expect { MyConfig.config.group.set4 }.to raise_error(NameError)
-  end
-
-  it 'should not call the proc until it has to' do
-    num_calls = 0
-    value_proc = proc do
-      num_calls += 1
-      num_calls
-    end
-    MyConfig.define_settings do
-      setting :set_with_proc, default_proc: value_proc
-    end
-    expect(num_calls).to eq(0)
-    expect(MyConfig.config.set_with_proc).to eq(1)
-    # calling twice should not call the proc again
-    expect(MyConfig.config.set_with_proc).to eq(1)
-    expect(num_calls).to eq(1)
-  end
-
-  it "should raise error when setting configs that don't exist" do
-    expect { MyConfig.configure { set15 'some_value' } }.to raise_error(NameError)
-  end
-
-  it 'should add values' do
-    MyConfig.configure do |config|
-      config.set1 = 5 # config.x syntax
-      set2 nil # method_missing syntax
-      config.group.set3 = true
-    end
-
-    # second configure should not blow anything away
-    MyConfig.configure do
-      listy do
-        list1 0
-        list2 1
-      end
-      listy do
-        list1 100
-        list2 200
-      end
-    end
-
-    expect(MyConfig.config.set1).to eq(5)
-    expect(MyConfig.config.set2).to be_nil
-    expect(MyConfig.config.group.set3).to eq(true)
-    expect(MyConfig.config.listy_objects.map(&:to_h)).
-      to eq([
-              { list1: 0, list2: 1 },
-              { list1: 100, list2: 200 }
-            ])
-
-    # test reset!
-    MyConfig.config.reset!
-    expect(MyConfig.config.set1).to be_nil
-    expect(MyConfig.config.set2).to eq('hi mom')
-    expect(MyConfig.config.group.set3).to eq(false)
-    expect(MyConfig.config.listy_objects).to be_empty
-  end
-
-  it 'should add with block syntax' do
-    MyConfig.configure do
-      group do
-        set5(proc { 10 })
-      end
-    end
-    expect(MyConfig.config.group.set5.call).to eq(10)
-  end
-
-  it 'should add or redefine settings' do
-    MyConfig.define_settings do
-      setting :group do
-        setting :set6, 15
-        setting :set5, (proc { 15 })
-      end
-      setting_object :notey do
-        setting :note_title, 'some-title'
-      end
-    end
-
-    expect(MyConfig.config.group.set6).to eq(15)
-    expect(MyConfig.config.group.set5.call).to eq(15)
-    expect(MyConfig.config.listy_objects).to be_empty
-    expect(MyConfig.config.notey_objects).to be_empty
-
-    MyConfig.configure do
-      notey do
-        note_title 'hi mom'
-      end
-      listy do
-        list1 0
-      end
-    end
-    expect(MyConfig.config.notey_objects.size).to eq(1)
-    expect(MyConfig.config.notey_objects.first.note_title).to eq('hi mom')
-    expect(MyConfig.config.listy_objects.size).to eq(1)
-    expect(MyConfig.config.listy_objects.first.list1).to eq(0)
-
-    # This should not remove any keys
-    MyConfig.define_settings do
-      setting :group do
-        setting :set6, 20
-      end
-    end
-    expect(MyConfig.config.group.set6).to eq(20)
-    expect(MyConfig.config.group.set5.call).to eq(15)
-  end
-
-end