deimos-ruby 1.8.1.pre.beta3 → 1.8.1.pre.beta8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 380e0a86786dc4b308858186e5f97a102d13e0d0ac72ca172e941d42b6dfb81a
-  data.tar.gz: ca99315cf3ee096160c24f15d194ce73d77d95b1e7c27b3454e2a5e4165ef156
+  metadata.gz: ec9800e76e6cc4e78af1e9f52d1f5bd0ab46820a223c663684278f5ef7c80e68
+  data.tar.gz: 1b7f4b9cad7933bea186efdf1a7827f8dac5275b296339131ea12c37fcfdd5c7
 SHA512:
-  metadata.gz: 8aa602000e702a9ce3c271b0796f4ea864fdc0acaf0852fdf7bdc3bf51e0eaf2df3265420e2558f37e3c174a5a555d43920323bb3a1539753b8821ae00ad6f32
-  data.tar.gz: c505ef8b587d2a405f9773a3c0f5a178f27811a071a4579ecfbe46c9f75759f1375b773949a384cdd465638ca7e95c53e6e34d8a1b7dd27e5128e03a90c0c3b1
+  metadata.gz: 17ae44bfadfb402e0573ade637c2b5ab69cff91465f8bc3771948a2136e2bbb2c7080ca1dd8e18f7cd11c0f13aa7def2b3557255094e358f002c747915ad4eec
+  data.tar.gz: 1d885ffb79c43422881e48cde1def8a0fe11bfa1a36250792b09f03d0a3ef7444f711e6b7ae4995d2a0aeb9ef6c036d37df60f90ebb0025b3c3cc14d2e48dbd3

data/CHANGELOG.md

@@ -7,6 +7,35 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+## 1.8.1-beta8 - 2020-08-27
+### Fixes :wrench:
+- Moved the TestHelpers hook to `prepend_before` to allow for
+  overriding e.g. in integration tests.
+
+## 1.8.1-beta7 - 2020-08-25
+
+### Fixes :wrench:
+- Fix for crash when sending pending metrics with DB producer.
+- Fix for compacting messages if all the keys are already unique
+  (fixes [#75](https://github.com/flipp-oss/deimos/issues/75))
+
+## 1.8.1-beta6 - 2020-08-13
+
+### Fixes :wrench:
+
+- Fix for consuming nil payloads with Ruby 2.3.
+
+## 1.8.1-beta5 - 2020-08-13
+
+### Fixes :wrench:
+- Fix regression bug which introduces backwards incompatibility
+  with ActiveRecordProducer's `record_attributes` method.
+
+## 1.8.1-beta4 - 2020-08-12
+
+### Fixes :wrench:
+- Fix regression bug where arrays were not being encoded
+
 ## 1.8.1-beta3 - 2020-08-05
 
 ### Fixes :wrench:

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    deimos-ruby (1.8.1.pre.beta3)
+    deimos-ruby (1.8.1.pre.beta7)
       avro_turf (~> 0.11)
       phobos (~> 1.9)
       ruby-kafka (~> 0.7)

data/README.md

@@ -599,7 +599,7 @@ class MyConsumer < Deimos::ActiveRecordConsumer
  
   # Optional override to change the attributes of the record before they
   # are saved.
-  def record_attributes(payload)
+  def record_attributes(payload, key)
     super.merge(:some_field => 'some_value')
   end
 
@@ -680,7 +680,7 @@ class MyConsumer < Deimos::ActiveRecordConsumer
  
   # Optional override to change the attributes of the record before they
   # are saved.
-  def record_attributes(payload)
+  def record_attributes(payload, key)
     super.merge(:some_field => 'some_value')
   end
 end
@@ -921,6 +921,17 @@ Deimos::TestHelpers.schemas_compatible?(schema1, schema2)
 
 ### Integration Test Helpers
 
+When running integration tests, you'll want to override the default test helper settings:
+
+```ruby
+config.before(:each, :my_integration_metadata) do
+  Deimos.configure do
+    producers.backend :kafka
+    schema.backend :avro_schema_registry
+  end
+end
+```
+
 You can use the `InlineConsumer` class to help with integration testing,
 with a full external Kafka running.
 

data/lib/deimos/active_record_consume/batch_consumption.rb

@@ -88,8 +88,13 @@ module Deimos
 
         # Create payloads with payload + key attributes
         upserts = messages.map do |m|
-          record_attributes(m.payload, m.key)&.
-            merge(record_key(m.key))
+          attrs = if self.method(:record_attributes).parameters.size == 2
+                    record_attributes(m.payload, m.key)
+                  else
+                    record_attributes(m.payload)
+                  end
+
+          attrs&.merge(record_key(m.key))
         end
 
         # If overridden record_attributes indicated no record, skip

data/lib/deimos/active_record_consume/message_consumption.rb

@@ -37,7 +37,14 @@ module Deimos
           record = klass.new
           assign_key(record, payload, key)
         end
-        attrs = record_attributes(payload.with_indifferent_access, key)
+
+        # for backwards compatibility
+        # TODO next major release we should deprecate this
+        attrs = if self.method(:record_attributes).parameters.size == 2
+                  record_attributes(payload.with_indifferent_access, key)
+                else
+                  record_attributes(payload.with_indifferent_access)
+                end
         # don't use attributes= - bypass Rails < 5 attr_protected
         attrs.each do |k, v|
           record.send("#{k}=", v)

data/lib/deimos/consume/message_consumption.rb

@@ -10,7 +10,7 @@ module Deimos
 
       # :nodoc:
       def around_consume(payload, metadata)
-        decoded_payload = payload.dup
+        decoded_payload = payload.nil? ? nil : payload.dup
         new_metadata = metadata.dup
         benchmark = Benchmark.measure do
           _with_span do

data/lib/deimos/schema_backends/avro_schema_coercer.rb

@@ -75,6 +75,8 @@ module Deimos
         coerce_union(type, val)
       when :record
         coerce_record(type, val)
+      else
+        val
       end
     end
 

data/lib/deimos/test_helpers.rb

@@ -34,7 +34,7 @@ module Deimos
         end
       end
 
-      before(:each) do
+      prepend_before(:each) do
         client = double('client').as_null_object
         allow(client).to receive(:time) do |*_args, &block|
           block.call

data/lib/deimos/utils/db_producer.rb

@@ -161,7 +161,7 @@ module Deimos
           # the oldest message, or the last time we processed, whichever comes
           # last.
           if message_record
-            record_earliest = record.earliest
+            record_earliest = message_record.earliest
             # SQLite gives a string here
             if record_earliest.is_a?(String)
               record_earliest = Time.zone.parse(record_earliest)
@@ -231,7 +231,7 @@ module Deimos
         return batch if config.compact_topics != :all &&
                         !config.compact_topics.include?(topic)
 
-        batch.reverse.uniq!(&:key).reverse!
+        batch.reverse.uniq(&:key).reverse!
       end
     end
   end

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.8.1-beta3'
+  VERSION = '1.8.1-beta8'
 end

data/spec/consumer_spec.rb

@@ -32,6 +32,12 @@ module ConsumerTest
                            end
     end
 
+    it 'should consume a nil message' do
+      test_consume_message(MyConsumer, nil) do |payload, _metadata|
+        expect(payload).to be_nil
+      end
+    end
+
     it 'should consume a message idempotently' do
       # testing for a crash and re-consuming the same message/metadata
       key = { 'test_id' => 'foo' }

data/spec/kafka_listener_spec.rb

@@ -30,6 +30,7 @@ describe Deimos::KafkaListener do
     end
 
     it 'should listen to publishing errors and republish as Deimos events' do
+      allow(Deimos::Producer).to receive(:descendants).and_return([MyProducer])
       Deimos.subscribe('produce_error') do |event|
         expect(event.payload).to include(
           producer: MyProducer,

data/spec/producer_spec.rb

@@ -246,6 +246,7 @@ module ProducerTest
       MyNestedSchemaProducer.publish(
         'test_id' => 'foo',
         'test_float' => BigDecimal('123.456'),
+        'test_array' => ['1'],
         'some_nested_record' => {
           'some_int' => 123,
           'some_float' => BigDecimal('456.789'),
@@ -257,6 +258,7 @@ module ProducerTest
       expect(MyNestedSchemaProducer.topic).to have_sent(
         'test_id' => 'foo',
         'test_float' => 123.456,
+        'test_array' => ['1'],
         'some_nested_record' => {
           'some_int' => 123,
           'some_float' => 456.789,

data/spec/schemas/com/my-namespace/MyNestedSchema.avsc

@@ -14,6 +14,13 @@
       "type": "float",
       "doc": "test float"
     },
+    {
+      "name": "test_array",
+      "type": {
+        "type": "array",
+        "items": "string"
+      }
+    },
     {
       "name": "some_nested_record",
       "doc": "some nested record",

data/spec/utils/db_producer_spec.rb

@@ -155,6 +155,10 @@ each_db_config(Deimos::Utils::DbProducer) do
         Deimos.configure { |c| c.db_producer.compact_topics = [] }
       end
 
+      it 'should compact messages when all messages are unique' do
+        Deimos.configure { |c| c.db_producer.compact_topics = %w(my-topic my-topic2) }
+        expect(producer.compact_messages(deduped_batch)).to eq(deduped_batch)
+      end
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.8.1.pre.beta3
+  version: 1.8.1.pre.beta8
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-08-05 00:00:00.000000000 Z
+date: 2020-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf