deimos-ruby 1.8.1.pre.beta2 → 1.8.1.pre.beta7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e6f44746ef34713f973b24f4cf80954915545edec5e0f006e4d1a6c751478afa
-  data.tar.gz: 17adb23fdc3e735cf8eab51f4c1df2bbcf1140c9a5696c0f8e17dc38bd728ba9
+  metadata.gz: 6ed6bdbc70b4e29dfd893853d2df53ae6779fe3a95adc85a099b270674908135
+  data.tar.gz: ca1da1180acac7a76cb3abd4ce9c4ab16712b16e94f88b79ae5c952d98a74500
 SHA512:
-  metadata.gz: d920dc6b7b2020364a7095ab61a986a3bb16c5fac5367227ed8cbded422399e78887fc91ecce81d9857c66bce5739f6423335e835ca87612a1a2306c50cd93b0
-  data.tar.gz: 78fd60871b0852b03e91e2223afbe5ac419dc929d6de6340799f8376f89efffd2775e4d93025ab8089cac5a7579d00bc33c64c3f9ec50cc3fb603f6381e3d82a
+  metadata.gz: fd3b5728ce13e60b35ffc0b7dd025e402fa4bbd760edc000ca8cd587bca0c7f39d136eba951ebd631a711cf79171716021f04bda38c8262318ab344d8452be34
+  data.tar.gz: '0305200088b9d463c92562be1d9bf27d2727a06c10a147fdb6bace097adc795b2cfca6d5942eb4f06ee0da316e5a41db26b1fd04698d63987e2b45fdf7aea4c7'

data/CHANGELOG.md

@@ -7,6 +7,37 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+## 1.8.1-beta7 - 2020-08-25
+
+### Fixes :wrench:
+- Fix for crash when sending pending metrics with DB producer.
+- Fix for compacting messages if all the keys are already unique
+  (fixes [#75](https://github.com/flipp-oss/deimos/issues/75))
+
+## 1.8.1-beta6 - 2020-08-13
+
+### Fixes :wrench:
+
+- Fix for consuming nil payloads with Ruby 2.3.
+
+## 1.8.1-beta5 - 2020-08-13
+
+### Fixes :wrench:
+- Fix regression bug which introduces backwards incompatibility
+  with ActiveRecordProducer's `record_attributes` method.
+
+## 1.8.1-beta4 - 2020-08-12
+
+### Fixes :wrench:
+- Fix regression bug where arrays were not being encoded
+
+## 1.8.1-beta3 - 2020-08-05
+
+### Fixes :wrench:
+- Simplify decoding messages and handle producer not found
+- Consolidate types in sub-records recursively
+  (fixes [#72](https://github.com/flipp-oss/deimos/issues/72))
+
 ## 1.8.1-beta2 - 2020-07-28
 
 ### Features :star:
@@ -41,6 +72,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added `ActiveRecordConsumer` batch mode
 
 ### Fixes :wrench:
+- Fixes `send_produce_error` to decode `failed_messages` with built-in decoder. 
 - Lag calculation can be incorrect if no messages are being consumed.
 - Fixed bug where printing messages on a MessageSizeTooLarge
   error didn't work.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    deimos-ruby (1.8.1.pre.beta2)
+    deimos-ruby (1.8.1.pre.beta7)
       avro_turf (~> 0.11)
       phobos (~> 1.9)
       ruby-kafka (~> 0.7)

data/README.md

@@ -599,7 +599,7 @@ class MyConsumer < Deimos::ActiveRecordConsumer
  
   # Optional override to change the attributes of the record before they
   # are saved.
-  def record_attributes(payload)
+  def record_attributes(payload, key)
     super.merge(:some_field => 'some_value')
   end
 
@@ -680,7 +680,7 @@ class MyConsumer < Deimos::ActiveRecordConsumer
  
   # Optional override to change the attributes of the record before they
   # are saved.
-  def record_attributes(payload)
+  def record_attributes(payload, key)
     super.merge(:some_field => 'some_value')
   end
 end

data/lib/deimos/active_record_consume/batch_consumption.rb

@@ -88,8 +88,13 @@ module Deimos
 
         # Create payloads with payload + key attributes
         upserts = messages.map do |m|
-          record_attributes(m.payload, m.key)&.
-            merge(record_key(m.key))
+          attrs = if self.method(:record_attributes).parameters.size == 2
+                    record_attributes(m.payload, m.key)
+                  else
+                    record_attributes(m.payload)
+                  end
+
+          attrs&.merge(record_key(m.key))
         end
 
         # If overridden record_attributes indicated no record, skip

data/lib/deimos/active_record_consume/message_consumption.rb

@@ -37,7 +37,14 @@ module Deimos
           record = klass.new
           assign_key(record, payload, key)
         end
-        attrs = record_attributes(payload.with_indifferent_access, key)
+
+        # for backwards compatibility
+        # TODO next major release we should deprecate this
+        attrs = if self.method(:record_attributes).parameters.size == 2
+                  record_attributes(payload.with_indifferent_access, key)
+                else
+                  record_attributes(payload.with_indifferent_access)
+                end
         # don't use attributes= - bypass Rails < 5 attr_protected
         attrs.each do |k, v|
           record.send("#{k}=", v)

data/lib/deimos/consume/message_consumption.rb

@@ -10,7 +10,7 @@ module Deimos
 
       # :nodoc:
       def around_consume(payload, metadata)
-        decoded_payload = payload.dup
+        decoded_payload = payload.nil? ? nil : payload.dup
         new_metadata = metadata.dup
         benchmark = Benchmark.measure do
           _with_span do

data/lib/deimos/instrumentation.rb

@@ -46,13 +46,18 @@ module Deimos
 
       messages = exception.failed_messages
       messages.group_by(&:topic).each do |topic, batch|
-        next if batch.empty?
+        producer = Deimos::Producer.descendants.find { |c| c.topic == topic }
+        next if batch.empty? || !producer
 
-        producer = batch.first.metadata[:producer_name]
-        payloads = batch.map { |m| m.metadata[:decoded_payload] }
+        decoder = Deimos.schema_backend(schema: producer.config[:schema],
+                                        namespace: producer.config[:namespace])
+        payloads = batch.map { |m| decoder.decode(m.value) }
 
-        Deimos.config.metrics&.count('publish_error', payloads.size,
-                                     tags: %W(topic:#{topic}))
+        Deimos.config.metrics&.increment(
+          'publish_error',
+          tags: %W(topic:#{topic}),
+          by: payloads.size
+        )
         Deimos.instrument(
           'produce_error',
           producer: producer,

data/lib/deimos/schema_backends/avro_schema_coercer.rb

@@ -10,18 +10,37 @@ module Deimos
       @schema = schema
     end
 
-    # @param type [Symbol]
+    # Coerce sub-records in a payload to match the schema.
+    # @param type [Avro::Schema::UnionSchema]
+    # @param val [Object]
+    # @return [Object]
+    def coerce_union(type, val)
+      union_types = type.schemas.map { |s| s.type.to_sym }
+      return nil if val.nil? && union_types.include?(:null)
+
+      schema_type = type.schemas.find { |s| s.type.to_sym != :null }
+      coerce_type(schema_type, val)
+    end
+
+    # Coerce sub-records in a payload to match the schema.
+    # @param type [Avro::Schema::RecordSchema]
+    # @param val [Object]
+    # @return [Object]
+    def coerce_record(type, val)
+      record = val.map do |name, value|
+        field = type.fields.find { |f| f.name == name }
+        coerce_type(field.type, value)
+      end
+      val.keys.zip(record).to_h
+    end
+
+    # Coerce values in a payload to match the schema.
+    # @param type [Avro::Schema]
     # @param val [Object]
     # @return [Object]
     def coerce_type(type, val)
       int_classes = [Time, ActiveSupport::TimeWithZone]
       field_type = type.type.to_sym
-      if field_type == :union
-        union_types = type.schemas.map { |s| s.type.to_sym }
-        return nil if val.nil? && union_types.include?(:null)
-
-        field_type = union_types.find { |t| t != :null }
-      end
 
       case field_type
       when :int, :long
@@ -32,14 +51,12 @@ module Deimos
         else
           val # this will fail
         end
-
       when :float, :double
         if val.is_a?(Numeric) || _is_float_string?(val)
           val.to_f
         else
           val # this will fail
         end
-
       when :string
         if val.respond_to?(:to_str)
           val.to_s
@@ -54,6 +71,10 @@ module Deimos
         else
           true
         end
+      when :union
+        coerce_union(type, val)
+      when :record
+        coerce_record(type, val)
       else
         val
       end

data/lib/deimos/utils/db_producer.rb

@@ -161,7 +161,7 @@ module Deimos
           # the oldest message, or the last time we processed, whichever comes
           # last.
           if message_record
-            record_earliest = record.earliest
+            record_earliest = message_record.earliest
             # SQLite gives a string here
             if record_earliest.is_a?(String)
               record_earliest = Time.zone.parse(record_earliest)
@@ -231,7 +231,7 @@ module Deimos
         return batch if config.compact_topics != :all &&
                         !config.compact_topics.include?(topic)
 
-        batch.reverse.uniq!(&:key).reverse!
+        batch.reverse.uniq(&:key).reverse!
       end
     end
   end

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.8.1-beta2'
+  VERSION = '1.8.1-beta7'
 end

data/spec/consumer_spec.rb

@@ -32,6 +32,12 @@ module ConsumerTest
                            end
     end
 
+    it 'should consume a nil message' do
+      test_consume_message(MyConsumer, nil) do |payload, _metadata|
+        expect(payload).to be_nil
+      end
+    end
+
     it 'should consume a message idempotently' do
       # testing for a crash and re-consuming the same message/metadata
       key = { 'test_id' => 'foo' }

data/spec/kafka_listener_spec.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+describe Deimos::KafkaListener do
+  include_context 'with widgets'
+
+  prepend_before(:each) do
+    producer_class = Class.new(Deimos::Producer) do
+      schema 'MySchema'
+      namespace 'com.my-namespace'
+      topic 'my-topic'
+      key_config none: true
+    end
+    stub_const('MyProducer', producer_class)
+  end
+
+  before(:each) do
+    Deimos.configure do |c|
+      c.producers.backend = :kafka
+      c.schema.backend = :avro_local
+    end
+    allow_any_instance_of(Kafka::Cluster).to receive(:add_target_topics)
+    allow_any_instance_of(Kafka::Cluster).to receive(:partitions_for).
+      and_raise(Kafka::Error)
+  end
+
+  describe '.send_produce_error' do
+    let(:payloads) do
+      [{ 'test_id' => 'foo', 'some_int' => 123 },
+       { 'test_id' => 'bar', 'some_int' => 124 }]
+    end
+
+    it 'should listen to publishing errors and republish as Deimos events' do
+      allow(Deimos::Producer).to receive(:descendants).and_return([MyProducer])
+      Deimos.subscribe('produce_error') do |event|
+        expect(event.payload).to include(
+          producer: MyProducer,
+          topic: 'my-topic',
+          payloads: payloads
+        )
+      end
+      expect(Deimos.config.metrics).to receive(:increment).
+        with('publish_error', tags: %w(topic:my-topic), by: 2)
+      expect { MyProducer.publish_list(payloads) }.to raise_error(Kafka::DeliveryFailed)
+    end
+
+    it 'should not send any notifications when producer is not found' do
+      Deimos.subscribe('produce_error') do |_|
+        raise 'OH NOES'
+      end
+      allow(Deimos::Producer).to receive(:descendants).and_return([])
+      expect(Deimos.config.metrics).not_to receive(:increment).with('publish_error', anything)
+      expect { MyProducer.publish_list(payloads) }.to raise_error(Kafka::DeliveryFailed)
+    end
+  end
+end

data/spec/producer_spec.rb

@@ -41,6 +41,14 @@ module ProducerTest
       end
       stub_const('MyNoKeyProducer', producer_class)
 
+      producer_class = Class.new(Deimos::Producer) do
+        schema 'MyNestedSchema'
+        namespace 'com.my-namespace'
+        topic 'my-topic'
+        key_config field: 'test_id'
+      end
+      stub_const('MyNestedSchemaProducer', producer_class)
+
       producer_class = Class.new(Deimos::Producer) do
         schema 'MySchema'
         namespace 'com.my-namespace'
@@ -233,6 +241,34 @@ module ProducerTest
       )
     end
 
+    it 'should properly encode and coerce values with a nested record' do
+      expect(MyNestedSchemaProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'my-topic-key')
+      MyNestedSchemaProducer.publish(
+        'test_id' => 'foo',
+        'test_float' => BigDecimal('123.456'),
+        'test_array' => ['1'],
+        'some_nested_record' => {
+          'some_int' => 123,
+          'some_float' => BigDecimal('456.789'),
+          'some_string' => '123',
+          'some_optional_int' => nil
+        },
+        'some_optional_record' => nil
+      )
+      expect(MyNestedSchemaProducer.topic).to have_sent(
+        'test_id' => 'foo',
+        'test_float' => 123.456,
+        'test_array' => ['1'],
+        'some_nested_record' => {
+          'some_int' => 123,
+          'some_float' => 456.789,
+          'some_string' => '123',
+          'some_optional_int' => nil
+        },
+        'some_optional_record' => nil
+      )
+    end
+
     it 'should error with nothing set' do
       expect {
         MyErrorProducer.publish_list(

data/spec/schemas/com/my-namespace/MyNestedSchema.avsc

@@ -0,0 +1,62 @@
+{
+  "namespace": "com.my-namespace",
+  "name": "MyNestedSchema",
+  "type": "record",
+  "doc": "Test schema",
+  "fields": [
+    {
+      "name": "test_id",
+      "type": "string",
+      "doc": "test string"
+    },
+    {
+      "name": "test_float",
+      "type": "float",
+      "doc": "test float"
+    },
+    {
+      "name": "test_array",
+      "type": {
+        "type": "array",
+        "items": "string"
+      }
+    },
+    {
+      "name": "some_nested_record",
+      "doc": "some nested record",
+      "type": {
+        "name": "MyNestedRecord",
+        "type": "record",
+        "fields": [
+          {
+            "name": "some_int",
+            "type": "int",
+            "doc": "some int"
+          },
+          {
+            "name": "some_float",
+            "type": "float",
+            "doc": "some float"
+          },
+          {
+            "name": "some_string",
+            "type": "string",
+            "doc": "some string"
+          },
+          {
+            "name": "some_optional_int",
+            "type": [ "null", "int" ],
+            "doc": "some optional int",
+            "default": null
+          }
+        ]
+      }
+    },
+    {
+      "name": "some_optional_record",
+      "doc": "some optional record",
+      "type": [ "null", "MyNestedRecord" ],
+      "default": null
+    }
+  ]
+}

data/spec/utils/db_producer_spec.rb

@@ -155,6 +155,10 @@ each_db_config(Deimos::Utils::DbProducer) do
         Deimos.configure { |c| c.db_producer.compact_topics = [] }
       end
 
+      it 'should compact messages when all messages are unique' do
+        Deimos.configure { |c| c.db_producer.compact_topics = %w(my-topic my-topic2) }
+        expect(producer.compact_messages(deduped_batch)).to eq(deduped_batch)
+      end
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.8.1.pre.beta2
+  version: 1.8.1.pre.beta7
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-07-28 00:00:00.000000000 Z
+date: 2020-08-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf
@@ -428,6 +428,7 @@ files:
 - spec/generators/active_record_generator_spec.rb
 - spec/handlers/my_batch_consumer.rb
 - spec/handlers/my_consumer.rb
+- spec/kafka_listener_spec.rb
 - spec/kafka_source_spec.rb
 - spec/kafka_topic_info_spec.rb
 - spec/message_spec.rb
@@ -441,6 +442,7 @@ files:
 - spec/schema_backends/avro_validation_spec.rb
 - spec/schema_backends/base_spec.rb
 - spec/schemas/com/my-namespace/Generated.avsc
+- spec/schemas/com/my-namespace/MyNestedSchema.avsc
 - spec/schemas/com/my-namespace/MySchema-key.avsc
 - spec/schemas/com/my-namespace/MySchema.avsc
 - spec/schemas/com/my-namespace/MySchemaCompound-key.avsc
@@ -507,6 +509,7 @@ test_files:
 - spec/generators/active_record_generator_spec.rb
 - spec/handlers/my_batch_consumer.rb
 - spec/handlers/my_consumer.rb
+- spec/kafka_listener_spec.rb
 - spec/kafka_source_spec.rb
 - spec/kafka_topic_info_spec.rb
 - spec/message_spec.rb
@@ -520,6 +523,7 @@ test_files:
 - spec/schema_backends/avro_validation_spec.rb
 - spec/schema_backends/base_spec.rb
 - spec/schemas/com/my-namespace/Generated.avsc
+- spec/schemas/com/my-namespace/MyNestedSchema.avsc
 - spec/schemas/com/my-namespace/MySchema-key.avsc
 - spec/schemas/com/my-namespace/MySchema.avsc
 - spec/schemas/com/my-namespace/MySchemaCompound-key.avsc