deimos-ruby 1.8.1.pre.beta2 → 1.8.1.pre.beta3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e6f44746ef34713f973b24f4cf80954915545edec5e0f006e4d1a6c751478afa
-  data.tar.gz: 17adb23fdc3e735cf8eab51f4c1df2bbcf1140c9a5696c0f8e17dc38bd728ba9
+  metadata.gz: 380e0a86786dc4b308858186e5f97a102d13e0d0ac72ca172e941d42b6dfb81a
+  data.tar.gz: ca99315cf3ee096160c24f15d194ce73d77d95b1e7c27b3454e2a5e4165ef156
 SHA512:
-  metadata.gz: d920dc6b7b2020364a7095ab61a986a3bb16c5fac5367227ed8cbded422399e78887fc91ecce81d9857c66bce5739f6423335e835ca87612a1a2306c50cd93b0
-  data.tar.gz: 78fd60871b0852b03e91e2223afbe5ac419dc929d6de6340799f8376f89efffd2775e4d93025ab8089cac5a7579d00bc33c64c3f9ec50cc3fb603f6381e3d82a
+  metadata.gz: 8aa602000e702a9ce3c271b0796f4ea864fdc0acaf0852fdf7bdc3bf51e0eaf2df3265420e2558f37e3c174a5a555d43920323bb3a1539753b8821ae00ad6f32
+  data.tar.gz: c505ef8b587d2a405f9773a3c0f5a178f27811a071a4579ecfbe46c9f75759f1375b773949a384cdd465638ca7e95c53e6e34d8a1b7dd27e5128e03a90c0c3b1

data/CHANGELOG.md

@@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+## 1.8.1-beta3 - 2020-08-05
+
+### Fixes :wrench:
+- Simplify decoding messages and handle producer not found
+- Consolidate types in sub-records recursively
+  (fixes [#72](https://github.com/flipp-oss/deimos/issues/72))
+
 ## 1.8.1-beta2 - 2020-07-28
 
 ### Features :star:
@@ -41,6 +48,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added `ActiveRecordConsumer` batch mode
 
 ### Fixes :wrench:
+- Fixes `send_produce_error` to decode `failed_messages` with built-in decoder. 
 - Lag calculation can be incorrect if no messages are being consumed.
 - Fixed bug where printing messages on a MessageSizeTooLarge
   error didn't work.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    deimos-ruby (1.8.1.pre.beta2)
+    deimos-ruby (1.8.1.pre.beta3)
       avro_turf (~> 0.11)
       phobos (~> 1.9)
       ruby-kafka (~> 0.7)

data/lib/deimos/instrumentation.rb

@@ -46,13 +46,18 @@ module Deimos
 
       messages = exception.failed_messages
       messages.group_by(&:topic).each do |topic, batch|
-        next if batch.empty?
+        producer = Deimos::Producer.descendants.find { |c| c.topic == topic }
+        next if batch.empty? || !producer
 
-        producer = batch.first.metadata[:producer_name]
-        payloads = batch.map { |m| m.metadata[:decoded_payload] }
+        decoder = Deimos.schema_backend(schema: producer.config[:schema],
+                                        namespace: producer.config[:namespace])
+        payloads = batch.map { |m| decoder.decode(m.value) }
 
-        Deimos.config.metrics&.count('publish_error', payloads.size,
-                                     tags: %W(topic:#{topic}))
+        Deimos.config.metrics&.increment(
+          'publish_error',
+          tags: %W(topic:#{topic}),
+          by: payloads.size
+        )
         Deimos.instrument(
           'produce_error',
           producer: producer,

data/lib/deimos/schema_backends/avro_schema_coercer.rb

@@ -10,18 +10,37 @@ module Deimos
       @schema = schema
     end
 
-    # @param type [Symbol]
+    # Coerce sub-records in a payload to match the schema.
+    # @param type [Avro::Schema::UnionSchema]
+    # @param val [Object]
+    # @return [Object]
+    def coerce_union(type, val)
+      union_types = type.schemas.map { |s| s.type.to_sym }
+      return nil if val.nil? && union_types.include?(:null)
+
+      schema_type = type.schemas.find { |s| s.type.to_sym != :null }
+      coerce_type(schema_type, val)
+    end
+
+    # Coerce sub-records in a payload to match the schema.
+    # @param type [Avro::Schema::RecordSchema]
+    # @param val [Object]
+    # @return [Object]
+    def coerce_record(type, val)
+      record = val.map do |name, value|
+        field = type.fields.find { |f| f.name == name }
+        coerce_type(field.type, value)
+      end
+      val.keys.zip(record).to_h
+    end
+
+    # Coerce values in a payload to match the schema.
+    # @param type [Avro::Schema]
     # @param val [Object]
     # @return [Object]
     def coerce_type(type, val)
       int_classes = [Time, ActiveSupport::TimeWithZone]
       field_type = type.type.to_sym
-      if field_type == :union
-        union_types = type.schemas.map { |s| s.type.to_sym }
-        return nil if val.nil? && union_types.include?(:null)
-
-        field_type = union_types.find { |t| t != :null }
-      end
 
       case field_type
       when :int, :long
@@ -32,14 +51,12 @@ module Deimos
         else
           val # this will fail
         end
-
       when :float, :double
         if val.is_a?(Numeric) || _is_float_string?(val)
           val.to_f
         else
           val # this will fail
         end
-
       when :string
         if val.respond_to?(:to_str)
           val.to_s
@@ -54,8 +71,10 @@ module Deimos
         else
           true
         end
-      else
-        val
+      when :union
+        coerce_union(type, val)
+      when :record
+        coerce_record(type, val)
       end
     end
 

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.8.1-beta2'
+  VERSION = '1.8.1-beta3'
 end

data/spec/kafka_listener_spec.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+describe Deimos::KafkaListener do
+  include_context 'with widgets'
+
+  prepend_before(:each) do
+    producer_class = Class.new(Deimos::Producer) do
+      schema 'MySchema'
+      namespace 'com.my-namespace'
+      topic 'my-topic'
+      key_config none: true
+    end
+    stub_const('MyProducer', producer_class)
+  end
+
+  before(:each) do
+    Deimos.configure do |c|
+      c.producers.backend = :kafka
+      c.schema.backend = :avro_local
+    end
+    allow_any_instance_of(Kafka::Cluster).to receive(:add_target_topics)
+    allow_any_instance_of(Kafka::Cluster).to receive(:partitions_for).
+      and_raise(Kafka::Error)
+  end
+
+  describe '.send_produce_error' do
+    let(:payloads) do
+      [{ 'test_id' => 'foo', 'some_int' => 123 },
+       { 'test_id' => 'bar', 'some_int' => 124 }]
+    end
+
+    it 'should listen to publishing errors and republish as Deimos events' do
+      Deimos.subscribe('produce_error') do |event|
+        expect(event.payload).to include(
+          producer: MyProducer,
+          topic: 'my-topic',
+          payloads: payloads
+        )
+      end
+      expect(Deimos.config.metrics).to receive(:increment).
+        with('publish_error', tags: %w(topic:my-topic), by: 2)
+      expect { MyProducer.publish_list(payloads) }.to raise_error(Kafka::DeliveryFailed)
+    end
+
+    it 'should not send any notifications when producer is not found' do
+      Deimos.subscribe('produce_error') do |_|
+        raise 'OH NOES'
+      end
+      allow(Deimos::Producer).to receive(:descendants).and_return([])
+      expect(Deimos.config.metrics).not_to receive(:increment).with('publish_error', anything)
+      expect { MyProducer.publish_list(payloads) }.to raise_error(Kafka::DeliveryFailed)
+    end
+  end
+end

data/spec/producer_spec.rb

@@ -41,6 +41,14 @@ module ProducerTest
       end
       stub_const('MyNoKeyProducer', producer_class)
 
+      producer_class = Class.new(Deimos::Producer) do
+        schema 'MyNestedSchema'
+        namespace 'com.my-namespace'
+        topic 'my-topic'
+        key_config field: 'test_id'
+      end
+      stub_const('MyNestedSchemaProducer', producer_class)
+
       producer_class = Class.new(Deimos::Producer) do
         schema 'MySchema'
         namespace 'com.my-namespace'
@@ -233,6 +241,32 @@ module ProducerTest
       )
     end
 
+    it 'should properly encode and coerce values with a nested record' do
+      expect(MyNestedSchemaProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'my-topic-key')
+      MyNestedSchemaProducer.publish(
+        'test_id' => 'foo',
+        'test_float' => BigDecimal('123.456'),
+        'some_nested_record' => {
+          'some_int' => 123,
+          'some_float' => BigDecimal('456.789'),
+          'some_string' => '123',
+          'some_optional_int' => nil
+        },
+        'some_optional_record' => nil
+      )
+      expect(MyNestedSchemaProducer.topic).to have_sent(
+        'test_id' => 'foo',
+        'test_float' => 123.456,
+        'some_nested_record' => {
+          'some_int' => 123,
+          'some_float' => 456.789,
+          'some_string' => '123',
+          'some_optional_int' => nil
+        },
+        'some_optional_record' => nil
+      )
+    end
+
     it 'should error with nothing set' do
       expect {
         MyErrorProducer.publish_list(

data/spec/schemas/com/my-namespace/MyNestedSchema.avsc

@@ -0,0 +1,55 @@
+{
+  "namespace": "com.my-namespace",
+  "name": "MyNestedSchema",
+  "type": "record",
+  "doc": "Test schema",
+  "fields": [
+    {
+      "name": "test_id",
+      "type": "string",
+      "doc": "test string"
+    },
+    {
+      "name": "test_float",
+      "type": "float",
+      "doc": "test float"
+    },
+    {
+      "name": "some_nested_record",
+      "doc": "some nested record",
+      "type": {
+        "name": "MyNestedRecord",
+        "type": "record",
+        "fields": [
+          {
+            "name": "some_int",
+            "type": "int",
+            "doc": "some int"
+          },
+          {
+            "name": "some_float",
+            "type": "float",
+            "doc": "some float"
+          },
+          {
+            "name": "some_string",
+            "type": "string",
+            "doc": "some string"
+          },
+          {
+            "name": "some_optional_int",
+            "type": [ "null", "int" ],
+            "doc": "some optional int",
+            "default": null
+          }
+        ]
+      }
+    },
+    {
+      "name": "some_optional_record",
+      "doc": "some optional record",
+      "type": [ "null", "MyNestedRecord" ],
+      "default": null
+    }
+  ]
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.8.1.pre.beta2
+  version: 1.8.1.pre.beta3
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-07-28 00:00:00.000000000 Z
+date: 2020-08-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf
@@ -428,6 +428,7 @@ files:
 - spec/generators/active_record_generator_spec.rb
 - spec/handlers/my_batch_consumer.rb
 - spec/handlers/my_consumer.rb
+- spec/kafka_listener_spec.rb
 - spec/kafka_source_spec.rb
 - spec/kafka_topic_info_spec.rb
 - spec/message_spec.rb
@@ -441,6 +442,7 @@ files:
 - spec/schema_backends/avro_validation_spec.rb
 - spec/schema_backends/base_spec.rb
 - spec/schemas/com/my-namespace/Generated.avsc
+- spec/schemas/com/my-namespace/MyNestedSchema.avsc
 - spec/schemas/com/my-namespace/MySchema-key.avsc
 - spec/schemas/com/my-namespace/MySchema.avsc
 - spec/schemas/com/my-namespace/MySchemaCompound-key.avsc
@@ -507,6 +509,7 @@ test_files:
 - spec/generators/active_record_generator_spec.rb
 - spec/handlers/my_batch_consumer.rb
 - spec/handlers/my_consumer.rb
+- spec/kafka_listener_spec.rb
 - spec/kafka_source_spec.rb
 - spec/kafka_topic_info_spec.rb
 - spec/message_spec.rb
@@ -520,6 +523,7 @@ test_files:
 - spec/schema_backends/avro_validation_spec.rb
 - spec/schema_backends/base_spec.rb
 - spec/schemas/com/my-namespace/Generated.avsc
+- spec/schemas/com/my-namespace/MyNestedSchema.avsc
 - spec/schemas/com/my-namespace/MySchema-key.avsc
 - spec/schemas/com/my-namespace/MySchema.avsc
 - spec/schemas/com/my-namespace/MySchemaCompound-key.avsc