deimos-ruby 2.0.17 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a6b4516c780cd6c4acfe46092cf4ff7342a3db20b5d259f6fee59e85c2dd6c01
-  data.tar.gz: 02347f3bb35d9cd1c0435f097d453c6f7d90bccee6892d637954c862155164fd
+  metadata.gz: 4e9d857f4fd1fec065ad606bdda5aa5f157e91f69500708b60b3944636f178b8
+  data.tar.gz: 8652654cd562f1205713a485f3d6fd747e785dab31457643d0d3aaa8af8f9d20
 SHA512:
-  metadata.gz: 8b1c2de2fa812f2385b07f3e9fde5135e734571f5bb34d96dd31536ebaf7576c3acdccc7df03098632d549e264d8ef1a5d8f2589995287dda4628dbd535a1840
-  data.tar.gz: 600268d4111cb77b2f4c06a6e07ee95d7e7e4387c298a6099c4498f247a64a443426d639c09b1e30455e3f74ebca7a3790ca5b54abcf34d3ab8918811f58a765
+  metadata.gz: 112a711a7e4c6e596f0c78785b055538f0570296ccbc62aea233c18fb417127bf4ff63154f7555a1e155ed6435e359cc452d8748c041287e59e14ac84aa1dfae
+  data.tar.gz: 1a13b9ddfaeeca69a59b5be6a54cea67b5a5dd8088c13c4fa65d5e98df287a70f21a45ac2d5b4709a9583b3e901aec84d9250d435ee705032be5e461dd2d3781

data/CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+## 2.1.0 - 2025-07-17
+
+- ***BREAKING CHANGE***: Allow schema classes to be used with backwards-compatible Avro messages. In order to update to this version, you will need to regenerate your schema classes using the `deimos:generate_schema_classes` generator.
+
 ## 2.0.17 - 2025-07-03
 
 - Feature: Allow producers to specify `:message` and `:key` in the `publish` methods.

data/README.md

@@ -1083,6 +1083,8 @@ decoded = Deimos.decode(schema: 'MySchema', namespace: 'com.my-namespace', paylo
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/flipp-oss/deimos .
 
+If making changes to the generator, you should regenerate the test schema classes by running `./regenerate_test_schema_classes.rb` .
+
 You can/should re-generate RBS types when methods or classes change by running the following:
 
     rbs collection install # if you haven't done it 

data/karafka.rb

@@ -0,0 +1 @@
+# blank load file, used in regenerate_test_schema_classes.rb

data/lib/deimos/schema_class/enum.rb

@@ -39,7 +39,7 @@ module Deimos
       end
 
       # @return [SchemaClass::Enum]
-      def self.initialize_from_value(value)
+      def self.initialize_from_value(value, from_message: false)
         return nil if value.nil?
 
         value.is_a?(self) ? value : self.new(value)

data/lib/deimos/schema_class/record.rb

@@ -72,11 +72,28 @@ module Deimos
         validator.schema_fields.map(&:name)
       end
 
+      # Used internally within Deimos so that we don't crash on unknown fields that come from
+      # a backwards compatible schema.
+      # @param kwargs [Hash] the attributes to set on the new object.
       # @return [SchemaClass::Record]
-      def self.initialize_from_value(value)
+      def self.new_from_message(**kwargs)
+        @from_message = true
+        record = self.new
+        attrs = kwargs.select { |k, v| record.respond_to?("#{k}=") }
+        self.new(**attrs)
+      end
+
+      # @return [SchemaClass::Record]
+      # @param from_message [Boolean] whether it's being initialized from a real Avro message.
+      def self.initialize_from_value(value, from_message: false)
         return nil if value.nil?
 
-        value.is_a?(self) ? value : self.new(**value.symbolize_keys)
+        return value if value.is_a?(self)
+        if from_message
+          self.new_from_message(**value.symbolize_keys)
+        else
+          self.new(**value.symbolize_keys)
+        end
       end
     end
   end

data/lib/deimos/test_helpers.rb

@@ -147,11 +147,17 @@ module Deimos
                              payload,
                              key: nil,
                              call_original: nil,
-                             partition_key: nil)
+                             partition_key: nil,
+                             &block)
       unless call_original.nil?
         puts "test_consume_message(call_original: true) is deprecated and will be removed in the future. You can remove the call_original parameter."
       end
-      test_consume_batch(handler_class_or_topic, [payload], keys: [key], partition_keys: [partition_key], single: true)
+      test_consume_batch(handler_class_or_topic,
+                         [payload],
+                         keys: [key],
+                         partition_keys: [partition_key],
+                         single: true,
+                         &block)
     end
 
     # Test that a given handler will consume a given batch payload correctly,
@@ -171,7 +177,7 @@ module Deimos
                            keys: [],
                            call_original: nil,
                            single: false,
-                           partition_keys: [])
+                           partition_keys: [], &block)
       unless call_original.nil?
         puts "test_consume_batch(call_original: true) is deprecated and will be removed in the future. You can remove the call_original parameter."
       end
@@ -194,9 +200,15 @@ module Deimos
        karafka.produce(payload, {key: keys[i], partition_key: partition_keys[i], topic: consumer.topic.name})
       end
       if block_given?
-       allow_any_instance_of(consumer_class).to receive(:consume_batch) do
-         yield
-       end
+        if single
+          allow(consumer).to receive(:consume_message) do
+           yield consumer.messages.first.payload, consumer.messages.first.as_json['metadata']
+          end
+        else
+          allow(consumer).to receive(:consume_batch) do
+           yield consumer.messages
+          end
+        end
       end
 
       # sent_messages should only include messages sent by application code, not this method

data/lib/deimos/utils/schema_class.rb

@@ -43,7 +43,7 @@ module Deimos
           klass = klass(schema, namespace)
           return payload if klass.nil? || payload.nil?
 
-          klass.new(**payload.symbolize_keys)
+          klass.new_from_message(**payload.symbolize_keys)
         end
 
         # Determine and return the SchemaClass with the provided schema and namespace

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '2.0.17'
+  VERSION = '2.1.0'
 end

data/lib/generators/deimos/schema_class/templates/schema_record.rb.tt

@@ -49,8 +49,9 @@
     <%- @field_assignments.select{ |h| h[:is_complex_union] }.each do |method_definition| -%>
     # Helper method to determine which schema type to use for <%= method_definition[:field].name %>
     # @param value [Hash, nil]
+    # @param from_message [Boolean] whether this was initialized from a real Avro message
     # @return [Object, nil]
-    def initialize_<%= method_definition[:field].name %>_type(value)
+    def initialize_<%= method_definition[:field].name %>_type(value, from_message: false)
       return nil if value.nil?
 
       klass = [<%= method_definition[:field].type.schemas.reject { |s| s.type_sym == :null }.select { |s| s.type_sym == :record }.map { |s| Deimos::SchemaBackends::AvroBase.schema_classname(s) }.join(', ') %>].find do |candidate|
@@ -58,7 +59,7 @@
         (value.keys - fields).empty?
       end
 
-      klass.initialize_from_value(value)
+      klass.initialize_from_value(value, from_message: @from_message)
     end
 
     <%- end -%>

data/lib/generators/deimos/schema_class_generator.rb

@@ -245,7 +245,7 @@ module Deimos
           "record.tombstone_key = key\n      record.#{key_config[:field]} = key"
         elsif key_schema
           field_base_type = _field_type(key_schema)
-          "record.tombstone_key = #{field_base_type}.initialize_from_value(key)\n      record.payload_key = key"
+          "record.tombstone_key = #{field_base_type}.initialize_from_value(key, from_message: @from_message)\n      record.payload_key = key"
         else
           ''
         end
@@ -299,9 +299,9 @@ module Deimos
           field_initialization = method_argument
 
           if _is_complex_union?(field)
-            field_initialization = "initialize_#{field.name}_type(value)"
+            field_initialization = "initialize_#{field.name}_type(value, from_message: @from_message)"
           elsif is_schema_class
-            field_initialization = "#{field_base_type}.initialize_from_value(value)"
+            field_initialization = "#{field_base_type}.initialize_from_value(value, from_message: @from_message)"
           end
 
           result << {

data/spec/batch_consumer_spec.rb

@@ -5,7 +5,7 @@ module ConsumerTest
   describe Deimos::Consumer, 'Batch Consumer' do
     let(:schema) { 'MySchema' }
     let(:use_schema_classes) { false }
-    let(:reraise_errors) { false }
+    let(:should_reraise_errors) { false }
     let(:key_config) { { field: 'test_id' } }
     let(:consumer_class) do
       Class.new(described_class) do
@@ -22,12 +22,14 @@ module ConsumerTest
       route_schema = schema
       route_key = key_config
       route_use_classes = use_schema_classes
+      should_reraise = should_reraise_errors # for scoping
       Karafka::App.routes.redraw do
         topic 'my-topic' do
           consumer klass
           schema route_schema
           namespace 'com.my-namespace'
           key_config route_key
+          reraise_errors(should_reraise)
           use_schema_classes route_use_classes
         end
       end
@@ -77,7 +79,7 @@ module ConsumerTest
     end
 
     describe 'when reraising errors is disabled' do
-      let(:reraise_errors) { false }
+      let(:should_reraise_errors) { false }
 
       it 'should not fail when before_consume_batch fails' do
         expect {
@@ -98,23 +100,23 @@ module ConsumerTest
       end
 
       it 'should decode payloads for all messages in the batch' do
-        test_consume_batch('my-topic', batch) do
+        test_consume_batch('my-topic', batch) do |received|
           # Mock decoder simply returns the payload
-          expect(messages.payloads).to eq(batch)
+          expect(received.payloads).to eq(batch)
         end
       end
 
       it 'should decode keys for all messages in the batch' do
-        test_consume_batch('my-topic', batch, keys: keys) do
-          expect(messages.map(&:key)).to eq([{'test_id' => 'foo'}, {'test_id' => 'bar'}])
+        test_consume_batch('my-topic', batch, keys: keys) do |received|
+          expect(received.map(&:key)).to eq(%w(foo bar))
         end
       end
 
       context 'with plain keys' do
         let(:key_config) { { plain: true } }
         it 'should decode plain keys for all messages in the batch' do
-          test_consume_batch('my-topic', batch, keys: [1, 2]) do |_received, metadata|
-            expect(metadata[:keys]).to eq([1, 2])
+          test_consume_batch('my-topic', batch, keys: [1, 2]) do |received|
+            expect(received.map(&:key)).to eq(["1", "2"])
           end
         end
       end
@@ -173,8 +175,8 @@ module ConsumerTest
         #                               a_kind_of(Numeric),
         #                               tags: %w(time:time_delayed topic:my-topic)).twice
 
-        test_consume_batch('my-topic', batch_with_time) do
-          expect(messages.payloads).to eq(batch_with_time)
+        test_consume_batch('my-topic', batch_with_time) do |received|
+          expect(received.payloads).to eq(batch_with_time)
         end
       end
 
@@ -186,8 +188,8 @@ module ConsumerTest
         #                               a_kind_of(Numeric),
         #                               tags: %w(time:time_delayed topic:my-topic)).twice
 
-        test_consume_batch('my-topic', batch) do
-          expect(messages.payloads).to eq(batch)
+        test_consume_batch('my-topic', batch) do |received|
+          expect(received.payloads).to eq(batch)
         end
       end
     end

data/spec/consumer_spec.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'rails/generators'
+
 # :nodoc:
 # rubocop:disable Metrics/ModuleLength
 module ConsumerTest
@@ -12,7 +14,7 @@ module ConsumerTest
 
         # :nodoc:
         def fatal_error?(_exception, messages)
-          messages.payloads.first&.dig(:test_id) == ['fatal']
+          messages.payloads.first&.to_h&.dig(:test_id) == ['fatal']
         end
 
         # :nodoc:
@@ -29,6 +31,7 @@ module ConsumerTest
           namespace 'com.my-namespace'
           key_config field: 'test_id'
           consumer consumer_class
+          reraise_errors true
           use_schema_classes route_usc
           reraise_errors route_rre
         end
@@ -43,6 +46,10 @@ module ConsumerTest
           before(:each) do
             Deimos.configure do |config|
               config.schema.use_full_namespace = true
+              config.schema.schema_namespace_map = {
+                'com' => 'Schemas',
+                'com.my-namespace.my-suborg' => %w(Schemas MyNamespace)
+              }
             end
           end
 
@@ -56,8 +63,8 @@ module ConsumerTest
           end
 
           it 'should consume a nil message' do
-            test_consume_message(MyConsumer, nil, key: 'foo') do
-              expect(messages).to be_empty
+            test_consume_message(MyConsumer, nil, key: 'foo') do |message|
+              expect(message).to be_nil
             end
           end
 
@@ -118,7 +125,7 @@ module ConsumerTest
         end
       end
 
-      context 'with overriden schema classes' do
+      context 'with overridden schema classes' do
 
         before(:each) do
           set_karafka_config(:use_schema_classes, true)
@@ -142,6 +149,7 @@ module ConsumerTest
               namespace 'com.my-namespace'
               key_config field: 'test_id'
               consumer consumer_class
+              reraise_errors true
             end
           end
         end
@@ -160,8 +168,46 @@ module ConsumerTest
         end
 
       end
-    end
 
+      context 'with backwards compatible schema classes' do
+        before(:each) do
+          set_karafka_config(:use_schema_classes, true)
+          Deimos.configure do |config|
+            config.schema.use_full_namespace = true
+            config.schema.schema_namespace_map = {
+              'com' => 'Schemas',
+              'com.my-namespace.my-suborg' => %w(Schemas MyNamespace)
+            }
+          end
+          # update the Avro to include a new field
+          generator.append_to_file(schema_file, additional_field_json, after: '"fields": [')
+        end
+
+        after(:each) do
+          generator.gsub_file(schema_file, additional_field_json, '')
+        end
+
+        let(:generator) { Rails::Generators::Base.new }
+        let(:schema_file) { File.join(__dir__, 'schemas/com/my-namespace/MySchema.avsc') }
+        let(:additional_field_json) do
+          '{"name": "additional_field", "type": "string", "default": ""},'
+        end
+
+        it 'should consume correctly and ignore the additional field' do
+          test_consume_message('my_consume_topic',
+                               { 'test_id' => 'foo',
+                                 'additional_field' => 'bar',
+                                 'some_int' => 1 }) do |payload, _metadata|
+            expect(payload['test_id']).to eq('foo')
+            expect(payload['some_int']).to eq(1)
+            expect(payload.to_h).not_to have_key('additional_field')
+          end
+
+        end
+      end
+
+    end
   end
 end
+
 # rubocop:enable Metrics/ModuleLength

data/spec/schemas/my_namespace/generated.rb

@@ -81,12 +81,12 @@ module Schemas; module MyNamespace
     ### Attribute Writers ###
     # @return [AnEnum]
     def an_enum=(value)
-      @an_enum = AnEnum.initialize_from_value(value)
+      @an_enum = AnEnum.initialize_from_value(value, from_message: @from_message)
     end
 
     # @return [ARecord]
     def a_record=(value)
-      @a_record = ARecord.initialize_from_value(value)
+      @a_record = ARecord.initialize_from_value(value, from_message: @from_message)
     end
 
     # @override

data/spec/schemas/my_namespace/my_nested_schema.rb

@@ -71,12 +71,12 @@ module Schemas; module MyNamespace
     ### Attribute Writers ###
     # @return [MyNestedRecord]
     def some_nested_record=(value)
-      @some_nested_record = MyNestedRecord.initialize_from_value(value)
+      @some_nested_record = MyNestedRecord.initialize_from_value(value, from_message: @from_message)
     end
 
     # @return [nil, MyNestedRecord]
     def some_optional_record=(value)
-      @some_optional_record = MyNestedRecord.initialize_from_value(value)
+      @some_optional_record = MyNestedRecord.initialize_from_value(value, from_message: @from_message)
     end
 
     # @override

data/spec/schemas/my_namespace/my_schema.rb

@@ -19,7 +19,7 @@ module Schemas; module MyNamespace
     ### Attribute Writers ###
     # @return [MySchemaKey]
     def payload_key=(value)
-      @payload_key = MySchemaKey.initialize_from_value(value)
+      @payload_key = MySchemaKey.initialize_from_value(value, from_message: @from_message)
     end
 
     # @override
@@ -44,7 +44,7 @@ module Schemas; module MyNamespace
 
     def self.tombstone(key)
       record = self.allocate
-      record.tombstone_key = MySchemaKey.initialize_from_value(key)
+      record.tombstone_key = MySchemaKey.initialize_from_value(key, from_message: @from_message)
       record.payload_key = key
       record
     end

data/spec/schemas/my_namespace/my_schema_with_circular_reference.rb

@@ -47,7 +47,7 @@ module Schemas; module MyNamespace
     # @return [Hash<String, Property>]
     def properties=(values)
       @properties = values&.transform_values do |value|
-        Property.initialize_from_value(value)
+        Property.initialize_from_value(value, from_message: @from_message)
       end
     end
 

data/spec/schemas/my_namespace/my_schema_with_complex_type.rb

@@ -105,43 +105,43 @@ module Schemas; module MyNamespace
     ### Attribute Writers ###
     # @return [ARecord]
     def some_record=(value)
-      @some_record = ARecord.initialize_from_value(value)
+      @some_record = ARecord.initialize_from_value(value, from_message: @from_message)
     end
 
     # @return [nil, ARecord]
     def some_optional_record=(value)
-      @some_optional_record = ARecord.initialize_from_value(value)
+      @some_optional_record = ARecord.initialize_from_value(value, from_message: @from_message)
     end
 
     # @return [Array<ARecord>]
     def some_record_array=(values)
       @some_record_array = values&.map do |value|
-        ARecord.initialize_from_value(value)
+        ARecord.initialize_from_value(value, from_message: @from_message)
       end
     end
 
     # @return [Hash<String, ARecord>]
     def some_record_map=(values)
       @some_record_map = values&.transform_values do |value|
-        ARecord.initialize_from_value(value)
+        ARecord.initialize_from_value(value, from_message: @from_message)
       end
     end
 
     # @return [Array<AnEnum>]
     def some_enum_array=(values)
       @some_enum_array = values&.map do |value|
-        AnEnum.initialize_from_value(value)
+        AnEnum.initialize_from_value(value, from_message: @from_message)
       end
     end
 
     # @return [nil, AnotherEnum]
     def some_optional_enum=(value)
-      @some_optional_enum = AnotherEnum.initialize_from_value(value)
+      @some_optional_enum = AnotherEnum.initialize_from_value(value, from_message: @from_message)
     end
 
     # @return [YetAnotherEnum]
     def some_enum_with_default=(value)
-      @some_enum_with_default = YetAnotherEnum.initialize_from_value(value)
+      @some_enum_with_default = YetAnotherEnum.initialize_from_value(value, from_message: @from_message)
     end
 
     # @override
@@ -150,8 +150,8 @@ module Schemas; module MyNamespace
                    test_string_array: ["test"],
                    test_int_array: [123],
                    test_optional_int: 123,
-                   some_integer_map: { "abc"=>123 },
-                   some_record: { "a_record_field"=>"Test String" },
+                   some_integer_map: {"abc"=>123},
+                   some_record: {"a_record_field"=>"Test String"},
                    some_optional_record: nil,
                    some_record_array: nil,
                    some_record_map: nil,