deimos-ruby 2.1.2.pre.beta1 → 2.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c46b9a4b66276607dae95bc1e22f31d964be1f03241911a9d8d6645b70698429
-  data.tar.gz: 2457d19845c4577ceaf9cb33b2d317846bd1510b5308d53e9f061482d7c1e4cb
+  metadata.gz: 1875e1db2f1a907bc82f3dacb4c1a9fea9be64afb90f317a3f41bbcec5296993
+  data.tar.gz: 03ab989cc0eb681cd60f39a30843298588bc67519e2851a2c3665bad2d62e441
 SHA512:
-  metadata.gz: 4156e4d6911f4010d0cea070ba0345fae92df0c6859737a90e2cf5ef5dca2f170da7ccb093fb45ea4269b23644ec3a7b084becc17b5cb86e696befbcaeef6ca8
-  data.tar.gz: 9e1aff4fad7860e7f57dfd63a2031cfaac78ba5af818014e1861c248a3e32ddd7f88546f5daf563c1df5b9c2b78b61334a58e3831a9092eca1aeaba7933f9258
+  metadata.gz: 709f60ba95583f9e2f9c19055d90a3bd64a6a6184913e46c0152ca95b6592ed287a39f4feceb658ebecbdd9ccc2dcac42aec4a1d009ecbccd215b3687f432356
+  data.tar.gz: a4e884f2b11c789c6c162f3ed3c985842603ab2a529e4091a693d0cc444a45b3aa83d572c259e0b0a89b4ac4d319780132e58d78b2de9ea2d0757b587beb1923

data/CHANGELOG.md

@@ -7,7 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
-## 2.1.1 - 2025-07-21
+# 2.1.3 - 2025-07-22
+
+- Fix: Schema class generation was missing payload_key for a number of cases.
+- Fix: Possible crashes when using schema classes with Avro messages.
+
+## 2.1.2 - 2025-07-21
 
 - Fix: Backwards-compatible fix wasn't working correctly.
 

data/README.md

@@ -1083,7 +1083,7 @@ decoded = Deimos.decode(schema: 'MySchema', namespace: 'com.my-namespace', paylo
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/flipp-oss/deimos .
 
-If making changes to the generator, you should regenerate the test schema classes by running `./regenerate_test_schema_classes.rb` .
+If making changes to the generator, you should regenerate the test schema classes by running `bundle exec ./regenerate_test_schema_classes.rb` .
 
 You can/should re-generate RBS types when methods or classes change by running the following:
 

data/karafka.rb

@@ -1 +1,71 @@
-# blank load file, used in regenerate_test_schema_classes.rb
+# Used in regenerate_test_schema_classes.rb
+
+class MyConsumer < Deimos::Consumer
+  def consume(payload, metadata); end
+end
+
+require_relative "./lib/generators/deimos/schema_class_generator"
+
+Deimos.configure do |deimos_config|
+  deimos_config.schema.nest_child_schemas = true
+  deimos_config.schema.path = "spec/schemas"
+  deimos_config.schema.backend = :avro_validation
+  deimos_config.schema.generated_class_path = './spec/schemas'
+  deimos_config.schema.use_full_namespace = true
+  deimos_config.schema.schema_namespace_map = {
+    'com' => 'Schemas',
+    'com.my-namespace.my-suborg' => %w(Schemas MyNamespace)
+  }
+end
+
+class KarafkaApp < Karafka::App
+  setup do
+    config.kafka = {
+      "bootstrap.servers": '127.0.0.1:9092'
+    }
+  end
+  routes.draw do
+    topic 'MyTopic' do
+      consumer MyConsumer
+      schema 'Generated'
+      namespace 'com.my-namespace'
+      key_config({ field: :a_string })
+    end
+
+    topic 'MyTopic2' do
+      consumer MyConsumer
+      schema 'MySchemaWithComplexTypes'
+      namespace 'com.my-namespace'
+      key_config({ field: :test_id })
+    end
+
+    topic 'MyTopic3' do
+      consumer MyConsumer
+      schema 'MySchemaWithCircularReference'
+      namespace 'com.my-namespace'
+      key_config({ none: true })
+    end
+
+    topic 'MyTopic4' do
+      consumer MyConsumer
+      schema 'MyNestedSchema'
+      namespace 'com.my-namespace'
+      key_config({ field: :test_id })
+    end
+
+    topic 'MyTopic5' do
+      consumer MyConsumer
+      schema 'MyLongNamespaceSchema'
+      namespace 'com.my-namespace.my-suborg'
+      key_config({ field: :test_id })
+    end
+
+    topic 'MyTopic6' do
+      producer_class MyConsumer
+      schema 'MySchema'
+      namespace 'com.my-namespace'
+      key_config({ schema: 'MySchema_key' })
+    end
+
+  end
+end

data/lib/deimos/schema_backends/avro_schema_coercer.rb

@@ -65,6 +65,8 @@ module Deimos
     # @param val [Object]
     # @return [Object]
     def coerce_record(type, val)
+      return nil if val.nil?
+
       record = val.map do |name, value|
         field = type.fields.find { |f| f.name == name }
         coerce_type(field.type, value)

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '2.1.2-beta1'
+  VERSION = '2.1.3'
 end

data/lib/generators/deimos/schema_class_generator.rb

@@ -162,26 +162,22 @@ module Deimos
       def generate
         _validate
         Rails.logger.info("Generating schemas from Deimos.config to #{Deimos.config.schema.generated_class_path}")
-        found_schemas = Set.new
-        Deimos.config.producer_objects.each do |config|
+        found_schemas = {}
+        Deimos.karafka_configs.each do |config|
           schema_name = config.schema
+          next if schema_name.nil?
           namespace = config.namespace || Deimos.config.producers.schema_namespace
           key_schema_name = config.key_config[:schema]
-          found_schemas.add("#{namespace}.#{schema_name}")
-          found_schemas.add("#{namespace}.#{key_schema_name}") if key_schema_name
-          generate_classes(schema_name, namespace, config.key_config)
-        end
 
-        Deimos.config.consumer_objects.each do |config|
-          schema_name = config.schema
-          namespace = config.namespace
-          key_schema_name = config.key_config[:schema]
-          found_schemas.add("#{namespace}.#{schema_name}")
-          found_schemas.add("#{namespace}.#{key_schema_name}") if key_schema_name
+          # don't regenerate if the schema was already found and had a payload key
+          next if found_schemas["#{namespace}.#{schema_name}"].present?
+
+          found_schemas["#{namespace}.#{schema_name}"] = key_schema_name
+          found_schemas["#{namespace}.#{key_schema_name}"] = nil
           generate_classes(schema_name, namespace, config.key_config)
         end
 
-        generate_from_schema_files(found_schemas)
+        generate_from_schema_files(found_schemas.keys)
 
       end
 
@@ -245,7 +241,7 @@ module Deimos
           "record.tombstone_key = key\n      record.#{key_config[:field]} = key"
         elsif key_schema
           field_base_type = _field_type(key_schema)
-          "record.tombstone_key = #{field_base_type}.initialize_from_value(key, from_message: self._from_message)\n      record.payload_key = key"
+          "record.tombstone_key = #{field_base_type}.initialize_from_value(key)\n      record.payload_key = key"
         else
           ''
         end

data/regenerate_test_schema_classes.rb

@@ -10,70 +10,4 @@ class DeimosApp < Rails::Application
 end
 DeimosApp.initialize!
 
-class MyConsumer < Deimos::Consumer
-  def consume(payload, metadata); end
-end
-
-require_relative "./lib/generators/deimos/schema_class_generator"
-
-Deimos.configure do |deimos_config|
-  deimos_config.schema.nest_child_schemas = true
-  deimos_config.schema.path = "spec/schemas"
-  deimos_config.schema.backend = :avro_validation
-  deimos_config.schema.generated_class_path = './spec/schemas'
-  deimos_config.schema.use_full_namespace = true
-  deimos_config.schema.schema_namespace_map = {
-    'com' => 'Schemas',
-    'com.my-namespace.my-suborg' => %w(Schemas MyNamespace)
-  }
-
-  consumer do
-    class_name 'MyConsumer'
-    topic 'MyTopic'
-    schema 'Generated'
-    namespace 'com.my-namespace'
-    key_config field: :a_string
-  end
-
-  consumer do
-    class_name 'MyConsumer'
-    topic 'MyTopic'
-    schema 'MySchemaWithComplexTypes'
-    namespace 'com.my-namespace'
-    key_config field: :test_id
-  end
-
-  consumer do
-    class_name 'MyConsumer'
-    topic 'MyTopic'
-    schema 'MySchemaWithCircularReference'
-    namespace 'com.my-namespace'
-    key_config none: true
-  end
-
-  consumer do
-    class_name 'MyConsumer'
-    topic 'MyTopic'
-    schema 'MyNestedSchema'
-    namespace 'com.my-namespace'
-    key_config field: :test_id
-  end
-
-  consumer do
-    class_name 'MyConsumer'
-    topic 'MyTopic'
-    schema 'MyLongNamespaceSchema'
-    namespace 'com.my-namespace.my-suborg'
-    key_config field: :test_id
-  end
-
-  producer do
-    class_name 'MyConsumer'
-    topic 'MyTopic'
-    schema 'MySchema'
-    namespace 'com.my-namespace'
-    key_config schema: 'MySchema_key'
-  end
-end
-
 Deimos::Generators::SchemaClassGenerator.new.generate

data/spec/consumer_spec.rb

@@ -180,7 +180,19 @@ module ConsumerTest
             }
           end
           # update the Avro to include a new field
-          generator.append_to_file(schema_file, additional_field_json, after: '"fields": [')
+          generator.insert_into_file(schema_file, additional_field_json, after: '"fields": [', force: true)
+
+          Karafka::App.routes.redraw do
+            topic 'my_consume_topic' do
+              schema 'MyNestedSchema'
+              namespace 'com.my-namespace'
+              key_config field: 'test_id'
+              consumer ConsumerTest::MyConsumer
+              reraise_errors true
+              use_schema_classes true
+              reraise_errors true
+            end
+          end
         end
 
         after(:each) do
@@ -188,7 +200,7 @@ module ConsumerTest
         end
 
         let(:generator) { Rails::Generators::Base.new }
-        let(:schema_file) { File.join(__dir__, 'schemas/com/my-namespace/MySchema.avsc') }
+        let(:schema_file) { File.join(__dir__, 'schemas/com/my-namespace/MyNestedSchema.avsc') }
         let(:additional_field_json) do
           '{"name": "additional_field", "type": "string", "default": ""},'
         end
@@ -196,11 +208,20 @@ module ConsumerTest
         it 'should consume correctly and ignore the additional field' do
           test_consume_message('my_consume_topic',
                                { 'test_id' => 'foo',
+                                 'test_float' => 4.0,
+                                 'test_array' => ["1", "2"],
                                  'additional_field' => 'bar',
-                                 'some_int' => 1 }) do |payload, _metadata|
+                                 'some_nested_record' => {
+                                    'some_int' => 1,
+                                    'some_float' => 10.0,
+                                    'some_string' => 'hi mom',
+                                    'additional_field' => 'baz'
+                                 } }) do |payload, _metadata|
             expect(payload['test_id']).to eq('foo')
-            expect(payload['some_int']).to eq(1)
+            expect(payload['test_float']).to eq(4.0)
+            expect(payload['some_nested_record']['some_int']).to eq(1)
             expect(payload.to_h).not_to have_key('additional_field')
+            expect(payload.to_h['some_nested_record']).not_to have_key('additional_field')
           end
 
         end

data/spec/schemas/my_namespace/generated.rb

@@ -15,7 +15,8 @@ module Schemas; module MyNamespace
       attr_accessor :a_record_field
   
       # @override
-      def initialize(a_record_field: nil)
+      def initialize(_from_message: false, a_record_field: nil)
+        @_from_message = _from_message
         super
         self.a_record_field = a_record_field
       end
@@ -81,16 +82,16 @@ module Schemas; module MyNamespace
     ### Attribute Writers ###
     # @return [AnEnum]
     def an_enum=(value)
-      @an_enum = AnEnum.initialize_from_value(value, from_message: @from_message)
+      @an_enum = AnEnum.initialize_from_value(value, from_message: self._from_message)
     end
 
     # @return [ARecord]
     def a_record=(value)
-      @a_record = ARecord.initialize_from_value(value, from_message: @from_message)
+      @a_record = ARecord.initialize_from_value(value, from_message: self._from_message)
     end
 
     # @override
-    def initialize(a_string: nil,
+    def initialize(_from_message: false, a_string: nil,
                    a_int: nil,
                    a_long: nil,
                    a_float: nil,
@@ -102,6 +103,7 @@ module Schemas; module MyNamespace
                    timestamp: nil,
                    message_id: nil,
                    a_record: nil)
+      @_from_message = _from_message
       super
       self.a_string = a_string
       self.a_int = a_int

data/spec/schemas/my_namespace/my_long_namespace_schema.rb

@@ -13,8 +13,9 @@ module Schemas; module MyNamespace
     attr_accessor :some_int
 
     # @override
-    def initialize(test_id: nil,
+    def initialize(_from_message: false, test_id: nil,
                    some_int: nil)
+      @_from_message = _from_message
       super
       self.test_id = test_id
       self.some_int = some_int

data/spec/schemas/my_namespace/my_nested_schema.rb

@@ -21,10 +21,11 @@ module Schemas; module MyNamespace
       attr_accessor :some_optional_int
   
       # @override
-      def initialize(some_int: nil,
+      def initialize(_from_message: false, some_int: nil,
                      some_float: nil,
                      some_string: nil,
                      some_optional_int: nil)
+        @_from_message = _from_message
         super
         self.some_int = some_int
         self.some_float = some_float
@@ -71,20 +72,21 @@ module Schemas; module MyNamespace
     ### Attribute Writers ###
     # @return [MyNestedRecord]
     def some_nested_record=(value)
-      @some_nested_record = MyNestedRecord.initialize_from_value(value, from_message: @from_message)
+      @some_nested_record = MyNestedRecord.initialize_from_value(value, from_message: self._from_message)
     end
 
     # @return [nil, MyNestedRecord]
     def some_optional_record=(value)
-      @some_optional_record = MyNestedRecord.initialize_from_value(value, from_message: @from_message)
+      @some_optional_record = MyNestedRecord.initialize_from_value(value, from_message: self._from_message)
     end
 
     # @override
-    def initialize(test_id: nil,
+    def initialize(_from_message: false, test_id: nil,
                    test_float: nil,
                    test_array: nil,
                    some_nested_record: nil,
                    some_optional_record: nil)
+      @_from_message = _from_message
       super
       self.test_id = test_id
       self.test_float = test_float

data/spec/schemas/my_namespace/my_schema.rb

@@ -19,13 +19,14 @@ module Schemas; module MyNamespace
     ### Attribute Writers ###
     # @return [MySchemaKey]
     def payload_key=(value)
-      @payload_key = MySchemaKey.initialize_from_value(value, from_message: @from_message)
+      @payload_key = MySchemaKey.initialize_from_value(value, from_message: self._from_message)
     end
 
     # @override
-    def initialize(test_id: nil,
+    def initialize(_from_message: false, test_id: nil,
                    some_int: nil,
                    payload_key: nil)
+      @_from_message = _from_message
       super
       self.test_id = test_id
       self.some_int = some_int
@@ -44,7 +45,7 @@ module Schemas; module MyNamespace
 
     def self.tombstone(key)
       record = self.allocate
-      record.tombstone_key = MySchemaKey.initialize_from_value(key, from_message: @from_message)
+      record.tombstone_key = MySchemaKey.initialize_from_value(key)
       record.payload_key = key
       record
     end

data/spec/schemas/my_namespace/my_schema_compound_key.rb

@@ -13,8 +13,9 @@ module Schemas; module MyNamespace
     attr_accessor :part_two
 
     # @override
-    def initialize(part_one: nil,
+    def initialize(_from_message: false, part_one: nil,
                    part_two: nil)
+      @_from_message = _from_message
       super
       self.part_one = part_one
       self.part_two = part_two

data/spec/schemas/my_namespace/my_schema_id_key.rb

@@ -11,7 +11,8 @@ module Schemas; module MyNamespace
     attr_accessor :id
 
     # @override
-    def initialize(id: nil)
+    def initialize(_from_message: false, id: nil)
+      @_from_message = _from_message
       super
       self.id = id
     end

data/spec/schemas/my_namespace/my_schema_key.rb

@@ -11,7 +11,8 @@ module Schemas; module MyNamespace
     attr_accessor :test_id
 
     # @override
-    def initialize(test_id: nil)
+    def initialize(_from_message: false, test_id: nil)
+      @_from_message = _from_message
       super
       self.test_id = test_id
     end

data/spec/schemas/my_namespace/my_schema_with_boolean.rb

@@ -13,8 +13,9 @@ module Schemas; module MyNamespace
     attr_accessor :some_bool
 
     # @override
-    def initialize(test_id: nil,
+    def initialize(_from_message: false, test_id: nil,
                    some_bool: nil)
+      @_from_message = _from_message
       super
       self.test_id = test_id
       self.some_bool = some_bool

data/spec/schemas/my_namespace/my_schema_with_circular_reference.rb

@@ -15,7 +15,8 @@ module Schemas; module MyNamespace
       attr_accessor :property
   
       # @override
-      def initialize(property: nil)
+      def initialize(_from_message: false, property: nil)
+        @_from_message = _from_message
         super
         self.property = property
       end
@@ -47,12 +48,13 @@ module Schemas; module MyNamespace
     # @return [Hash<String, Property>]
     def properties=(values)
       @properties = values&.transform_values do |value|
-        Property.initialize_from_value(value, from_message: @from_message)
+        Property.initialize_from_value(value, from_message: self._from_message)
       end
     end
 
     # @override
-    def initialize(properties: {})
+    def initialize(_from_message: false, properties: {})
+      @_from_message = _from_message
       super
       self.properties = properties
     end

data/spec/schemas/my_namespace/my_schema_with_complex_type.rb

@@ -15,7 +15,8 @@ module Schemas; module MyNamespace
       attr_accessor :a_record_field
   
       # @override
-      def initialize(a_record_field: nil)
+      def initialize(_from_message: false, a_record_field: nil)
+        @_from_message = _from_message
         super
         self.a_record_field = a_record_field
       end
@@ -105,47 +106,47 @@ module Schemas; module MyNamespace
     ### Attribute Writers ###
     # @return [ARecord]
     def some_record=(value)
-      @some_record = ARecord.initialize_from_value(value, from_message: @from_message)
+      @some_record = ARecord.initialize_from_value(value, from_message: self._from_message)
     end
 
     # @return [nil, ARecord]
     def some_optional_record=(value)
-      @some_optional_record = ARecord.initialize_from_value(value, from_message: @from_message)
+      @some_optional_record = ARecord.initialize_from_value(value, from_message: self._from_message)
     end
 
     # @return [Array<ARecord>]
     def some_record_array=(values)
       @some_record_array = values&.map do |value|
-        ARecord.initialize_from_value(value, from_message: @from_message)
+        ARecord.initialize_from_value(value, from_message: self._from_message)
       end
     end
 
     # @return [Hash<String, ARecord>]
     def some_record_map=(values)
       @some_record_map = values&.transform_values do |value|
-        ARecord.initialize_from_value(value, from_message: @from_message)
+        ARecord.initialize_from_value(value, from_message: self._from_message)
       end
     end
 
     # @return [Array<AnEnum>]
     def some_enum_array=(values)
       @some_enum_array = values&.map do |value|
-        AnEnum.initialize_from_value(value, from_message: @from_message)
+        AnEnum.initialize_from_value(value, from_message: self._from_message)
       end
     end
 
     # @return [nil, AnotherEnum]
     def some_optional_enum=(value)
-      @some_optional_enum = AnotherEnum.initialize_from_value(value, from_message: @from_message)
+      @some_optional_enum = AnotherEnum.initialize_from_value(value, from_message: self._from_message)
     end
 
     # @return [YetAnotherEnum]
     def some_enum_with_default=(value)
-      @some_enum_with_default = YetAnotherEnum.initialize_from_value(value, from_message: @from_message)
+      @some_enum_with_default = YetAnotherEnum.initialize_from_value(value, from_message: self._from_message)
     end
 
     # @override
-    def initialize(test_id: nil,
+    def initialize(_from_message: false, test_id: nil,
                    test_float: nil,
                    test_string_array: ["test"],
                    test_int_array: [123],
@@ -158,6 +159,7 @@ module Schemas; module MyNamespace
                    some_enum_array: nil,
                    some_optional_enum: nil,
                    some_enum_with_default: "sym6")
+      @_from_message = _from_message
       super
       self.test_id = test_id
       self.test_float = test_float

data/spec/schemas/my_namespace/my_schema_with_date_time.rb

@@ -19,11 +19,12 @@ module Schemas; module MyNamespace
     attr_accessor :timestamp
 
     # @override
-    def initialize(test_id: nil,
+    def initialize(_from_message: false, test_id: nil,
                    updated_at: nil,
                    some_int: nil,
                    some_datetime_int: nil,
                    timestamp: nil)
+      @_from_message = _from_message
       super
       self.test_id = test_id
       self.updated_at = updated_at

data/spec/schemas/my_namespace/my_schema_with_id.rb

@@ -17,10 +17,11 @@ module Schemas; module MyNamespace
     attr_accessor :timestamp
 
     # @override
-    def initialize(test_id: nil,
+    def initialize(_from_message: false, test_id: nil,
                    some_int: nil,
                    message_id: nil,
                    timestamp: nil)
+      @_from_message = _from_message
       super
       self.test_id = test_id
       self.some_int = some_int

data/spec/schemas/my_namespace/my_schema_with_title.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+# This file is autogenerated by Deimos, Do NOT modify
+module Schemas; module MyNamespace
+  ### Primary Schema Class ###
+  # Autogenerated Schema for Record at com.my-namespace.MySchemaWithTitle
+  class MySchemaWithTitle < Deimos::SchemaClass::Record
+
+    ### Attribute Accessors ###
+    # @return [String]
+    attr_accessor :test_id
+    # @return [Integer]
+    attr_accessor :some_int
+    # @return [String]
+    attr_accessor :title
+
+    # @override
+    def initialize(_from_message: false, test_id: nil,
+                   some_int: nil,
+                   title: nil)
+      @_from_message = _from_message
+      super
+      self.test_id = test_id
+      self.some_int = some_int
+      self.title = title
+    end
+
+    # @override
+    def schema
+      'MySchemaWithTitle'
+    end
+
+    # @override
+    def namespace
+      'com.my-namespace'
+    end
+
+    # @override
+    def as_json(_opts={})
+      {
+        'test_id' => @test_id,
+        'some_int' => @some_int,
+        'title' => @title
+      }
+    end
+  end
+end; end