deimos-ruby 2.1.2 → 2.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 60b8a371a3669c29d7f59694bc49af057412313aeb08eebab2b88d4dbc5b6da4
-  data.tar.gz: 9c7231954c6db7ee55ffd4d3e18351bc7470acd443555f270c589da37e054631
+  metadata.gz: 1875e1db2f1a907bc82f3dacb4c1a9fea9be64afb90f317a3f41bbcec5296993
+  data.tar.gz: 03ab989cc0eb681cd60f39a30843298588bc67519e2851a2c3665bad2d62e441
 SHA512:
-  metadata.gz: 294627ab365d88be5f0617422a88bbd15fe81b46eae81dc663aff66174377145ccbbac218c205b5fbaf2d0dbdef31f0586e43607381423e0cd13cbb673f6913a
-  data.tar.gz: 216b9300910d20feb4786f4798542fc32a6a1f957dc19dc50209877853424a5d13e208f08f42c38fe420c86ec73e6b620e95a8f20d8700c1a7b3713f8f54472b
+  metadata.gz: 709f60ba95583f9e2f9c19055d90a3bd64a6a6184913e46c0152ca95b6592ed287a39f4feceb658ebecbdd9ccc2dcac42aec4a1d009ecbccd215b3687f432356
+  data.tar.gz: a4e884f2b11c789c6c162f3ed3c985842603ab2a529e4091a693d0cc444a45b3aa83d572c259e0b0a89b4ac4d319780132e58d78b2de9ea2d0757b587beb1923

data/CHANGELOG.md

@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+# 2.1.3 - 2025-07-22
+
+- Fix: Schema class generation was missing payload_key for a number of cases.
+- Fix: Possible crashes when using schema classes with Avro messages.
+
 ## 2.1.2 - 2025-07-21
 
 - Fix: Backwards-compatible fix wasn't working correctly.

data/karafka.rb

@@ -1 +1,71 @@
-# blank load file, used in regenerate_test_schema_classes.rb
+# Used in regenerate_test_schema_classes.rb
+
+class MyConsumer < Deimos::Consumer
+  def consume(payload, metadata); end
+end
+
+require_relative "./lib/generators/deimos/schema_class_generator"
+
+Deimos.configure do |deimos_config|
+  deimos_config.schema.nest_child_schemas = true
+  deimos_config.schema.path = "spec/schemas"
+  deimos_config.schema.backend = :avro_validation
+  deimos_config.schema.generated_class_path = './spec/schemas'
+  deimos_config.schema.use_full_namespace = true
+  deimos_config.schema.schema_namespace_map = {
+    'com' => 'Schemas',
+    'com.my-namespace.my-suborg' => %w(Schemas MyNamespace)
+  }
+end
+
+class KarafkaApp < Karafka::App
+  setup do
+    config.kafka = {
+      "bootstrap.servers": '127.0.0.1:9092'
+    }
+  end
+  routes.draw do
+    topic 'MyTopic' do
+      consumer MyConsumer
+      schema 'Generated'
+      namespace 'com.my-namespace'
+      key_config({ field: :a_string })
+    end
+
+    topic 'MyTopic2' do
+      consumer MyConsumer
+      schema 'MySchemaWithComplexTypes'
+      namespace 'com.my-namespace'
+      key_config({ field: :test_id })
+    end
+
+    topic 'MyTopic3' do
+      consumer MyConsumer
+      schema 'MySchemaWithCircularReference'
+      namespace 'com.my-namespace'
+      key_config({ none: true })
+    end
+
+    topic 'MyTopic4' do
+      consumer MyConsumer
+      schema 'MyNestedSchema'
+      namespace 'com.my-namespace'
+      key_config({ field: :test_id })
+    end
+
+    topic 'MyTopic5' do
+      consumer MyConsumer
+      schema 'MyLongNamespaceSchema'
+      namespace 'com.my-namespace.my-suborg'
+      key_config({ field: :test_id })
+    end
+
+    topic 'MyTopic6' do
+      producer_class MyConsumer
+      schema 'MySchema'
+      namespace 'com.my-namespace'
+      key_config({ schema: 'MySchema_key' })
+    end
+
+  end
+end

data/lib/deimos/schema_backends/avro_schema_coercer.rb

@@ -65,6 +65,8 @@ module Deimos
     # @param val [Object]
     # @return [Object]
     def coerce_record(type, val)
+      return nil if val.nil?
+
       record = val.map do |name, value|
         field = type.fields.find { |f| f.name == name }
         coerce_type(field.type, value)

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '2.1.2'
+  VERSION = '2.1.3'
 end

data/lib/generators/deimos/schema_class_generator.rb

@@ -162,26 +162,22 @@ module Deimos
       def generate
         _validate
         Rails.logger.info("Generating schemas from Deimos.config to #{Deimos.config.schema.generated_class_path}")
-        found_schemas = Set.new
-        Deimos.config.producer_objects.each do |config|
+        found_schemas = {}
+        Deimos.karafka_configs.each do |config|
           schema_name = config.schema
+          next if schema_name.nil?
           namespace = config.namespace || Deimos.config.producers.schema_namespace
           key_schema_name = config.key_config[:schema]
-          found_schemas.add("#{namespace}.#{schema_name}")
-          found_schemas.add("#{namespace}.#{key_schema_name}") if key_schema_name
-          generate_classes(schema_name, namespace, config.key_config)
-        end
 
-        Deimos.config.consumer_objects.each do |config|
-          schema_name = config.schema
-          namespace = config.namespace
-          key_schema_name = config.key_config[:schema]
-          found_schemas.add("#{namespace}.#{schema_name}")
-          found_schemas.add("#{namespace}.#{key_schema_name}") if key_schema_name
+          # don't regenerate if the schema was already found and had a payload key
+          next if found_schemas["#{namespace}.#{schema_name}"].present?
+
+          found_schemas["#{namespace}.#{schema_name}"] = key_schema_name
+          found_schemas["#{namespace}.#{key_schema_name}"] = nil
           generate_classes(schema_name, namespace, config.key_config)
         end
 
-        generate_from_schema_files(found_schemas)
+        generate_from_schema_files(found_schemas.keys)
 
       end
 
@@ -245,7 +241,7 @@ module Deimos
           "record.tombstone_key = key\n      record.#{key_config[:field]} = key"
         elsif key_schema
           field_base_type = _field_type(key_schema)
-          "record.tombstone_key = #{field_base_type}.initialize_from_value(key, from_message: self._from_message)\n      record.payload_key = key"
+          "record.tombstone_key = #{field_base_type}.initialize_from_value(key)\n      record.payload_key = key"
         else
           ''
         end

data/regenerate_test_schema_classes.rb

@@ -10,70 +10,4 @@ class DeimosApp < Rails::Application
 end
 DeimosApp.initialize!
 
-class MyConsumer < Deimos::Consumer
-  def consume(payload, metadata); end
-end
-
-require_relative "./lib/generators/deimos/schema_class_generator"
-
-Deimos.configure do |deimos_config|
-  deimos_config.schema.nest_child_schemas = true
-  deimos_config.schema.path = "spec/schemas"
-  deimos_config.schema.backend = :avro_validation
-  deimos_config.schema.generated_class_path = './spec/schemas'
-  deimos_config.schema.use_full_namespace = true
-  deimos_config.schema.schema_namespace_map = {
-    'com' => 'Schemas',
-    'com.my-namespace.my-suborg' => %w(Schemas MyNamespace)
-  }
-
-  consumer do
-    class_name 'MyConsumer'
-    topic 'MyTopic'
-    schema 'Generated'
-    namespace 'com.my-namespace'
-    key_config field: :a_string
-  end
-
-  consumer do
-    class_name 'MyConsumer'
-    topic 'MyTopic'
-    schema 'MySchemaWithComplexTypes'
-    namespace 'com.my-namespace'
-    key_config field: :test_id
-  end
-
-  consumer do
-    class_name 'MyConsumer'
-    topic 'MyTopic'
-    schema 'MySchemaWithCircularReference'
-    namespace 'com.my-namespace'
-    key_config none: true
-  end
-
-  consumer do
-    class_name 'MyConsumer'
-    topic 'MyTopic'
-    schema 'MyNestedSchema'
-    namespace 'com.my-namespace'
-    key_config field: :test_id
-  end
-
-  consumer do
-    class_name 'MyConsumer'
-    topic 'MyTopic'
-    schema 'MyLongNamespaceSchema'
-    namespace 'com.my-namespace.my-suborg'
-    key_config field: :test_id
-  end
-
-  producer do
-    class_name 'MyConsumer'
-    topic 'MyTopic'
-    schema 'MySchema'
-    namespace 'com.my-namespace'
-    key_config schema: 'MySchema_key'
-  end
-end
-
 Deimos::Generators::SchemaClassGenerator.new.generate

data/spec/schemas/my_namespace/my_schema.rb

@@ -45,7 +45,7 @@ module Schemas; module MyNamespace
 
     def self.tombstone(key)
       record = self.allocate
-      record.tombstone_key = MySchemaKey.initialize_from_value(key, from_message: self._from_message)
+      record.tombstone_key = MySchemaKey.initialize_from_value(key)
       record.payload_key = key
       record
     end

data/spec/schemas/my_namespace/my_updated_schema.rb

@@ -8,7 +8,8 @@ module Schemas; module Com; module MyNamespace
 
     attr_accessor :super_int
 
-    def initialize(test_id: nil,
+    def initialize(_from_message: false,
+                   test_id: nil,
                    some_int: nil)
       super
       self.super_int = some_int.nil? ? 10 : some_int * 9000

data/spec/snapshots/consumers-no-nest.snap

@@ -12,7 +12,8 @@ module Schemas
     attr_accessor :a_record_field
 
     # @override
-    def initialize(a_record_field: nil)
+    def initialize(_from_message: false, a_record_field: nil)
+      @_from_message = _from_message
       super
       self.a_record_field = a_record_field
     end
@@ -89,7 +90,8 @@ module Schemas
     attr_accessor :response_id
 
     # @override
-    def initialize(response_id: nil)
+    def initialize(_from_message: false, response_id: nil)
+      @_from_message = _from_message
       super
       self.response_id = response_id
     end
@@ -163,7 +165,7 @@ module Schemas
     end
 
     # @override
-    def initialize(a_string: nil,
+    def initialize(_from_message: false, a_string: nil,
                    a_int: nil,
                    a_long: nil,
                    a_float: nil,
@@ -175,6 +177,7 @@ module Schemas
                    timestamp: nil,
                    message_id: nil,
                    a_record: nil)
+      @_from_message = _from_message
       super
       self.a_string = a_string
       self.a_int = a_int
@@ -200,13 +203,6 @@ module Schemas
       'com.my-namespace'
     end
 
-    def self.tombstone(key)
-      record = self.allocate
-      record.tombstone_key = key
-      record.a_string = key
-      record
-    end
-
     # @override
     def as_json(_opts={})
       {
@@ -242,7 +238,8 @@ module Schemas
     attr_accessor :response_id
 
     # @override
-    def initialize(response_id: nil)
+    def initialize(_from_message: false, response_id: nil)
+      @_from_message = _from_message
       super
       self.response_id = response_id
     end
@@ -283,8 +280,9 @@ module Schemas
     attr_accessor :some_int
 
     # @override
-    def initialize(test_id: nil,
+    def initialize(_from_message: false, test_id: nil,
                    some_int: nil)
+      @_from_message = _from_message
       super
       self.test_id = test_id
       self.some_int = some_int
@@ -331,10 +329,11 @@ module Schemas
     attr_accessor :some_optional_int
 
     # @override
-    def initialize(some_int: nil,
+    def initialize(_from_message: false, some_int: nil,
                    some_float: nil,
                    some_string: nil,
                    some_optional_int: nil)
+      @_from_message = _from_message
       super
       self.some_int = some_int
       self.some_float = some_float
@@ -400,11 +399,12 @@ module Schemas
     end
 
     # @override
-    def initialize(test_id: nil,
+    def initialize(_from_message: false, test_id: nil,
                    test_float: nil,
                    test_array: nil,
                    some_nested_record: nil,
                    some_optional_record: nil)
+      @_from_message = _from_message
       super
       self.test_id = test_id
       self.test_float = test_float
@@ -423,6 +423,13 @@ module Schemas
       'com.my-namespace'
     end
 
+    def self.tombstone(key)
+      record = self.allocate
+      record.tombstone_key = key
+      record.test_id = key
+      record
+    end
+
     # @override
     def as_json(_opts={})
       {
@@ -453,8 +460,9 @@ module Schemas
     attr_accessor :some_int
 
     # @override
-    def initialize(test_id: nil,
+    def initialize(_from_message: false, test_id: nil,
                    some_int: nil)
+      @_from_message = _from_message
       super
       self.test_id = test_id
       self.some_int = some_int
@@ -497,8 +505,9 @@ module Schemas
     attr_accessor :part_two
 
     # @override
-    def initialize(part_one: nil,
+    def initialize(_from_message: false, part_one: nil,
                    part_two: nil)
+      @_from_message = _from_message
       super
       self.part_one = part_one
       self.part_two = part_two
@@ -539,7 +548,8 @@ module Schemas
     attr_accessor :id
 
     # @override
-    def initialize(id: nil)
+    def initialize(_from_message: false, id: nil)
+      @_from_message = _from_message
       super
       self.id = id
     end
@@ -578,7 +588,8 @@ module Schemas
     attr_accessor :test_id
 
     # @override
-    def initialize(test_id: nil)
+    def initialize(_from_message: false, test_id: nil)
+      @_from_message = _from_message
       super
       self.test_id = test_id
     end
@@ -619,8 +630,9 @@ module Schemas
     attr_accessor :some_bool
 
     # @override
-    def initialize(test_id: nil,
+    def initialize(_from_message: false, test_id: nil,
                    some_bool: nil)
+      @_from_message = _from_message
       super
       self.test_id = test_id
       self.some_bool = some_bool
@@ -669,7 +681,8 @@ module Schemas
     end
 
     # @override
-    def initialize(properties: {})
+    def initialize(_from_message: false, properties: {})
+      @_from_message = _from_message
       super
       self.properties = properties
     end
@@ -776,7 +789,7 @@ module Schemas
     end
 
     # @override
-    def initialize(test_id: nil,
+    def initialize(_from_message: false, test_id: nil,
                    test_float: nil,
                    test_string_array: ["test"],
                    test_int_array: [123],
@@ -789,6 +802,7 @@ module Schemas
                    some_enum_array: nil,
                    some_optional_enum: nil,
                    some_enum_with_default: "sym6")
+      @_from_message = _from_message
       super
       self.test_id = test_id
       self.test_float = test_float
@@ -859,11 +873,12 @@ module Schemas
     attr_accessor :timestamp
 
     # @override
-    def initialize(test_id: nil,
+    def initialize(_from_message: false, test_id: nil,
                    updated_at: nil,
                    some_int: nil,
                    some_datetime_int: nil,
                    timestamp: nil)
+      @_from_message = _from_message
       super
       self.test_id = test_id
       self.updated_at = updated_at
@@ -916,10 +931,11 @@ module Schemas
     attr_accessor :timestamp
 
     # @override
-    def initialize(test_id: nil,
+    def initialize(_from_message: false, test_id: nil,
                    some_int: nil,
                    message_id: nil,
                    timestamp: nil)
+      @_from_message = _from_message
       super
       self.test_id = test_id
       self.some_int = some_int
@@ -968,9 +984,10 @@ module Schemas
     attr_accessor :title
 
     # @override
-    def initialize(test_id: nil,
+    def initialize(_from_message: false, test_id: nil,
                    some_int: nil,
                    title: nil)
+      @_from_message = _from_message
       super
       self.test_id = test_id
       self.some_int = some_int
@@ -1040,9 +1057,10 @@ module Schemas
     end
 
     # @override
-    def initialize(test_id: "",
+    def initialize(_from_message: false, test_id: "",
                    test_long: nil,
                    test_union_type: nil)
+      @_from_message = _from_message
       super
       self.test_id = test_id
       self.test_long = test_long
@@ -1093,11 +1111,12 @@ module Schemas
     attr_accessor :timestamp
 
     # @override
-    def initialize(id: nil,
+    def initialize(_from_message: false, id: nil,
                    test_id: nil,
                    some_int: nil,
                    message_id: nil,
                    timestamp: nil)
+      @_from_message = _from_message
       super
       self.id = id
       self.test_id = test_id
@@ -1144,7 +1163,8 @@ module Schemas
     attr_accessor :property
 
     # @override
-    def initialize(property: nil)
+    def initialize(_from_message: false, property: nil)
+      @_from_message = _from_message
       super
       self.property = property
     end
@@ -1185,8 +1205,9 @@ module Schemas
     attr_accessor :record1_id
 
     # @override
-    def initialize(record1_map: {},
+    def initialize(_from_message: false, record1_map: {},
                    record1_id: 0)
+      @_from_message = _from_message
       super
       self.record1_map = record1_map
       self.record1_id = record1_id
@@ -1227,7 +1248,8 @@ module Schemas
     attr_accessor :record2_id
 
     # @override
-    def initialize(record2_id: "")
+    def initialize(_from_message: false, record2_id: "")
+      @_from_message = _from_message
       super
       self.record2_id = record2_id
     end
@@ -1266,7 +1288,8 @@ module Schemas
     attr_accessor :record3_id
 
     # @override
-    def initialize(record3_id: 0.0)
+    def initialize(_from_message: false, record3_id: 0.0)
+      @_from_message = _from_message
       super
       self.record3_id = record3_id
     end
@@ -1305,7 +1328,8 @@ module Schemas
     attr_accessor :record4_id
 
     # @override
-    def initialize(record4_id: 0)
+    def initialize(_from_message: false, record4_id: 0)
+      @_from_message = _from_message
       super
       self.record4_id = record4_id
     end
@@ -1344,7 +1368,8 @@ module Schemas
     attr_accessor :update_request_id
 
     # @override
-    def initialize(update_request_id: nil)
+    def initialize(_from_message: false, update_request_id: nil)
+      @_from_message = _from_message
       super
       self.update_request_id = update_request_id
     end
@@ -1383,7 +1408,8 @@ module Schemas
     attr_accessor :update_response_id
 
     # @override
-    def initialize(update_response_id: nil)
+    def initialize(_from_message: false, update_response_id: nil)
+      @_from_message = _from_message
       super
       self.update_response_id = update_response_id
     end
@@ -1438,7 +1464,7 @@ module Schemas
     attr_accessor :created_at
 
     # @override
-    def initialize(id: nil,
+    def initialize(_from_message: false, id: nil,
                    wibble_id: nil,
                    name: nil,
                    floop: nil,
@@ -1447,6 +1473,7 @@ module Schemas
                    birthday_optional: nil,
                    updated_at: nil,
                    created_at: nil)
+      @_from_message = _from_message
       super
       self.id = id
       self.wibble_id = wibble_id
@@ -1509,11 +1536,12 @@ module Schemas
     attr_accessor :created_at
 
     # @override
-    def initialize(id: nil,
+    def initialize(_from_message: false, id: nil,
                    widget_id: nil,
                    name: nil,
                    updated_at: nil,
                    created_at: nil)
+      @_from_message = _from_message
       super
       self.id = id
       self.widget_id = widget_id
@@ -1568,11 +1596,12 @@ module Schemas
     attr_accessor :created_at
 
     # @override
-    def initialize(id: nil,
+    def initialize(_from_message: false, id: nil,
                    widget_id: nil,
                    model_id: nil,
                    updated_at: nil,
                    created_at: nil)
+      @_from_message = _from_message
       super
       self.id = id
       self.widget_id = widget_id