deimos-ruby 2.0.4 → 2.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 277d2715eefd6d0ddc262ee4e514fe3df2542aacf13c60854f66f326a46664ae
-  data.tar.gz: 6a5b890b14258eeba1d08b9e84f146030a7c9f37784008a8840311dbafca49a1
+  metadata.gz: 53cd09a846fed14b74b20cad18fc63374887c0a9deae4444f2d6548ea06b5b72
+  data.tar.gz: 95f7b6402d43d2c2332f5abbc8c0374655b1ac713b484d3f4b516321b4e2207d
 SHA512:
-  metadata.gz: e8c0f52c88c5106c8a22acb6a15f7e850b307aa8e34aa5ac5cc140a1bd7d10cebc891ad5ac26d386ecc8f76e080c53791e9a5d841dc1db56a170df49fc596daf
-  data.tar.gz: b2f50f1d14dd55330a78159645bc335e82217ce59284066cd02f37e98cc66d932bf8ff78ae54d01f942602cf1759ab513239a90898042a400d4047c55dbc11e2
+  metadata.gz: 928b533075a3a98f5731f15e2e9436cd1ad0754006bcf18816bdc19933e0990478aa7cd06152e450f0041524cfcf04da839c1116a4895bf3c9e692e70aefa4f0
+  data.tar.gz: b363497c5b9f1e9ea6dd71bf2d4e4bd59e7b96fe229cc58ee9caccb3ca86a7cd1ef4bf0636e7a354d496c790aa867fcad3405bdd4b353e72fce1d1e22ba32876

data/CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+## 2.0.5 - 2025-03-19
+
+- Fix: Added support to handle producing union type of multiple records & data types.
+
 ## 2.0.4 - 2025-03-17
 
 - Feature: Added `producers.truncate_columns` config.

data/lib/deimos/schema_backends/avro_schema_coercer.rb

@@ -18,10 +18,48 @@ module Deimos
       union_types = type.schemas.map { |s| s.type.to_sym }
       return nil if val.nil? && union_types.include?(:null)
 
-      schema_type = type.schemas.find { |s| s.type.to_sym != :null }
+      schema_type = find_schema_type(type, val)
       coerce_type(schema_type, val)
     end
 
+    # Find the right schema for val from a UnionSchema.
+    # @param type [Avro::Schema::UnionSchema]
+    # @param val [Object]
+    # @return [Avro::Schema::PrimitiveSchema]
+    def find_schema_type(type, val)
+      int_classes = [Time, ActiveSupport::TimeWithZone]
+
+      schema_type = type.schemas.find do |schema|
+        field_type = schema.type.to_sym
+
+        case field_type
+        when :int, :long
+          val.is_a?(Integer) ||
+            _is_integer_string?(val) ||
+            int_classes.any? { |klass| val.is_a?(klass) }
+        when :float, :double
+          val.is_a?(Numeric) || _is_float_string?(val)
+        when :array
+          val.is_a?(Array)
+        when :record
+          if val.is_a?(Hash)
+            schema_fields_set = Set.new(schema.fields.map(&:name))
+            Set.new(val.keys).subset?(schema_fields_set)
+          else
+            # If the value is not a hash, we can't coerce it to a record.
+            # Keep looking for another schema
+            false
+          end
+        else
+          schema.type.to_sym != :null
+        end
+      end
+
+      raise "No Schema type found for VALUE: #{val}\n TYPE: #{type}" if schema_type.nil?
+
+      schema_type
+    end
+
     # Coerce sub-records in a payload to match the schema.
     # @param type [Avro::Schema::RecordSchema]
     # @param val [Object]

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '2.0.4'
+  VERSION = '2.0.5'
 end

data/lib/generators/deimos/schema_class/templates/schema_record.rb.tt

@@ -44,6 +44,24 @@
     end
 
   <%- end -%>
+<% end -%>
+<%- if @field_assignments.select{ |h| h[:is_complex_union] }.any? -%>
+    <%- @field_assignments.select{ |h| h[:is_complex_union] }.each do |method_definition| -%>
+    # Helper method to determine which schema type to use for <%= method_definition[:field].name %>
+    # @param value [Hash, nil]
+    # @return [Object, nil]
+    def initialize_<%= method_definition[:field].name %>_type(value)
+      return nil if value.nil?
+
+      klass = [<%= method_definition[:field].type.schemas.reject { |s| s.type_sym == :null }.select { |s| s.type_sym == :record }.map { |s| Deimos::SchemaBackends::AvroBase.schema_classname(s) }.join(', ') %>].find do |candidate|
+        fields = candidate.new.as_json.keys
+        (value.keys - fields).empty?
+      end
+
+      klass.initialize_from_value(value)
+    end
+
+    <%- end -%>
 <% end -%>
     # @override
     <%= @initialization_definition %>

data/lib/generators/deimos/schema_class_generator.rb

@@ -298,7 +298,9 @@ module Deimos
 
           field_initialization = method_argument
 
-          if is_schema_class
+          if _is_complex_union?(field)
+            field_initialization = "initialize_#{field.name}_type(value)"
+          elsif is_schema_class
             field_initialization = "#{field_base_type}.initialize_from_value(value)"
           end
 
@@ -308,13 +310,26 @@ module Deimos
             is_schema_class: is_schema_class,
             method_argument: method_argument,
             deimos_type: deimos_field_type(field),
-            field_initialization: field_initialization
+            field_initialization: field_initialization,
+            is_complex_union: _is_complex_union?(field)
           }
         end
 
         result
       end
 
+      # Helper method to detect if a field is a complex union type with multiple record schemas
+      # @param field [Deimos::SchemaField]
+      # @return [Boolean]
+      def _is_complex_union?(field)
+        return false unless field.type.type_sym == :union
+
+        non_null_schemas = field.type.schemas.reject { |s| s.type_sym == :null }
+
+        record_schemas = non_null_schemas.select { |s| s.type_sym == :record }
+        record_schemas.length > 1
+      end
+
       # Converts Avro::Schema::NamedSchema's to String form for generated YARD docs.
       # Recursively handles the typing for Arrays, Maps and Unions.
       # @param avro_schema [Avro::Schema::NamedSchema]

data/spec/active_record_producer_spec.rb

@@ -3,6 +3,7 @@
 describe Deimos::ActiveRecordProducer do
 
   include_context 'with widgets'
+  include_context 'with widget_with_union_types'
 
   prepend_before(:each) do
     producer_class = Class.new(Deimos::ActiveRecordProducer)
@@ -40,6 +41,12 @@ describe Deimos::ActiveRecordProducer do
     end
 
     stub_const('MyProducerWithPostProcess', producer_class)
+
+    producer_class = Class.new(Deimos::ActiveRecordProducer) do
+      record_class WidgetWithUnionType
+    end
+    stub_const('MyProducerWithUnionType', producer_class)
+
     Karafka::App.routes.redraw do
       topic 'my-topic' do
         schema 'MySchema'
@@ -71,6 +78,13 @@ describe Deimos::ActiveRecordProducer do
         key_config none: true
         producer_class MyProducerWithPostProcess
       end
+      topic 'my-topic-with-union-type' do
+        schema 'MySchemaWithUnionType'
+        namespace 'com.my-namespace'
+        key_config none: true
+        producer_class MyProducerWithUnionType
+      end
+
     end
 
   end
@@ -90,6 +104,98 @@ describe Deimos::ActiveRecordProducer do
           expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
         end
 
+        it 'should coerce values for a UnionSchema' do
+          MyProducerWithUnionType.send_event(WidgetWithUnionType.new(
+            test_id: "abc",
+            test_long: 399999,
+            test_union_type: %w(hello world)
+          ))
+
+          expect('my-topic-with-union-type').to have_sent(
+                                                  test_id: "abc",
+                                                  test_long: 399999,
+                                                  test_union_type: %w(hello world)
+                                                )
+
+          MyProducerWithUnionType.send_event(WidgetWithUnionType.new(
+            test_id: "abc",
+            test_long: 399999,
+            test_union_type: {
+              record1_map:{ a:9999, b:234 },
+              record1_id: 567
+            }
+          ))
+
+          expect('my-topic-with-union-type').to have_sent(
+                                                  test_id: "abc",
+                                                  test_long: 399999,
+                                                  test_union_type:{
+                                                    record1_map:{ a:9999, b:234 },
+                                                    record1_id: 567
+                                                  }
+                                                )
+
+          MyProducerWithUnionType.send_event(WidgetWithUnionType.new(
+            test_id: "abc",
+            test_long: 399999,
+            test_union_type: 1010101
+          ))
+
+          expect('my-topic-with-union-type').to have_sent(
+                                                  test_id: "abc",
+                                                  test_long: 399999,
+                                                  test_union_type:1010101
+                                                )
+
+          MyProducerWithUnionType.send_event(WidgetWithUnionType.new(
+            test_id: "abc",
+            test_long: 399999,
+            test_union_type: {
+              record2_id: "hello world"
+            }
+          ))
+
+          expect('my-topic-with-union-type').to have_sent(
+                                                  test_id: "abc",
+                                                  test_long: 399999,
+                                                  test_union_type: {
+                                                    record2_id: "hello world"
+                                                  }
+                                                )
+
+          MyProducerWithUnionType.send_event(WidgetWithUnionType.new(
+            test_id: "abc",
+            test_long: 399999,
+            test_union_type: {
+              record3_id:10.1010
+            }
+          ))
+
+          expect('my-topic-with-union-type').to have_sent(
+                                                  test_id: "abc",
+                                                  test_long: 399999,
+                                                  test_union_type: {
+                                                    record3_id:10.1010
+                                                  }
+                                                )
+
+          MyProducerWithUnionType.send_event(WidgetWithUnionType.new(
+            test_id: "abc",
+            test_long: 399999,
+            test_union_type: {
+              record4_id:101010
+            }
+          ))
+
+          expect('my-topic-with-union-type').to have_sent(
+                                                  test_id: "abc",
+                                                  test_long: 399999,
+                                                  test_union_type: {
+                                                    record4_id:101010
+                                                  }
+                                                )
+        end
+
         it 'should coerce values' do
           MyProducer.send_event(Widget.new(test_id: 'abc', some_int: '3'))
           MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 4.5))
@@ -109,11 +215,11 @@ describe Deimos::ActiveRecordProducer do
           widget = Widget.create!(test_id: 'abc2', some_int: 3)
           MyProducerWithID.send_event({id: widget.id, test_id: 'abc2', some_int: 3})
           expect('my-topic-with-id').to have_sent(
-            test_id: 'abc2',
-            some_int: 3,
-            message_id: 'generated_id',
-            timestamp: anything
-          )
+                                          test_id: 'abc2',
+                                          some_int: 3,
+                                          message_id: 'generated_id',
+                                          timestamp: anything
+                                        )
         end
 
         it 'should post process the batch of records in #send_events' do

data/spec/schemas/com/my-namespace/MySchemaWithUnionType.avsc

@@ -0,0 +1,91 @@
+{
+  "namespace": "com.my-namespace",
+  "name": "MySchemaWithUnionType",
+  "type": "record",
+  "doc": "Test schema",
+  "fields": [
+    {
+      "name": "test_id",
+      "type": "string",
+      "default": ""
+    },
+    {
+      "name": "test_long",
+      "type": [
+        "null",
+        "long"
+      ],
+      "default": null
+    },
+    {
+      "name": "test_union_type",
+      "type": [
+        "null",
+        {
+          "type": "record",
+          "name": "Record1",
+          "namespace": "com.flipp.content",
+          "fields": [
+            {
+              "name": "record1_map",
+              "type": {
+                "type": "map",
+                "values": "long"
+              },
+              "default": {}
+            },
+            {
+              "name": "record1_id",
+              "type": "int",
+              "default": 0
+            }
+          ]
+        },
+        {
+          "type": "record",
+          "name": "Record2",
+          "namespace": "com.flipp.content",
+          "fields": [
+            {
+              "name": "record2_id",
+              "type": "string",
+              "default": ""
+            }
+          ]
+        },
+        {
+          "type": "record",
+          "name": "Record3",
+          "namespace": "com.flipp.content",
+          "fields": [
+            {
+              "name": "record3_id",
+              "type": "float",
+              "default": 0.0
+            }
+          ]
+        },
+        {
+          "type": "record",
+          "name": "Record4",
+          "namespace": "com.flipp.content",
+          "fields": [
+            {
+              "name": "record4_id",
+              "type": "int",
+              "default": 0
+            }
+          ]
+        },
+        "int",
+        {
+          "name": "test_array_of_strings",
+          "type": "array",
+          "default": [],
+          "items":"string"
+        }
+      ],
+      "default": null
+    }
+  ]
+}

data/spec/schemas/my_namespace/my_schema_with_union_type.rb

@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+
+# This file is autogenerated by Deimos, Do NOT modify
+module Schemas
+  ### Primary Schema Class ###
+  # Autogenerated Schema for Record at com.my-namespace.MySchemaWithUnionType
+  class MySchemaWithUnionType < Deimos::SchemaClass::Record
+
+    ### Secondary Schema Classes ###
+    # Autogenerated Schema for Record at com.flipp.content.Record1
+    class Record1 < Deimos::SchemaClass::Record
+
+      ### Attribute Accessors ###
+      # @return [Hash<String, Integer>]
+      attr_accessor :record1_map
+      # @return [Integer]
+      attr_accessor :record1_id
+
+      # @override
+      def initialize(record1_map: {},
+                     record1_id: 0)
+        super
+        self.record1_map = record1_map
+        self.record1_id = record1_id
+      end
+
+      # @override
+      def schema
+        'Record1'
+      end
+
+      # @override
+      def namespace
+        'com.flipp.content'
+      end
+
+      # @override
+      def as_json(_opts={})
+        {
+          'record1_map' => @record1_map,
+          'record1_id' => @record1_id
+        }
+      end
+    end
+
+    # Autogenerated Schema for Record at com.flipp.content.Record2
+    class Record2 < Deimos::SchemaClass::Record
+
+      ### Attribute Accessors ###
+      # @return [String]
+      attr_accessor :record2_id
+
+      # @override
+      def initialize(record2_id: "")
+        super
+        self.record2_id = record2_id
+      end
+
+      # @override
+      def schema
+        'Record2'
+      end
+
+      # @override
+      def namespace
+        'com.flipp.content'
+      end
+
+      # @override
+      def as_json(_opts={})
+        {
+          'record2_id' => @record2_id
+        }
+      end
+    end
+
+    # Autogenerated Schema for Record at com.flipp.content.Record3
+    class Record3 < Deimos::SchemaClass::Record
+
+      ### Attribute Accessors ###
+      # @return [Float]
+      attr_accessor :record3_id
+
+      # @override
+      def initialize(record3_id: 0.0)
+        super
+        self.record3_id = record3_id
+      end
+
+      # @override
+      def schema
+        'Record3'
+      end
+
+      # @override
+      def namespace
+        'com.flipp.content'
+      end
+
+      # @override
+      def as_json(_opts={})
+        {
+          'record3_id' => @record3_id
+        }
+      end
+    end
+
+    # Autogenerated Schema for Record at com.flipp.content.Record4
+    class Record4 < Deimos::SchemaClass::Record
+
+      ### Attribute Accessors ###
+      # @return [Integer]
+      attr_accessor :record4_id
+
+      # @override
+      def initialize(record4_id: 0)
+        super
+        self.record4_id = record4_id
+      end
+
+      # @override
+      def schema
+        'Record4'
+      end
+
+      # @override
+      def namespace
+        'com.flipp.content'
+      end
+
+      # @override
+      def as_json(_opts={})
+        {
+          'record4_id' => @record4_id
+        }
+      end
+    end
+
+
+    ### Attribute Readers ###
+    # @return [nil, Record1, Record2, Record3, Record4, Integer, Array<String>]
+    attr_reader :test_union_type
+
+    ### Attribute Accessors ###
+    # @return [String]
+    attr_accessor :test_id
+    # @return [nil, Integer]
+    attr_accessor :test_long
+
+    ### Attribute Writers ###
+    # @return [nil, Record1, Record2, Record3, Record4, Integer, Array<String>]
+    def test_union_type=(value)
+      @test_union_type = Record1.initialize_from_value(value)
+    end
+
+    # @override
+    def initialize(test_id: "",
+                   test_long: nil,
+                   test_union_type: nil)
+      super
+      self.test_id = test_id
+      self.test_long = test_long
+      self.test_union_type = test_union_type
+    end
+
+    # @override
+    def schema
+      'MySchemaWithUnionType'
+    end
+
+    # @override
+    def namespace
+      'com.my-namespace'
+    end
+
+    # @override
+    def as_json(_opts={})
+      {
+        'test_id' => @test_id,
+        'test_long' => @test_long,
+        'test_union_type' => @test_union_type&.as_json
+      }
+    end
+  end
+end