deimos-ruby 2.0.3 → 2.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1fac86a45a84b2698c358a6c94476567047ef006b53dcf4cfd005451d9f78a10
-  data.tar.gz: b547072ec4e0c9bb18880a497e28d567169da144397bba748d65f5bcf2b53e17
+  metadata.gz: 53cd09a846fed14b74b20cad18fc63374887c0a9deae4444f2d6548ea06b5b72
+  data.tar.gz: 95f7b6402d43d2c2332f5abbc8c0374655b1ac713b484d3f4b516321b4e2207d
 SHA512:
-  metadata.gz: 2e871a8e7aa75c568f6c8cfd0fadce2acf955d6703842b784385d32f55f579f4f239924c589066d088dd9dc3bc88e789e4597494aabc4c61bf6fc37979cca5c6
-  data.tar.gz: a595322011ec7bda0d80d6bad141c0dfbcc49dff3609c15299d3ad7ff9fab225cb8063343fdcab2443ac7bde60dc4e16557568912a0df8d630b9b65266df9e4e
+  metadata.gz: 928b533075a3a98f5731f15e2e9436cd1ad0754006bcf18816bdc19933e0990478aa7cd06152e450f0041524cfcf04da839c1116a4895bf3c9e692e70aefa4f0
+  data.tar.gz: b363497c5b9f1e9ea6dd71bf2d4e4bd59e7b96fe229cc58ee9caccb3ca86a7cd1ef4bf0636e7a354d496c790aa867fcad3405bdd4b353e72fce1d1e22ba32876

data/CHANGELOG.md

@@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+## 2.0.5 - 2025-03-19
+
+- Fix: Added support to handle producing union type of multiple records & data types.
+
+## 2.0.4 - 2025-03-17
+
+- Feature: Added `producers.truncate_columns` config.
+
 ## 2.0.3 - 2025-03-12
 - Fix: `ActiveRecordProducer.config` could crash if there were non-producer configs.
 

data/docs/CONFIGURATION.md

@@ -43,11 +43,12 @@ things you need to reference into local variables before calling `configure`.
 
 ### Producer Configuration
 
-| Config name            | Default        | Description                                                                                                                                                                                    |
-|------------------------|----------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| producers.topic_prefix | nil            | Add a prefix to all topic names. This can be useful if you're using the same Kafka broker for different environments that are producing the same topics.                                       |
-| producers.disabled     | false          | Disable all actual message producing. Generally more useful to use the `disable_producers` method instead.                                                                                     |
-| producers.backend      | `:kafka_async` | Currently can be set to `:db`, `:kafka`, or `:kafka_async`. If using Kafka directly, a good pattern is to set to async in your user-facing app, and sync in your consumers or delayed workers. |
+| Config name                | Default        | Description                                                                                                                                                                                    |
+|----------------------------|----------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| producers.topic_prefix     | nil            | Add a prefix to all topic names. This can be useful if you're using the same Kafka broker for different environments that are producing the same topics.                                       |
+| producers.disabled         | false          | Disable all actual message producing. Generally more useful to use the `disable_producers` method instead.                                                                                     |
+| producers.backend          | `:kafka_async` | Currently can be set to `:db`, `:kafka`, or `:kafka_async`. If using Kafka directly, a good pattern is to set to async in your user-facing app, and sync in your consumers or delayed workers. |
+| producers.truncate_columns | false          | If set to true, will truncate values to their database limits when using KafkaSource.                                                                                                          |
 
 ### Schema Configuration
 

data/lib/deimos/config/configuration.rb

@@ -133,6 +133,10 @@ module Deimos # rubocop:disable Metrics/ModuleLength
       # sync in your consumers or delayed workers.
       # @return [Symbol]
       setting :backend, :kafka_async
+
+      # If set to true, KafkaSource will automatically truncate fields to match the column
+      # length in the database.
+      setting :truncate_columns
     end
 
     setting :schema do

data/lib/deimos/kafka_source.rb

@@ -6,10 +6,6 @@ module Deimos
   module KafkaSource
     extend ActiveSupport::Concern
 
-    # @return [String]
-    DEPRECATION_WARNING = 'The kafka_producer interface will be deprecated ' \
-        'in future releases. Please use kafka_producers instead.'
-
     included do
       after_create(:send_kafka_event_on_create)
       after_update(:send_kafka_event_on_update)
@@ -22,6 +18,7 @@ module Deimos
       return unless self.persisted?
       return unless self.class.kafka_config[:create]
 
+      self.truncate_columns if Deimos.config.producers.truncate_columns
       self.class.kafka_producers.each { |p| p.send_event(self) }
     end
 
@@ -39,6 +36,7 @@ module Deimos
         field_change.present? && field_change[0] != field_change[1]
       end
       return unless any_changes
+      self.truncate_columns if Deimos.config.producers.truncate_columns
 
       producers.each { |p| p.send_event(self) }
     end
@@ -124,5 +122,17 @@ module Deimos
         results
       end
     end
+
+    # check if any field has value longer than the field limit
+    def truncate_columns
+      self.class.columns.each do |col|
+        next unless col.type == :string
+        next if self[col.name].blank?
+        if self[col.name].to_s.length > col.limit
+          self[col.name] = self[col.name][0..col.limit - 1]
+        end
+      end
+      false
+    end
   end
 end

data/lib/deimos/schema_backends/avro_schema_coercer.rb

@@ -18,10 +18,48 @@ module Deimos
       union_types = type.schemas.map { |s| s.type.to_sym }
       return nil if val.nil? && union_types.include?(:null)
 
-      schema_type = type.schemas.find { |s| s.type.to_sym != :null }
+      schema_type = find_schema_type(type, val)
       coerce_type(schema_type, val)
     end
 
+    # Find the right schema for val from a UnionSchema.
+    # @param type [Avro::Schema::UnionSchema]
+    # @param val [Object]
+    # @return [Avro::Schema::PrimitiveSchema]
+    def find_schema_type(type, val)
+      int_classes = [Time, ActiveSupport::TimeWithZone]
+
+      schema_type = type.schemas.find do |schema|
+        field_type = schema.type.to_sym
+
+        case field_type
+        when :int, :long
+          val.is_a?(Integer) ||
+            _is_integer_string?(val) ||
+            int_classes.any? { |klass| val.is_a?(klass) }
+        when :float, :double
+          val.is_a?(Numeric) || _is_float_string?(val)
+        when :array
+          val.is_a?(Array)
+        when :record
+          if val.is_a?(Hash)
+            schema_fields_set = Set.new(schema.fields.map(&:name))
+            Set.new(val.keys).subset?(schema_fields_set)
+          else
+            # If the value is not a hash, we can't coerce it to a record.
+            # Keep looking for another schema
+            false
+          end
+        else
+          schema.type.to_sym != :null
+        end
+      end
+
+      raise "No Schema type found for VALUE: #{val}\n TYPE: #{type}" if schema_type.nil?
+
+      schema_type
+    end
+
     # Coerce sub-records in a payload to match the schema.
     # @param type [Avro::Schema::RecordSchema]
     # @param val [Object]

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '2.0.3'
+  VERSION = '2.0.5'
 end

data/lib/generators/deimos/schema_class/templates/schema_record.rb.tt

@@ -44,6 +44,24 @@
     end
 
   <%- end -%>
+<% end -%>
+<%- if @field_assignments.select{ |h| h[:is_complex_union] }.any? -%>
+    <%- @field_assignments.select{ |h| h[:is_complex_union] }.each do |method_definition| -%>
+    # Helper method to determine which schema type to use for <%= method_definition[:field].name %>
+    # @param value [Hash, nil]
+    # @return [Object, nil]
+    def initialize_<%= method_definition[:field].name %>_type(value)
+      return nil if value.nil?
+
+      klass = [<%= method_definition[:field].type.schemas.reject { |s| s.type_sym == :null }.select { |s| s.type_sym == :record }.map { |s| Deimos::SchemaBackends::AvroBase.schema_classname(s) }.join(', ') %>].find do |candidate|
+        fields = candidate.new.as_json.keys
+        (value.keys - fields).empty?
+      end
+
+      klass.initialize_from_value(value)
+    end
+
+    <%- end -%>
 <% end -%>
     # @override
     <%= @initialization_definition %>

data/lib/generators/deimos/schema_class_generator.rb

@@ -298,7 +298,9 @@ module Deimos
 
           field_initialization = method_argument
 
-          if is_schema_class
+          if _is_complex_union?(field)
+            field_initialization = "initialize_#{field.name}_type(value)"
+          elsif is_schema_class
             field_initialization = "#{field_base_type}.initialize_from_value(value)"
           end
 
@@ -308,13 +310,26 @@ module Deimos
             is_schema_class: is_schema_class,
             method_argument: method_argument,
             deimos_type: deimos_field_type(field),
-            field_initialization: field_initialization
+            field_initialization: field_initialization,
+            is_complex_union: _is_complex_union?(field)
           }
         end
 
         result
       end
 
+      # Helper method to detect if a field is a complex union type with multiple record schemas
+      # @param field [Deimos::SchemaField]
+      # @return [Boolean]
+      def _is_complex_union?(field)
+        return false unless field.type.type_sym == :union
+
+        non_null_schemas = field.type.schemas.reject { |s| s.type_sym == :null }
+
+        record_schemas = non_null_schemas.select { |s| s.type_sym == :record }
+        record_schemas.length > 1
+      end
+
       # Converts Avro::Schema::NamedSchema's to String form for generated YARD docs.
       # Recursively handles the typing for Arrays, Maps and Unions.
       # @param avro_schema [Avro::Schema::NamedSchema]

data/spec/active_record_producer_spec.rb

@@ -3,6 +3,7 @@
 describe Deimos::ActiveRecordProducer do
 
   include_context 'with widgets'
+  include_context 'with widget_with_union_types'
 
   prepend_before(:each) do
     producer_class = Class.new(Deimos::ActiveRecordProducer)
@@ -40,6 +41,12 @@ describe Deimos::ActiveRecordProducer do
     end
 
     stub_const('MyProducerWithPostProcess', producer_class)
+
+    producer_class = Class.new(Deimos::ActiveRecordProducer) do
+      record_class WidgetWithUnionType
+    end
+    stub_const('MyProducerWithUnionType', producer_class)
+
     Karafka::App.routes.redraw do
       topic 'my-topic' do
         schema 'MySchema'
@@ -71,6 +78,13 @@ describe Deimos::ActiveRecordProducer do
         key_config none: true
         producer_class MyProducerWithPostProcess
       end
+      topic 'my-topic-with-union-type' do
+        schema 'MySchemaWithUnionType'
+        namespace 'com.my-namespace'
+        key_config none: true
+        producer_class MyProducerWithUnionType
+      end
+
     end
 
   end
@@ -90,6 +104,98 @@ describe Deimos::ActiveRecordProducer do
           expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
         end
 
+        it 'should coerce values for a UnionSchema' do
+          MyProducerWithUnionType.send_event(WidgetWithUnionType.new(
+            test_id: "abc",
+            test_long: 399999,
+            test_union_type: %w(hello world)
+          ))
+
+          expect('my-topic-with-union-type').to have_sent(
+                                                  test_id: "abc",
+                                                  test_long: 399999,
+                                                  test_union_type: %w(hello world)
+                                                )
+
+          MyProducerWithUnionType.send_event(WidgetWithUnionType.new(
+            test_id: "abc",
+            test_long: 399999,
+            test_union_type: {
+              record1_map:{ a:9999, b:234 },
+              record1_id: 567
+            }
+          ))
+
+          expect('my-topic-with-union-type').to have_sent(
+                                                  test_id: "abc",
+                                                  test_long: 399999,
+                                                  test_union_type:{
+                                                    record1_map:{ a:9999, b:234 },
+                                                    record1_id: 567
+                                                  }
+                                                )
+
+          MyProducerWithUnionType.send_event(WidgetWithUnionType.new(
+            test_id: "abc",
+            test_long: 399999,
+            test_union_type: 1010101
+          ))
+
+          expect('my-topic-with-union-type').to have_sent(
+                                                  test_id: "abc",
+                                                  test_long: 399999,
+                                                  test_union_type:1010101
+                                                )
+
+          MyProducerWithUnionType.send_event(WidgetWithUnionType.new(
+            test_id: "abc",
+            test_long: 399999,
+            test_union_type: {
+              record2_id: "hello world"
+            }
+          ))
+
+          expect('my-topic-with-union-type').to have_sent(
+                                                  test_id: "abc",
+                                                  test_long: 399999,
+                                                  test_union_type: {
+                                                    record2_id: "hello world"
+                                                  }
+                                                )
+
+          MyProducerWithUnionType.send_event(WidgetWithUnionType.new(
+            test_id: "abc",
+            test_long: 399999,
+            test_union_type: {
+              record3_id:10.1010
+            }
+          ))
+
+          expect('my-topic-with-union-type').to have_sent(
+                                                  test_id: "abc",
+                                                  test_long: 399999,
+                                                  test_union_type: {
+                                                    record3_id:10.1010
+                                                  }
+                                                )
+
+          MyProducerWithUnionType.send_event(WidgetWithUnionType.new(
+            test_id: "abc",
+            test_long: 399999,
+            test_union_type: {
+              record4_id:101010
+            }
+          ))
+
+          expect('my-topic-with-union-type').to have_sent(
+                                                  test_id: "abc",
+                                                  test_long: 399999,
+                                                  test_union_type: {
+                                                    record4_id:101010
+                                                  }
+                                                )
+        end
+
         it 'should coerce values' do
           MyProducer.send_event(Widget.new(test_id: 'abc', some_int: '3'))
           MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 4.5))
@@ -109,11 +215,11 @@ describe Deimos::ActiveRecordProducer do
           widget = Widget.create!(test_id: 'abc2', some_int: 3)
           MyProducerWithID.send_event({id: widget.id, test_id: 'abc2', some_int: 3})
           expect('my-topic-with-id').to have_sent(
-            test_id: 'abc2',
-            some_int: 3,
-            message_id: 'generated_id',
-            timestamp: anything
-          )
+                                          test_id: 'abc2',
+                                          some_int: 3,
+                                          message_id: 'generated_id',
+                                          timestamp: anything
+                                        )
         end
 
         it 'should post process the batch of records in #send_events' do

data/spec/kafka_source_spec.rb

@@ -10,7 +10,7 @@ module KafkaSourceSpec
         t.integer(:widget_id)
         t.string(:description)
         t.string(:model_id, default: '')
-        t.string(:name)
+        t.string(:name, limit: 100)
         t.timestamps
       end
       ActiveRecord::Base.connection.add_index(:widgets, :widget_id)
@@ -105,6 +105,54 @@ module KafkaSourceSpec
       expect('my-topic-the-second').to have_sent(nil, widget.id)
     end
 
+    context 'with truncation off' do
+      before(:each) do
+        Deimos.config.producers.truncate_columns = false
+      end
+      it 'should not truncate values' do
+        widget = Widget.create!(widget_id: 1, name: 'a'*500)
+        expect('my-topic').to have_sent({
+                                          widget_id: 1,
+                                          name: 'a'*500,
+                                          id: widget.id,
+                                          created_at: anything,
+                                          updated_at: anything
+                                        }, 1)
+        widget.update_attribute(:name, 'b'*500)
+        expect('my-topic').to have_sent({
+                                          widget_id: 1,
+                                          name: 'b'*500,
+                                          id: widget.id,
+                                          created_at: anything,
+                                          updated_at: anything
+                                        }, 1)
+      end
+    end
+
+    context 'with truncation on' do
+      before(:each) do
+        Deimos.config.producers.truncate_columns = true
+      end
+      it 'should truncate values' do
+        widget = Widget.create!(widget_id: 1, name: 'a'*500)
+        expect('my-topic').to have_sent({
+                                          widget_id: 1,
+                                          name: 'a'*100,
+                                          id: widget.id,
+                                          created_at: anything,
+                                          updated_at: anything
+                                        }, 1)
+        widget.update_attribute(:name, 'b'*500)
+        expect('my-topic').to have_sent({
+                                          widget_id: 1,
+                                          name: 'b'*100,
+                                          id: widget.id,
+                                          created_at: anything,
+                                          updated_at: anything
+                                        }, 1)
+      end
+    end
+
     it 'should send events on import' do
       widgets = (1..3).map do |i|
         Widget.new(widget_id: i, name: "Widget #{i}")

data/spec/schemas/com/my-namespace/MySchemaWithUnionType.avsc

@@ -0,0 +1,91 @@
+{
+  "namespace": "com.my-namespace",
+  "name": "MySchemaWithUnionType",
+  "type": "record",
+  "doc": "Test schema",
+  "fields": [
+    {
+      "name": "test_id",
+      "type": "string",
+      "default": ""
+    },
+    {
+      "name": "test_long",
+      "type": [
+        "null",
+        "long"
+      ],
+      "default": null
+    },
+    {
+      "name": "test_union_type",
+      "type": [
+        "null",
+        {
+          "type": "record",
+          "name": "Record1",
+          "namespace": "com.flipp.content",
+          "fields": [
+            {
+              "name": "record1_map",
+              "type": {
+                "type": "map",
+                "values": "long"
+              },
+              "default": {}
+            },
+            {
+              "name": "record1_id",
+              "type": "int",
+              "default": 0
+            }
+          ]
+        },
+        {
+          "type": "record",
+          "name": "Record2",
+          "namespace": "com.flipp.content",
+          "fields": [
+            {
+              "name": "record2_id",
+              "type": "string",
+              "default": ""
+            }
+          ]
+        },
+        {
+          "type": "record",
+          "name": "Record3",
+          "namespace": "com.flipp.content",
+          "fields": [
+            {
+              "name": "record3_id",
+              "type": "float",
+              "default": 0.0
+            }
+          ]
+        },
+        {
+          "type": "record",
+          "name": "Record4",
+          "namespace": "com.flipp.content",
+          "fields": [
+            {
+              "name": "record4_id",
+              "type": "int",
+              "default": 0
+            }
+          ]
+        },
+        "int",
+        {
+          "name": "test_array_of_strings",
+          "type": "array",
+          "default": [],
+          "items":"string"
+        }
+      ],
+      "default": null
+    }
+  ]
+}