deimos-ruby 2.0.0.pre.beta7 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4aeb59ae53a1c0b617581b477fd035fc76182fb2662e2c1655b4f9814747a26a
-  data.tar.gz: 58c80ee0fde7651ac46c8d99a8652598500c04757ce5364aa0ddda7b4ca02f9c
+  metadata.gz: 03e440f70d2ffedc89bae21e3eaef6e317c89d52a53d48d5319fcbea028d775d
+  data.tar.gz: 2a8737062e2404f9bbbb67d35124083c9da740a2f42c670bd0d8a776522377a8
 SHA512:
-  metadata.gz: 7b366555ec5a9fd5f11a54e12c6db102c14a7675da3bfb6afc07b57729bfe21adf643505fb1290c28dcf072e451926d6e2c8cd55ae3ee58713630809d13c01ce
-  data.tar.gz: 48104ebcafa8ac7b0b911928e9aa65ad897747acf8f51279d6ae4bc4b66f26dba471c7329207bf723ec738eea645203cc8546d8f26fccfd204b652ee8eb70cf1
+  metadata.gz: d8beb919e34f80e9fca1b6ca5e0a378b2f8494f06e7f27b29f8bcad4028ae22f9c1747050bd64e4523a6d32e631e1c696ae917e4583859be0313817522788298
+  data.tar.gz: bd6754c7848d7c874e2bcf6f7e27cc65bbe5a1fce8f95fbca27538b56a52f14c8c2ac113c65185ad1cd349a2e25dbd498ea81c5af49f5bfe20cf6be136df7057

data/.github/workflows/ci.yml

@@ -36,7 +36,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.7', '3.0', '3.1', '3.2', '3.3' ]
+        ruby: [ '3.0', '3.1', '3.2', '3.3', '3.4' ]
 
     steps:
       - uses: actions/checkout@v3

data/CHANGELOG.md

@@ -6,9 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## UNRELEASED
+- Fix: Added support to handle producing union type of multiple records & data types.
+- Fix: Outbox producer was trying to double-encode messages.
 
-## 2.0.0 - TBA
-- Full rewrite of Deimos to work with Karafka. Please see [Upgrading](./docs/UPGRADING.md) for full information.
+## 2.0.0 - 2025-03-03
+- ***Full rewrite of Deimos to work with Karafka.*** Please see [Upgrading](./docs/UPGRADING.md) for full information.
 
 # 1.24.3 - 2024-05-13
 - Feature: Enable `producers.persistent_connections` phobos setting

data/Gemfile

@@ -4,3 +4,9 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in boilerplate.gemspec
 gemspec
+
+# for older activesupport
+gem 'mutex_m'
+gem 'bigdecimal'
+gem 'benchmark'
+gem 'drb'

data/README.md

@@ -10,8 +10,10 @@ A Ruby framework for marrying Kafka, a schema definition like Avro, and/or Activ
 a useful toolbox of goodies for Ruby-based Kafka development.
 Built on [Karafka](https://karafka.io/).
 
-[!IMPORTANT]  
-Deimos 2.x is a major rewrite from 1.x. Please see the [Upgrading Guide](./docs/UPGRADING.md) for information on the changes and how to upgrade.
+> [!IMPORTANT]  
+> Deimos 2.x is a major rewrite from 1.x. Please see the [Upgrading Guide](./docs/UPGRADING.md) for information on the changes and how to upgrade.
+
+Version 1.0 documentation can be found [here](https://github.com/flipp-oss/deimos/tree/v1).
 
 <!--ts-->
    * [Additional Documentation](#additional-documentation)

data/lib/deimos/ext/producer_middleware.rb

@@ -11,6 +11,7 @@ module Deimos
         ) do
           config = Deimos.karafka_config_for(topic: message[:topic])
           return message if config.nil? || config.schema.nil?
+          return message if message.delete(:already_encoded)
           return if message[:payload] && !message[:payload].is_a?(Hash) && !message[:payload].is_a?(SchemaClass::Record)
 
           m = Deimos::Message.new(message[:payload].to_h,

data/lib/deimos/kafka_message.rb

@@ -54,7 +54,8 @@ module Deimos
         payload: self.message,
         partition_key: self.partition_key,
         key: self.key,
-        topic: self.topic
+        topic: self.topic,
+        already_encoded: true
       }
     end
 

data/lib/deimos/schema_backends/avro_schema_coercer.rb

@@ -18,10 +18,48 @@ module Deimos
       union_types = type.schemas.map { |s| s.type.to_sym }
       return nil if val.nil? && union_types.include?(:null)
 
-      schema_type = type.schemas.find { |s| s.type.to_sym != :null }
+      schema_type = find_schema_type(type, val)
       coerce_type(schema_type, val)
     end
 
+    # Find the right schema for val from a UnionSchema.
+    # @param type [Avro::Schema::UnionSchema]
+    # @param val [Object]
+    # @return [Avro::Schema::PrimitiveSchema]
+    def find_schema_type(type, val)
+      int_classes = [Time, ActiveSupport::TimeWithZone]
+
+      schema_type = type.schemas.find do |schema|
+        field_type = schema.type.to_sym
+
+        case field_type
+        when :int, :long
+          val.is_a?(Integer) ||
+            _is_integer_string?(val) ||
+            int_classes.any? { |klass| val.is_a?(klass) }
+        when :float, :double
+          val.is_a?(Float) || _is_float_string?(val)
+        when :array
+          val.is_a?(Array)
+        when :record
+          if val.is_a?(Hash)
+            schema_fields_set = Set.new(schema.fields.map(&:name))
+            Set.new(val.keys).subset?(schema_fields_set)
+          else
+            # If the value is not a hash, we can't coerce it to a record.
+            # Keep looking for another schema
+            false
+          end
+        else
+          schema.type.to_sym != :null
+        end
+      end
+
+      raise "No Schema type found for VALUE: #{val}\n TYPE: #{type}" if schema_type.nil?
+
+      schema_type
+    end
+
     # Coerce sub-records in a payload to match the schema.
     # @param type [Avro::Schema::RecordSchema]
     # @param val [Object]

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '2.0.0-beta7'
+  VERSION = '2.0.1'
 end

data/spec/active_record_producer_spec.rb

@@ -3,6 +3,7 @@
 describe Deimos::ActiveRecordProducer do
 
   include_context 'with widgets'
+  include_context 'with widget_with_union_types'
 
   prepend_before(:each) do
     producer_class = Class.new(Deimos::ActiveRecordProducer)
@@ -40,6 +41,12 @@ describe Deimos::ActiveRecordProducer do
     end
 
     stub_const('MyProducerWithPostProcess', producer_class)
+
+    producer_class = Class.new(Deimos::ActiveRecordProducer) do
+      record_class WidgetWithUnionType
+    end
+    stub_const('MyProducerWithUnionType', producer_class)
+
     Karafka::App.routes.redraw do
       topic 'my-topic' do
         schema 'MySchema'
@@ -71,6 +78,13 @@ describe Deimos::ActiveRecordProducer do
         key_config none: true
         producer_class MyProducerWithPostProcess
       end
+      topic 'my-topic-with-union-type' do
+        schema 'MySchemaWithUnionType'
+        namespace 'com.my-namespace'
+        key_config none: true
+        producer_class MyProducerWithUnionType
+      end
+
     end
 
   end
@@ -90,6 +104,98 @@ describe Deimos::ActiveRecordProducer do
           expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
         end
 
+        it 'should coerce values for a UnionSchema' do
+          MyProducerWithUnionType.send_event(WidgetWithUnionType.new(
+            test_id: "abc",
+            test_long: 399999,
+            test_union_type: %w(hello world)
+          ))
+
+          expect('my-topic-with-union-type').to have_sent(
+                                                  test_id: "abc",
+                                                  test_long: 399999,
+                                                  test_union_type: %w(hello world)
+                                                )
+
+          MyProducerWithUnionType.send_event(WidgetWithUnionType.new(
+            test_id: "abc",
+            test_long: 399999,
+            test_union_type: {
+              record1_map:{ a:9999, b:234 },
+              record1_id: 567
+            }
+          ))
+
+          expect('my-topic-with-union-type').to have_sent(
+                                                  test_id: "abc",
+                                                  test_long: 399999,
+                                                  test_union_type:{
+                                                    record1_map:{ a:9999, b:234 },
+                                                    record1_id: 567
+                                                  }
+                                                )
+
+          MyProducerWithUnionType.send_event(WidgetWithUnionType.new(
+            test_id: "abc",
+            test_long: 399999,
+            test_union_type: 1010101
+          ))
+
+          expect('my-topic-with-union-type').to have_sent(
+                                                  test_id: "abc",
+                                                  test_long: 399999,
+                                                  test_union_type:1010101
+                                                )
+
+          MyProducerWithUnionType.send_event(WidgetWithUnionType.new(
+            test_id: "abc",
+            test_long: 399999,
+            test_union_type: {
+              record2_id: "hello world"
+            }
+          ))
+
+          expect('my-topic-with-union-type').to have_sent(
+                                                  test_id: "abc",
+                                                  test_long: 399999,
+                                                  test_union_type: {
+                                                    record2_id: "hello world"
+                                                  }
+                                                )
+
+          MyProducerWithUnionType.send_event(WidgetWithUnionType.new(
+            test_id: "abc",
+            test_long: 399999,
+            test_union_type: {
+              record3_id:10.1010
+            }
+          ))
+
+          expect('my-topic-with-union-type').to have_sent(
+                                                  test_id: "abc",
+                                                  test_long: 399999,
+                                                  test_union_type: {
+                                                    record3_id:10.1010
+                                                  }
+                                                )
+
+          MyProducerWithUnionType.send_event(WidgetWithUnionType.new(
+            test_id: "abc",
+            test_long: 399999,
+            test_union_type: {
+              record4_id:101010
+            }
+          ))
+
+          expect('my-topic-with-union-type').to have_sent(
+                                                  test_id: "abc",
+                                                  test_long: 399999,
+                                                  test_union_type: {
+                                                    record4_id:101010
+                                                  }
+                                                )
+        end
+
         it 'should coerce values' do
           MyProducer.send_event(Widget.new(test_id: 'abc', some_int: '3'))
           MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 4.5))
@@ -109,11 +215,11 @@ describe Deimos::ActiveRecordProducer do
           widget = Widget.create!(test_id: 'abc2', some_int: 3)
           MyProducerWithID.send_event({id: widget.id, test_id: 'abc2', some_int: 3})
           expect('my-topic-with-id').to have_sent(
-            test_id: 'abc2',
-            some_int: 3,
-            message_id: 'generated_id',
-            timestamp: anything
-          )
+                                          test_id: 'abc2',
+                                          some_int: 3,
+                                          message_id: 'generated_id',
+                                          timestamp: anything
+                                        )
         end
 
         it 'should post process the batch of records in #send_events' do

data/spec/generators/schema_class_generator_spec.rb

@@ -4,8 +4,17 @@ require 'generators/deimos/schema_class_generator'
 require 'fileutils'
 
 class MultiFileSerializer
+  def process_string(s)
+    # Ruby 3.4 changes how hashes are printed
+    if Gem::Version.new(RUBY_VERSION) > Gem::Version.new('3.4.0')
+      s.gsub(/{"(.*)" => /, '{"\1"=>')
+    else
+      s
+    end
+
+  end
   def dump(value)
-    value.keys.sort.map { |k| "#{k}:\n#{value[k]}\n" }.join("\n")
+    value.keys.sort.map { |k| "#{k}:\n#{process_string(value[k])}\n" }.join("\n")
   end
 end
 

data/spec/schemas/com/my-namespace/MySchemaWithUnionType.avsc

@@ -0,0 +1,91 @@
+{
+  "namespace": "com.my-namespace",
+  "name": "MySchemaWithUnionType",
+  "type": "record",
+  "doc": "Test schema",
+  "fields": [
+    {
+      "name": "test_id",
+      "type": "string",
+      "default": ""
+    },
+    {
+      "name": "test_long",
+      "type": [
+        "null",
+        "long"
+      ],
+      "default": null
+    },
+    {
+      "name": "test_union_type",
+      "type": [
+        "null",
+        {
+          "type": "record",
+          "name": "Record1",
+          "namespace": "com.flipp.content",
+          "fields": [
+            {
+              "name": "record1_map",
+              "type": {
+                "type": "map",
+                "values": "long"
+              },
+              "default": {}
+            },
+            {
+              "name": "record1_id",
+              "type": "int",
+              "default": 0
+            }
+          ]
+        },
+        {
+          "type": "record",
+          "name": "Record2",
+          "namespace": "com.flipp.content",
+          "fields": [
+            {
+              "name": "record2_id",
+              "type": "string",
+              "default": ""
+            }
+          ]
+        },
+        {
+          "type": "record",
+          "name": "Record3",
+          "namespace": "com.flipp.content",
+          "fields": [
+            {
+              "name": "record3_id",
+              "type": "float",
+              "default": 0.0
+            }
+          ]
+        },
+        {
+          "type": "record",
+          "name": "Record4",
+          "namespace": "com.flipp.content",
+          "fields": [
+            {
+              "name": "record4_id",
+              "type": "int",
+              "default": 0
+            }
+          ]
+        },
+        "int",
+        {
+          "name": "test_array_of_strings",
+          "type": "array",
+          "default": [],
+          "items":"string"
+        }
+      ],
+      "default": null
+    }
+  ]
+}

data/spec/schemas/my_namespace/my_schema_with_union_type.rb

@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+
+# This file is autogenerated by Deimos, Do NOT modify
+module Schemas
+  ### Primary Schema Class ###
+  # Autogenerated Schema for Record at com.my-namespace.MySchemaWithUnionType
+  class MySchemaWithUnionType < Deimos::SchemaClass::Record
+
+    ### Secondary Schema Classes ###
+    # Autogenerated Schema for Record at com.flipp.content.Record1
+    class Record1 < Deimos::SchemaClass::Record
+
+      ### Attribute Accessors ###
+      # @return [Hash<String, Integer>]
+      attr_accessor :record1_map
+      # @return [Integer]
+      attr_accessor :record1_id
+
+      # @override
+      def initialize(record1_map: {},
+                     record1_id: 0)
+        super
+        self.record1_map = record1_map
+        self.record1_id = record1_id
+      end
+
+      # @override
+      def schema
+        'Record1'
+      end
+
+      # @override
+      def namespace
+        'com.flipp.content'
+      end
+
+      # @override
+      def as_json(_opts={})
+        {
+          'record1_map' => @record1_map,
+          'record1_id' => @record1_id
+        }
+      end
+    end
+
+    # Autogenerated Schema for Record at com.flipp.content.Record2
+    class Record2 < Deimos::SchemaClass::Record
+
+      ### Attribute Accessors ###
+      # @return [String]
+      attr_accessor :record2_id
+
+      # @override
+      def initialize(record2_id: "")
+        super
+        self.record2_id = record2_id
+      end
+
+      # @override
+      def schema
+        'Record2'
+      end
+
+      # @override
+      def namespace
+        'com.flipp.content'
+      end
+
+      # @override
+      def as_json(_opts={})
+        {
+          'record2_id' => @record2_id
+        }
+      end
+    end
+
+    # Autogenerated Schema for Record at com.flipp.content.Record3
+    class Record3 < Deimos::SchemaClass::Record
+
+      ### Attribute Accessors ###
+      # @return [Float]
+      attr_accessor :record3_id
+
+      # @override
+      def initialize(record3_id: 0.0)
+        super
+        self.record3_id = record3_id
+      end
+
+      # @override
+      def schema
+        'Record3'
+      end
+
+      # @override
+      def namespace
+        'com.flipp.content'
+      end
+
+      # @override
+      def as_json(_opts={})
+        {
+          'record3_id' => @record3_id
+        }
+      end
+    end
+
+    # Autogenerated Schema for Record at com.flipp.content.Record4
+    class Record4 < Deimos::SchemaClass::Record
+
+      ### Attribute Accessors ###
+      # @return [Integer]
+      attr_accessor :record4_id
+
+      # @override
+      def initialize(record4_id: 0)
+        super
+        self.record4_id = record4_id
+      end
+
+      # @override
+      def schema
+        'Record4'
+      end
+
+      # @override
+      def namespace
+        'com.flipp.content'
+      end
+
+      # @override
+      def as_json(_opts={})
+        {
+          'record4_id' => @record4_id
+        }
+      end
+    end
+
+
+    ### Attribute Readers ###
+    # @return [nil, Record1, Record2, Record3, Record4, Integer, Array<String>]
+    attr_reader :test_union_type
+
+    ### Attribute Accessors ###
+    # @return [String]
+    attr_accessor :test_id
+    # @return [nil, Integer]
+    attr_accessor :test_long
+
+    ### Attribute Writers ###
+    # @return [nil, Record1, Record2, Record3, Record4, Integer, Array<String>]
+    def test_union_type=(value)
+      @test_union_type = Record1.initialize_from_value(value)
+    end
+
+    # @override
+    def initialize(test_id: "",
+                   test_long: nil,
+                   test_union_type: nil)
+      super
+      self.test_id = test_id
+      self.test_long = test_long
+      self.test_union_type = test_union_type
+    end
+
+    # @override
+    def schema
+      'MySchemaWithUnionType'
+    end
+
+    # @override
+    def namespace
+      'com.my-namespace'
+    end
+
+    # @override
+    def as_json(_opts={})
+      {
+        'test_id' => @test_id,
+        'test_long' => @test_long,
+        'test_union_type' => @test_union_type&.as_json
+      }
+    end
+  end
+end