deimos-ruby 2.3.0.pre.beta4 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f8e48e6252ea21b5af3a7f2ec921f7a82db95e6286eaf52ac15f94ad8e1043d6
-  data.tar.gz: 99bf9b5ecb064e644897e1ef2177704d460b9531a9fc51e551b8462f22fe1943
+  metadata.gz: 9811f05eec0d9b6777e2ed047a2e81b0a23bd770b52866bf8c8a79f84c8ec586
+  data.tar.gz: c84cba2ecf750f4349ba86e47ddab2d01a4dd8e5f8fd9ab008b3fcfeb14b1425
 SHA512:
-  metadata.gz: 7cf3edfdf000781943adb58eddc2e73e2fa2c8b7cede12cfbebc1407235af33459b44402602a9e26534187f81df9ed67a38029ffc917d0c94e4e0a8924645929
-  data.tar.gz: da90d82a5919b02893b7877e668a287f3deb64c766f6b5a5543d6217b2782c14ab4b0c87f87f0dcad8e537905b5b480d53f756530eb1b4c6e187c0569f13b0f0
+  metadata.gz: b8319ada2f2b715e750e7de5caf6c6117d6154da6277988e12851f420c69a2f3adc0ab30594a23bca86ec8ffa57ef2cc3e5ab9758421e26fbe91cf682ad28e04
+  data.tar.gz: 2174b8d776b547689422d6af381f423680d61df368be813e53f213eb8d084971b338a3382d8775e07ef324b9443fc1f5427fed834c54629d52c88728d016fd49

data/.rubocop.yml

@@ -4,7 +4,7 @@ plugins:
   - rubocop-rspec
 
 AllCops:
-  TargetRubyVersion: 3.0
+  TargetRubyVersion: 2.5
   Exclude:
     - lib/deimos/monkey_patches/*.rb
     - spec/gen/**/*.rb

data/CHANGELOG.md

@@ -7,7 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
-- Major change: Switch from using `avro_turf` and `proto_turf` to use `schema_registry_client`, which handles both Avro and Protobuf.
+# 2.3.0 - 2026-01-13
+
+- Feature: Support broker setting per topic in producer configs.
 
 # 2.2.2 - 2025-11-7
 

data/README.md

@@ -94,6 +94,8 @@ Currently we have the following possible schema backends:
 * Protobuf Schema Registry (use Protobuf with the Confluent Schema Registry)
 * Mock (no actual encoding/decoding).
 
+Note that to use Protobuf, you must include the [proto_turf](https://github.com/flipp-oss/proto_turf) gem in your Gemfile.
+
 Other possible schemas could [JSONSchema](https://json-schema.org/), etc. Feel free to
 contribute!
 
@@ -139,6 +141,12 @@ end
 
 Note that if you are using Protobuf, you need to pass a Protobuf message object as the payload - you can't use a bare hash.
 
+## Multiple clusters
+
+If you have topics that are being routed to different clusters via Karafka configs, you can continue to make use of Deimos producers without having to instantiate the producer itself. Instead of calling `MyProducer.produce(message)`, you can call `Deimos.producer_for('MyTopic').produce(message)`.
+
+Deimos will keep around one producer per broker server (i.e. `bootstrap.servers` config) that it sees on startup.
+
 ## Auto-added Fields
 
 If your schema has a field called `message_id`, and the payload you give

data/deimos-ruby.gemspec

@@ -17,10 +17,10 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
+  spec.add_dependency('avro_turf', '>= 1.4', '< 2')
   spec.add_dependency('benchmark', '~> 0.5')
   spec.add_dependency('fig_tree', '~> 0.2.0')
   spec.add_dependency('karafka', '~> 2.0')
-  spec.add_dependency('schema_registry_client')
   spec.add_dependency('sigurd', '>= 0.1.0', '< 1.0')
 
   spec.add_development_dependency('activerecord-import')
@@ -33,6 +33,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency('guard-rubocop', '~> 1')
   spec.add_development_dependency('karafka-testing', '~> 2.0')
   spec.add_development_dependency('pg', '~> 1.1')
+  spec.add_development_dependency('proto_turf')
   spec.add_development_dependency('rails', '~> 8.0')
   spec.add_development_dependency('rake', '~> 13')
   spec.add_development_dependency('rspec', '~> 3')
@@ -41,7 +42,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency('rspec-snapshot', '~> 2.0')
   spec.add_development_dependency('rubocop', '~> 1.0')
   spec.add_development_dependency('rubocop-rspec', '3.8')
-  spec.add_development_dependency('schema_registry_client')
   spec.add_development_dependency('sord', '>= 5.0')
   spec.add_development_dependency('sqlite3', '~> 2.7')
   spec.add_development_dependency('steep', '~> 1.0')

data/lib/deimos/backends/kafka.rb

@@ -6,7 +6,7 @@ module Deimos
     class Kafka < Base
       # :nodoc:
       def self.execute(producer_class:, messages:)
-        Karafka.producer.produce_many_sync(messages)
+        Deimos.producer_for(producer_class.topic).produce_many_sync(messages)
       end
     end
   end

data/lib/deimos/backends/kafka_async.rb

@@ -4,9 +4,10 @@ module Deimos
   module Backends
     # Backend which produces to Kafka via an async producer.
     class KafkaAsync < Base
+
       # :nodoc:
       def self.execute(producer_class:, messages:)
-        Karafka.producer.produce_many_async(messages)
+        Deimos.producer_for(producer_class.topic).produce_many_async(messages)
       end
     end
   end

data/lib/deimos/config/configuration.rb

@@ -43,7 +43,7 @@ module Deimos
               'Please provide a directory.'
       end
 
-      Dir["./#{Deimos.config.schema.generated_class_path}/**/*.rb"].
+      Dir["./#{Deimos.config.schema.generated_class_path}/**/*.rb"].sort.
         each { |f| require f }
     rescue LoadError
       raise 'Cannot load schema classes. Please regenerate classes with' \

data/lib/deimos/ext/producer_middleware.rb

@@ -83,7 +83,7 @@ module Deimos
                                     nil
                                   else
                                     encoder.encode(message.payload,
-                                                   topic: "#{Deimos.config.producers.topic_prefix}#{config.name}")
+                                                   topic: "#{Deimos.config.producers.topic_prefix}#{config.name}-value")
                                   end
       end
 
@@ -96,7 +96,7 @@ module Deimos
         if config.deserializers[:key].respond_to?(:encode_key)
           config.deserializers[:key].encode_key(key)
         elsif key
-          config.deserializers[:payload].encode(key, is_key: true)
+          config.deserializers[:payload].encode(key)
         else
           key
         end

data/lib/deimos/kafka_source.rb

@@ -118,7 +118,7 @@ end
           unique_columns = column_names.map(&:to_s) -
                            options[:on_duplicate_key_update].map(&:to_s) - %w(id created_at)
           records = hashes_without_id.map do |hash|
-            self.where(unique_columns.to_h { |c| [c, hash[c]] }).first
+            self.where(unique_columns.map { |c| [c, hash[c]] }.to_h).first
           end
           self.kafka_producers.each { |p| p.send_events(records) }
         else

data/lib/deimos/metrics/datadog.rb

@@ -58,7 +58,9 @@ module Deimos
           end
         end
         Karafka::Setup::Config.setup if Karafka.producer.nil?
-        Karafka.producer.monitor.subscribe(waterdrop_listener)
+        Deimos.waterdrop_producers.each do |producer|
+          producer.monitor.subscribe(waterdrop_listener)
+        end
       end
 
       # :nodoc:

data/lib/deimos/schema_backends/avro_base.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
 require_relative 'base'
-require 'schema_registry_client'
+require 'avro'
+require 'avro_turf'
+require 'avro_turf/mutable_schema_store'
 require_relative 'avro_schema_coercer'
 
 module Deimos
@@ -13,7 +15,7 @@ module Deimos
       # @override
       def initialize(schema:, namespace:)
         super
-        @schema_store = SchemaRegistry::AvroSchemaStore.new(path: Deimos.config.schema.path)
+        @schema_store = AvroTurf::MutableSchemaStore.new(path: Deimos.config.schema.path)
       end
 
       def supports_key_schemas?
@@ -29,7 +31,7 @@ module Deimos
       def encode_key(key_id, key, topic: nil)
         begin
           @key_schema ||= @schema_store.find("#{@schema}_key")
-        rescue SchemaRegistry::SchemaNotFoundError
+        rescue AvroTurf::SchemaNotFoundError
           @key_schema = generate_key_schema(key_id)
         end
         field_name = _field_name_from_schema(@key_schema)
@@ -182,7 +184,7 @@ module Deimos
       # @return [Avro::Schema]
       def avro_schema(schema=nil)
         schema ||= @schema
-        @schema_store.find("#{@namespace}.#{schema}")
+        @schema_store.find(schema, @namespace)
       end
 
       # @param value_schema [Hash]

data/lib/deimos/schema_backends/avro_local.rb

@@ -8,24 +8,23 @@ module Deimos
     class AvroLocal < AvroBase
       # @override
       def decode_payload(payload, schema:)
-        stream = StringIO.new(payload)
-        schema = @schema_store.find("#{@namespace}.#{schema}")
-        reader = Avro::IO::DatumReader.new(nil, schema)
-        Avro::DataFile::Reader.new(stream, reader).first
+        avro_turf.decode(payload, schema_name: schema, namespace: @namespace)
       end
 
       # @override
-      def encode_payload(payload, schema: nil, subject: nil)
-        stream = StringIO.new
-        schema = schema_store.find("#{@namespace}.#{schema}")
-        writer = Avro::IO::DatumWriter.new(schema)
-
-        dw = Avro::DataFile::Writer.new(stream, writer, schema)
-        dw << payload.to_h
-        dw.close
-        stream.string
+      def encode_payload(payload, schema: nil, topic: nil)
+        avro_turf.encode(payload, schema_name: schema, namespace: @namespace)
       end
 
+    private
+
+      # @return [AvroTurf]
+      def avro_turf
+        @avro_turf ||= AvroTurf.new(
+          schemas_path: Deimos.config.schema.path,
+          schema_store: @schema_store
+        )
+      end
     end
   end
 end

data/lib/deimos/schema_backends/avro_schema_registry.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'avro_base'
+require 'avro_turf/messaging'
 
 module Deimos
   module SchemaBackends
@@ -8,27 +9,27 @@ module Deimos
     class AvroSchemaRegistry < AvroBase
       # @override
       def decode_payload(payload, schema:)
-        schema_registry.decode(payload.to_s)
+        avro_turf_messaging.decode(payload.to_s, schema_name: schema)
       end
 
       # @override
-      def encode_payload(payload, schema: nil, subject: nil)
-        schema_registry.encode(payload, subject: subject || schema, schema_name: "#{@namespace}.#{schema}")
+      def encode_payload(payload, schema: nil, topic: nil)
+        avro_turf_messaging.encode(payload, schema_name: schema, subject: topic || schema)
       end
 
     private
 
-      # @return [SchemaRegistry::Client]
-      def schema_registry
-        @schema_registry ||= SchemaRegistry::Client.new(
-          registry_url: Deimos.config.schema.registry_url,
-          logger: Karafka.logger,
-          user: Deimos.config.schema.user,
-          password: Deimos.config.schema.password,
-          schema_type: SchemaRegistry::Schema::Avro
-        )
-        SchemaRegistry.avro_schema_path = Deimos.config.schema.path
-        @schema_registry
+      # @return [AvroTurf::Messaging]
+      def avro_turf_messaging
+         @avro_turf_messaging ||= AvroTurf::Messaging.new(
+           schema_store: @schema_store,
+           registry_url: Deimos.config.schema.registry_url,
+           schemas_path: Deimos.config.schema.path,
+           user: Deimos.config.schema.user,
+           password: Deimos.config.schema.password,
+           namespace: @namespace,
+           logger: Karafka.logger
+         )
       end
     end
   end

data/lib/deimos/schema_backends/avro_validation.rb

@@ -13,7 +13,7 @@ module Deimos
       end
 
       # @override
-      def encode_payload(payload, schema: nil, subject: nil)
+      def encode_payload(payload, schema: nil, topic: nil)
         payload.to_h.with_indifferent_access.to_json
       end
     end

data/lib/deimos/schema_backends/base.rb

@@ -56,10 +56,9 @@ module Deimos
       # @param schema [String,Symbol]
       # @param topic [String]
       # @return [String]
-      def encode(payload, schema: nil, topic: nil, is_key: false)
+      def encode(payload, schema: nil, topic: nil)
         validate(payload, schema: schema || @schema)
-        subject = is_key ? "#{topic}-key" : "#{topic}-value"
-        encode_payload(payload, schema: schema || @schema, subject: subject)
+        encode_payload(payload, schema: schema || @schema, topic: topic)
       end
 
       # Decode a payload with a schema. Public method.
@@ -115,9 +114,9 @@ module Deimos
       # Encode a payload. To be defined by subclass.
       # @param payload [Hash]
       # @param schema [String,Symbol]
-      # @param subject [String]
+      # @param topic [String]
       # @return [String]
-      def encode_payload(_payload, schema:, subject: nil)
+      def encode_payload(_payload, schema:, topic: nil)
         raise MissingImplementationError
       end
 

data/lib/deimos/schema_backends/mock.rb

@@ -15,7 +15,7 @@ module Deimos
       end
 
       # @override
-      def encode_payload(payload, schema:, subject: nil)
+      def encode_payload(payload, schema:, topic: nil)
         payload.is_a?(String) ? 'payload-encoded' : payload.map { |k, v| [k, "encoded-#{v}"] }.to_json
       end
 

data/lib/deimos/schema_backends/plain.rb

@@ -15,7 +15,7 @@ module Deimos
       end
 
       # @override
-      def encode_payload(payload, schema:, subject: nil)
+      def encode_payload(payload, schema:, topic: nil)
         payload.to_s
       end
 

data/lib/deimos/schema_backends/proto_base.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'base'
-require 'schema_registry_client'
+require 'proto_turf'
 
 module Deimos
   module SchemaBackends
@@ -24,51 +24,26 @@ module Deimos
         float: :float,
         message: :record
       }.freeze
-
       def proto_schema(schema=@schema)
-        proto = Google::Protobuf::DescriptorPool.generated_pool.lookup(schema)
-        if proto.nil?
-          raise "Could not find Protobuf schema '#{schema}'."
-        end
-
-        proto
+        Google::Protobuf::DescriptorPool.generated_pool.lookup(schema)
       end
 
       # @override
       def encode_key(key_id, key, topic: nil)
-        if key.respond_to?(:to_h)
-          hash = if key_id
-                   key_id.to_s.split('.')[...-1].each do |k|
-                     key = key.with_indifferent_access[k]
-                   end
-                   key.to_h.with_indifferent_access.slice(key_id.split('.').last)
-                 else
-                   key.to_h.sort.to_h
-                 end
-          self.encode_proto_key(hash, topic: topic, field: key_id)
-        elsif key_id
-          hash = { key_id.to_s.split('.').last => key }
-          self.encode_proto_key(hash, topic: topic, field: key_id)
+        if key.is_a?(Hash)
+          key_id ? key.with_indifferent_access[key_id].to_s : key.sort.to_h.to_json
         else
           key.to_s
         end
       end
 
-      # @param hash [Hash]
-      # @return [String]
-      def encode_proto_key(hash, topic: nil)
-        hash.sort.to_h.to_json
-      end
-
-      def decode_proto_key(payload)
-        JSON.parse(payload)
-      rescue StandardError
-        payload
-      end
-
       # @override
       def decode_key(payload, key_id)
-        val = decode_proto_key(payload)
+        val = begin
+                JSON.parse(payload)
+        rescue StandardError
+                payload
+        end
         key_id ? val[key_id.to_s] : val
       end
 
@@ -110,8 +85,8 @@ module Deimos
         :mock
       end
 
-      def supports_key_schemas?
-        false
+      def generate_key_schema(_field_name)
+        raise 'Protobuf cannot generate key schemas! Please use field_config :plain'
       end
 
     end

data/lib/deimos/schema_backends/proto_local.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'proto_base'
-require 'schema_registry_client'
+require 'proto_turf'
 
 module Deimos
   module SchemaBackends
@@ -14,14 +14,14 @@ module Deimos
       end
 
       # @override
-      def encode_payload(payload, schema: nil, subject: nil)
+      def encode_payload(payload, schema: nil, topic: nil)
         msg = payload.is_a?(Hash) ? proto_schema.msgclass.new(**payload) : payload
         proto_schema.msgclass.encode(msg)
       end
 
-      # @return [SchemaRegistry::Client]
-      def self.schema_registry
-        @schema_registry ||= SchemaRegistry::Client.new(
+      # @return [ProtoTurf]
+      def self.proto_turf
+        @proto_turf ||= ProtoTurf.new(
           registry_url: Deimos.config.schema.registry_url,
           logger: Karafka.logger
         )

data/lib/deimos/schema_backends/proto_schema_registry.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'proto_base'
-require 'schema_registry_client'
+require 'proto_turf'
 
 module Deimos
   module SchemaBackends
@@ -10,47 +10,22 @@ module Deimos
 
       # @override
       def decode_payload(payload, schema:)
-        self.class.schema_registry.decode(payload)
+        self.class.proto_turf.decode(payload)
       end
 
       # @override
-      def encode_payload(payload, schema: nil, subject: nil)
+      def encode_payload(payload, schema: nil, topic: nil)
         msg = payload.is_a?(Hash) ? proto_schema.msgclass.new(**payload) : payload
-        encoder = subject&.ends_with?('-key') ? self.class.key_schema_registry : self.class.schema_registry
-        encoder.encode(msg, subject: subject)
+        self.class.proto_turf.encode(msg, subject: topic)
       end
 
-      # @override
-      def encode_proto_key(key, topic: nil, field: nil)
-        schema_text = SchemaRegistry::Output::JsonSchema.output(proto_schema.to_proto, path: field)
-        self.class.key_schema_registry.encode(key, subject: "#{topic}-key", schema_text: schema_text)
-      end
-
-      # @override
-      def decode_proto_key(payload)
-        self.class.key_schema_registry.decode(payload)
-      end
-
-      # @return [SchemaRegistry::Client]
-      def self.schema_registry
-        @schema_registry ||= SchemaRegistry::Client.new(
+      # @return [ProtoTurf]
+      def self.proto_turf
+        @proto_turf ||= ProtoTurf.new(
           registry_url: Deimos.config.schema.registry_url,
-          user: Deimos.config.schema.user,
-          password: Deimos.config.schema.password,
           logger: Karafka.logger
         )
       end
-
-      def self.key_schema_registry
-        @key_schema_registry ||= SchemaRegistry::Client.new(
-          registry_url: Deimos.config.schema.registry_url,
-          user: Deimos.config.schema.user,
-          password: Deimos.config.schema.password,
-          logger: Karafka.logger,
-          schema_type: SchemaRegistry::Schema::ProtoJsonSchema
-        )
-      end
-
     end
   end
 end

data/lib/deimos/test_helpers.rb

@@ -16,6 +16,14 @@ module Deimos
     def self.included(base)
       super
       base.include Karafka::Testing::RSpec::Helpers
+
+      # Ensure that we only use Karafka.producer, not the producers we set up for multi-broker
+      # configs. Only Karafka.producer works with Karafka test helpers.
+      RSpec.configure do |config|
+        config.before(:each) do
+          allow(Deimos).to receive(:producer_for).and_return(Karafka.producer)
+        end
+      end
     end
 
     # @return [Array<Hash>]

data/lib/deimos/transcoder.rb

@@ -36,7 +36,7 @@ module Deimos
       if self.key_field
         self.backend.encode_key(self.key_field, key, topic: @topic)
       else
-        self.backend.encode(key, topic: @topic, is_key: true)
+        self.backend.encode(key, topic: @topic)
       end
     end
 

data/lib/deimos/utils/outbox_producer.rb

@@ -205,9 +205,9 @@ module Deimos
         batch_size = batch.size
         current_index = 0
 
-        batch[current_index..].in_groups_of(batch_size, false).each do |group|
+        batch[current_index..-1].in_groups_of(batch_size, false).each do |group|
           @logger.debug("Publishing #{group.size} messages to #{@current_topic}")
-          Karafka.producer.produce_many_sync(group)
+          Deimos.producer_for(@current_topic).produce_many_sync(group)
           current_index += group.size
           @logger.info("Sent #{group.size} messages to #{@current_topic}")
         end

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '2.3.0-beta4'
+  VERSION = '2.3.0'
 end

data/lib/deimos.rb

@@ -142,23 +142,34 @@ module Deimos
       signal_handler.run!
     end
 
-    def setup_karafka
-      Karafka.producer.middleware.append(Deimos::ProducerMiddleware)
-      # for multiple setup calls
-      Karafka.producer.config.kafka =
-        Karafka::Setup::AttributesMap.producer(Karafka::Setup::Config.config.kafka.dup)
-      EVENT_TYPES.each { |type| Karafka.monitor.notifications_bus.register_event(type) }
-
-      Karafka.producer.monitor.subscribe(ProducerMetricsListener.new)
+    def setup_producers
+      @producers = {}
+      producers_by_broker = {}
+      Deimos.karafka_configs.each do |topic|
+        broker = topic.kafka[:'bootstrap.servers']
+        producers_by_broker[broker] ||= ::WaterDrop::Producer.new do |p_config|
+          config_hash = Karafka::Setup::Config.config.kafka.merge(topic.kafka)
+          p_config.kafka = Karafka::Setup::AttributesMap.producer(config_hash)
+        end
+        @producers[topic.name] = producers_by_broker[broker]
+      end
+    end
 
-      Karafka.producer.monitor.subscribe('error.occurred') do |event|
-        if event.payload.key?(:messages)
-          topic = event[:messages].first[:topic]
-          config = Deimos.karafka_config_for(topic: topic)
-          message = Deimos::Logging.messages_log_text(config&.payload_log, event[:messages])
-          Karafka.logger.error("Error producing messages: #{event[:error].message} #{message.to_json}")
+    def setup_karafka
+      setup_producers
+      waterdrop_producers.each do |producer|
+        producer.middleware.append(Deimos::ProducerMiddleware)
+        producer.monitor.subscribe(ProducerMetricsListener.new)
+        producer.monitor.subscribe('error.occurred') do |event|
+          if event.payload.key?(:messages)
+            topic = event[:messages].first[:topic]
+            config = Deimos.karafka_config_for(topic: topic)
+            message = Deimos::Logging.messages_log_text(config&.payload_log, event[:messages])
+            Karafka.logger.error("Error producing messages: #{event[:error].message} #{message.to_json}")
+          end
         end
       end
+      EVENT_TYPES.each { |type| Karafka.monitor.notifications_bus.register_event(type) }
     end
 
     # @return [Array<Karafka::Routing::Topic]
@@ -176,7 +187,16 @@ module Deimos
       end
     end
 
-    # @param handler_class [Class]
+    # @return [Array<::WaterDrop::Producer>]
+    def waterdrop_producers
+      (@producers.values + [Karafka.producer]).uniq
+    end
+
+    # @param topic [String]
+    def producer_for(topic)
+      @producers[topic] || Karafka.producer
+    end
+
     # @return [String,nil]
     def topic_for_consumer(handler_class)
       Deimos.karafka_configs.each do |topic|

data/lib/generators/deimos/active_record_generator.rb

@@ -48,7 +48,7 @@ module Deimos
         # @return [String]
         def schema
           last_dot = self.full_schema.rindex('.')
-          self.full_schema[(last_dot + 1)..]
+          self.full_schema[(last_dot + 1)..-1]
         end
 
         # @return [String]