deimos-ruby 2.3.0.pre.beta4 → 2.4.0.pre.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f8e48e6252ea21b5af3a7f2ec921f7a82db95e6286eaf52ac15f94ad8e1043d6
-  data.tar.gz: 99bf9b5ecb064e644897e1ef2177704d460b9531a9fc51e551b8462f22fe1943
+  metadata.gz: 597f226ba74e9b5acce2906ee0fea563a3ff2b7a5b47bef0d4e66d840410ed42
+  data.tar.gz: 876b9a70bc598bb99084a0bad47d6285c728efa54d98346761346c4313dda968
 SHA512:
-  metadata.gz: 7cf3edfdf000781943adb58eddc2e73e2fa2c8b7cede12cfbebc1407235af33459b44402602a9e26534187f81df9ed67a38029ffc917d0c94e4e0a8924645929
-  data.tar.gz: da90d82a5919b02893b7877e668a287f3deb64c766f6b5a5543d6217b2782c14ab4b0c87f87f0dcad8e537905b5b480d53f756530eb1b4c6e187c0569f13b0f0
+  metadata.gz: 58d7dfcf2cfbbbda7c4ce3459931a89ba6bf6261f6bd4a2cdf1bc2de640200fdc2a9a36f0af414bbcfef3de45fe8b3053795cdd5b139cf10a747a16a82989ccc
+  data.tar.gz: 35ab518ca7bcbc48d2810ec67ce30a9331816fdc08caa52c602406f6b79346f7bf806af3adc50b427e5d412ce478c55f5e29b49f0bc61f2c88dc1ca81b11f993

data/CHANGELOG.md

@@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Major change: Switch from using `avro_turf` and `proto_turf` to use `schema_registry_client`, which handles both Avro and Protobuf.
 
+# 2.3.0 - 2026-01-13
+
+- Feature: Support broker setting per topic in producer configs.
+
 # 2.2.2 - 2025-11-7
 
 - Fix: Calling `producer_classes` on a topic config could make all topics disappear. :(

data/README.md

@@ -139,6 +139,12 @@ end
 
 Note that if you are using Protobuf, you need to pass a Protobuf message object as the payload - you can't use a bare hash.
 
+## Multiple clusters
+
+If you have topics that are being routed to different clusters via Karafka configs, you can continue to make use of Deimos producers without having to instantiate the producer itself. Instead of calling `MyProducer.produce(message)`, you can call `Deimos.producer_for('MyTopic').produce(message)`.
+
+Deimos will keep around one producer per broker server (i.e. `bootstrap.servers` config) that it sees on startup.
+
 ## Auto-added Fields
 
 If your schema has a field called `message_id`, and the payload you give

data/lib/deimos/backends/kafka.rb

@@ -6,7 +6,7 @@ module Deimos
     class Kafka < Base
       # :nodoc:
       def self.execute(producer_class:, messages:)
-        Karafka.producer.produce_many_sync(messages)
+        Deimos.producer_for(producer_class.topic).produce_many_sync(messages)
       end
     end
   end

data/lib/deimos/backends/kafka_async.rb

@@ -4,9 +4,10 @@ module Deimos
   module Backends
     # Backend which produces to Kafka via an async producer.
     class KafkaAsync < Base
+
       # :nodoc:
       def self.execute(producer_class:, messages:)
-        Karafka.producer.produce_many_async(messages)
+        Deimos.producer_for(producer_class.topic).produce_many_async(messages)
       end
     end
   end

data/lib/deimos/metrics/datadog.rb

@@ -58,7 +58,9 @@ module Deimos
           end
         end
         Karafka::Setup::Config.setup if Karafka.producer.nil?
-        Karafka.producer.monitor.subscribe(waterdrop_listener)
+        Deimos.waterdrop_producers.each do |producer|
+          producer.monitor.subscribe(waterdrop_listener)
+        end
       end
 
       # :nodoc:

data/lib/deimos/test_helpers.rb

@@ -16,6 +16,14 @@ module Deimos
     def self.included(base)
       super
       base.include Karafka::Testing::RSpec::Helpers
+
+      # Ensure that we only use Karafka.producer, not the producers we set up for multi-broker
+      # configs. Only Karafka.producer works with Karafka test helpers.
+      RSpec.configure do |config|
+        config.before(:each) do
+          allow(Deimos).to receive(:producer_for).and_return(Karafka.producer)
+        end
+      end
     end
 
     # @return [Array<Hash>]

data/lib/deimos/utils/outbox_producer.rb

@@ -207,7 +207,7 @@ module Deimos
 
         batch[current_index..].in_groups_of(batch_size, false).each do |group|
           @logger.debug("Publishing #{group.size} messages to #{@current_topic}")
-          Karafka.producer.produce_many_sync(group)
+          Deimos.producer_for(@current_topic).produce_many_sync(group)
           current_index += group.size
           @logger.info("Sent #{group.size} messages to #{@current_topic}")
         end

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '2.3.0-beta4'
+  VERSION = '2.4.0-beta1'
 end

data/lib/deimos.rb

@@ -142,23 +142,34 @@ module Deimos
       signal_handler.run!
     end
 
-    def setup_karafka
-      Karafka.producer.middleware.append(Deimos::ProducerMiddleware)
-      # for multiple setup calls
-      Karafka.producer.config.kafka =
-        Karafka::Setup::AttributesMap.producer(Karafka::Setup::Config.config.kafka.dup)
-      EVENT_TYPES.each { |type| Karafka.monitor.notifications_bus.register_event(type) }
-
-      Karafka.producer.monitor.subscribe(ProducerMetricsListener.new)
+    def setup_producers
+      @producers = {}
+      producers_by_broker = {}
+      Deimos.karafka_configs.each do |topic|
+        broker = topic.kafka[:'bootstrap.servers']
+        producers_by_broker[broker] ||= ::WaterDrop::Producer.new do |p_config|
+          config_hash = Karafka::Setup::Config.config.kafka.merge(topic.kafka)
+          p_config.kafka = Karafka::Setup::AttributesMap.producer(config_hash)
+        end
+        @producers[topic.name] = producers_by_broker[broker]
+      end
+    end
 
-      Karafka.producer.monitor.subscribe('error.occurred') do |event|
-        if event.payload.key?(:messages)
-          topic = event[:messages].first[:topic]
-          config = Deimos.karafka_config_for(topic: topic)
-          message = Deimos::Logging.messages_log_text(config&.payload_log, event[:messages])
-          Karafka.logger.error("Error producing messages: #{event[:error].message} #{message.to_json}")
+    def setup_karafka
+      setup_producers
+      waterdrop_producers.each do |producer|
+        producer.middleware.append(Deimos::ProducerMiddleware)
+        producer.monitor.subscribe(ProducerMetricsListener.new)
+        producer.monitor.subscribe('error.occurred') do |event|
+          if event.payload.key?(:messages)
+            topic = event[:messages].first[:topic]
+            config = Deimos.karafka_config_for(topic: topic)
+            message = Deimos::Logging.messages_log_text(config&.payload_log, event[:messages])
+            Karafka.logger.error("Error producing messages: #{event[:error].message} #{message.to_json}")
+          end
         end
       end
+      EVENT_TYPES.each { |type| Karafka.monitor.notifications_bus.register_event(type) }
     end
 
     # @return [Array<Karafka::Routing::Topic]
@@ -176,7 +187,16 @@ module Deimos
       end
     end
 
-    # @param handler_class [Class]
+    # @return [Array<::WaterDrop::Producer>]
+    def waterdrop_producers
+      (@producers.values + [Karafka.producer]).uniq
+    end
+
+    # @param topic [String]
+    def producer_for(topic)
+      @producers[topic] || Karafka.producer
+    end
+
     # @return [String,nil]
     def topic_for_consumer(handler_class)
       Deimos.karafka_configs.each do |topic|

data/spec/deimos_spec.rb

@@ -50,4 +50,36 @@ describe Deimos do
     end
   end
 
+  specify '#producer_for' do
+    allow(described_class).to receive(:producer_for).and_call_original
+    Karafka::App.routes.redraw do
+      topic 'main-broker' do
+        active false
+        kafka({
+                'bootstrap.servers': 'broker1:9092'
+              })
+      end
+      topic 'main-broker2' do
+        active false
+        kafka({
+                'bootstrap.servers': 'broker1:9092'
+              })
+      end
+      topic 'other-broker' do
+        active false
+        kafka({
+                'bootstrap.servers': 'broker2:9092'
+              })
+      end
+    end
+    described_class.setup_producers
+
+    producer1 = described_class.producer_for('main-broker')
+    producer2 = described_class.producer_for('main-broker2')
+    producer3 = described_class.producer_for('other-broker')
+    expect(producer1).to eq(producer2)
+    expect(producer1.config.kafka[:'bootstrap.servers']).to eq('broker1:9092')
+    expect(producer3.config.kafka[:'bootstrap.servers']).to eq('broker2:9092')
+  end
+
 end

data/spec/generators/schema_class_generator_spec.rb

@@ -37,10 +37,9 @@ RSpec.describe Deimos::Generators::SchemaClassGenerator do
 
   context 'with a Consumers Schema' do
     before(:each) do
-      Deimos.configure do
-        consumer do
-          class_name 'ConsumerTest::MyConsumer'
-          topic 'MyTopic'
+      Karafka::App.routes.redraw do
+        topic 'MyTopic' do
+          consumer ConsumerTest::MyConsumer
           schema 'Generated'
           namespace 'com.my-namespace'
           key_config field: :a_string

data/spec/snapshots/consumers-no-nest.snap

@@ -217,6 +217,13 @@ module Schemas
       'com.my-namespace'
     end
 
+    def self.tombstone(key)
+      record = self.allocate
+      record.tombstone_key = key
+      record.a_string = key
+      record
+    end
+
     # @override
     def as_json(_opts={})
       {
@@ -437,13 +444,6 @@ module Schemas
       'com.my-namespace'
     end
 
-    def self.tombstone(key)
-      record = self.allocate
-      record.tombstone_key = key
-      record.test_id = key
-      record
-    end
-
     # @override
     def as_json(_opts={})
       {

data/spec/snapshots/consumers.snap

@@ -177,6 +177,13 @@ module Schemas
       'com.my-namespace'
     end
 
+    def self.tombstone(key)
+      record = self.allocate
+      record.tombstone_key = key
+      record.a_string = key
+      record
+    end
+
     # @override
     def as_json(_opts={})
       {
@@ -391,13 +398,6 @@ module Schemas
       'com.my-namespace'
     end
 
-    def self.tombstone(key)
-      record = self.allocate
-      record.tombstone_key = key
-      record.test_id = key
-      record
-    end
-
     # @override
     def as_json(_opts={})
       {

data/spec/snapshots/consumers_and_producers-no-nest.snap

@@ -217,6 +217,13 @@ module Schemas
       'com.my-namespace'
     end
 
+    def self.tombstone(key)
+      record = self.allocate
+      record.tombstone_key = key
+      record.a_string = key
+      record
+    end
+
     # @override
     def as_json(_opts={})
       {
@@ -437,13 +444,6 @@ module Schemas
       'com.my-namespace'
     end
 
-    def self.tombstone(key)
-      record = self.allocate
-      record.tombstone_key = key
-      record.test_id = key
-      record
-    end
-
     # @override
     def as_json(_opts={})
       {

data/spec/snapshots/consumers_and_producers.snap

@@ -177,6 +177,13 @@ module Schemas
       'com.my-namespace'
     end
 
+    def self.tombstone(key)
+      record = self.allocate
+      record.tombstone_key = key
+      record.a_string = key
+      record
+    end
+
     # @override
     def as_json(_opts={})
       {
@@ -391,13 +398,6 @@ module Schemas
       'com.my-namespace'
     end
 
-    def self.tombstone(key)
-      record = self.allocate
-      record.tombstone_key = key
-      record.test_id = key
-      record
-    end
-
     # @override
     def as_json(_opts={})
       {

data/spec/snapshots/consumers_circular-no-nest.snap

@@ -217,6 +217,13 @@ module Schemas
       'com.my-namespace'
     end
 
+    def self.tombstone(key)
+      record = self.allocate
+      record.tombstone_key = key
+      record.a_string = key
+      record
+    end
+
     # @override
     def as_json(_opts={})
       {
@@ -437,13 +444,6 @@ module Schemas
       'com.my-namespace'
     end
 
-    def self.tombstone(key)
-      record = self.allocate
-      record.tombstone_key = key
-      record.test_id = key
-      record
-    end
-
     # @override
     def as_json(_opts={})
       {

data/spec/snapshots/consumers_circular.snap

@@ -177,6 +177,13 @@ module Schemas
       'com.my-namespace'
     end
 
+    def self.tombstone(key)
+      record = self.allocate
+      record.tombstone_key = key
+      record.a_string = key
+      record
+    end
+
     # @override
     def as_json(_opts={})
       {
@@ -391,13 +398,6 @@ module Schemas
       'com.my-namespace'
     end
 
-    def self.tombstone(key)
-      record = self.allocate
-      record.tombstone_key = key
-      record.test_id = key
-      record
-    end
-
     # @override
     def as_json(_opts={})
       {

data/spec/snapshots/consumers_complex_types-no-nest.snap

@@ -217,6 +217,13 @@ module Schemas
       'com.my-namespace'
     end
 
+    def self.tombstone(key)
+      record = self.allocate
+      record.tombstone_key = key
+      record.a_string = key
+      record
+    end
+
     # @override
     def as_json(_opts={})
       {
@@ -437,13 +444,6 @@ module Schemas
       'com.my-namespace'
     end
 
-    def self.tombstone(key)
-      record = self.allocate
-      record.tombstone_key = key
-      record.test_id = key
-      record
-    end
-
     # @override
     def as_json(_opts={})
       {

data/spec/snapshots/consumers_complex_types.snap

@@ -177,6 +177,13 @@ module Schemas
       'com.my-namespace'
     end
 
+    def self.tombstone(key)
+      record = self.allocate
+      record.tombstone_key = key
+      record.a_string = key
+      record
+    end
+
     # @override
     def as_json(_opts={})
       {
@@ -391,13 +398,6 @@ module Schemas
       'com.my-namespace'
     end
 
-    def self.tombstone(key)
-      record = self.allocate
-      record.tombstone_key = key
-      record.test_id = key
-      record
-    end
-
     # @override
     def as_json(_opts={})
       {

data/spec/snapshots/consumers_nested-no-nest.snap

@@ -217,6 +217,13 @@ module Schemas
       'com.my-namespace'
     end
 
+    def self.tombstone(key)
+      record = self.allocate
+      record.tombstone_key = key
+      record.a_string = key
+      record
+    end
+
     # @override
     def as_json(_opts={})
       {
@@ -437,13 +444,6 @@ module Schemas
       'com.my-namespace'
     end
 
-    def self.tombstone(key)
-      record = self.allocate
-      record.tombstone_key = key
-      record.test_id = key
-      record
-    end
-
     # @override
     def as_json(_opts={})
       {

data/spec/snapshots/consumers_nested.snap

@@ -177,6 +177,13 @@ module Schemas
       'com.my-namespace'
     end
 
+    def self.tombstone(key)
+      record = self.allocate
+      record.tombstone_key = key
+      record.a_string = key
+      record
+    end
+
     # @override
     def as_json(_opts={})
       {
@@ -391,13 +398,6 @@ module Schemas
       'com.my-namespace'
     end
 
-    def self.tombstone(key)
-      record = self.allocate
-      record.tombstone_key = key
-      record.test_id = key
-      record
-    end
-
     # @override
     def as_json(_opts={})
       {

data/spec/snapshots/namespace_folders.snap

@@ -137,6 +137,13 @@ module Schemas; module Com; module MyNamespace
       'com.my-namespace'
     end
 
+    def self.tombstone(key)
+      record = self.allocate
+      record.tombstone_key = key
+      record.a_string = key
+      record
+    end
+
     # @override
     def as_json(_opts={})
       {
@@ -266,13 +273,6 @@ module Schemas; module Com; module MyNamespace
       'com.my-namespace'
     end
 
-    def self.tombstone(key)
-      record = self.allocate
-      record.tombstone_key = key
-      record.test_id = key
-      record
-    end
-
     # @override
     def as_json(_opts={})
       {

data/spec/snapshots/namespace_map.snap

@@ -137,6 +137,13 @@ module Schemas; module MyNamespace
       'com.my-namespace'
     end
 
+    def self.tombstone(key)
+      record = self.allocate
+      record.tombstone_key = key
+      record.a_string = key
+      record
+    end
+
     # @override
     def as_json(_opts={})
       {
@@ -311,13 +318,6 @@ module Schemas; module MyNamespace
       'com.my-namespace'
     end
 
-    def self.tombstone(key)
-      record = self.allocate
-      record.tombstone_key = key
-      record.test_id = key
-      record
-    end
-
     # @override
     def as_json(_opts={})
       {

data/spec/snapshots/producers_with_key-no-nest.snap

@@ -217,6 +217,13 @@ module Schemas
       'com.my-namespace'
     end
 
+    def self.tombstone(key)
+      record = self.allocate
+      record.tombstone_key = key
+      record.a_string = key
+      record
+    end
+
     # @override
     def as_json(_opts={})
       {
@@ -437,13 +444,6 @@ module Schemas
       'com.my-namespace'
     end
 
-    def self.tombstone(key)
-      record = self.allocate
-      record.tombstone_key = key
-      record.test_id = key
-      record
-    end
-
     # @override
     def as_json(_opts={})
       {

data/spec/snapshots/producers_with_key.snap

@@ -177,6 +177,13 @@ module Schemas
       'com.my-namespace'
     end
 
+    def self.tombstone(key)
+      record = self.allocate
+      record.tombstone_key = key
+      record.a_string = key
+      record
+    end
+
     # @override
     def as_json(_opts={})
       {
@@ -391,13 +398,6 @@ module Schemas
       'com.my-namespace'
     end
 
-    def self.tombstone(key)
-      record = self.allocate
-      record.tombstone_key = key
-      record.test_id = key
-      record
-    end
-
     # @override
     def as_json(_opts={})
       {

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 2.3.0.pre.beta4
+  version: 2.4.0.pre.beta1
 platform: ruby
 authors:
 - Daniel Orner