deimos-ruby 2.2.2 → 2.2.3.pre.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ca14735549cff155e75b761beafaf9cc663da80a5b0cc5db9a7ad67934cb7276
-  data.tar.gz: 302d52cf0d80701337ce576b55ff201ebce7968b37194070bdfe5f269901f51b
+  metadata.gz: bf4ddbfc7afab6d397364f0108d2eb845e2b646e8615783d2376b92edc34d380
+  data.tar.gz: 5f3a404983d2dfa8f0c5324d8ba4d728b1f976940b9242756d81fcc53a6a6cab
 SHA512:
-  metadata.gz: b3044b241cb7dab960e1e394d1438c39fa0fda920db8ce57375ac9e864bd3698182b400886efbdd7057e8232dfbfafa02a97d8ab9313f1a6edf502360dbb5c14
-  data.tar.gz: 8b1c0b604484db8425fd3f5dc57fc136d46ef2bbdfbc11027638f89b10db90088a0496ff00ea0022cd897b55fca16afe8daa86b5ca13f777aa3a2509b13fa5d5
+  metadata.gz: db87d1ed05ccae6430adfd1749902b319e53b7039f29512fb8335cceef4395e4ad26ad0a6437b87ec622f301923a3b27f9855471f3b8a1b5c5a86e4991d51dc0
+  data.tar.gz: 996d61652323b29bf40152ccb9fb71c74b050948bc1363b0d1a26191e8595240a20344b6256a0cfa6821ae31c7f1ee64501b54bfb071564bb06abc81e831a790

data/.gitignore

@@ -41,3 +41,4 @@ test.sqlite3
 
 .idea
 /.gem_rbs_collection/
+.claude

data/CHANGELOG.md

@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+- Feature: Support broker setting per topic in producer configs.
+
 # 2.2.2 - 2025-11-7
 
 - Fix: Calling `producer_classes` on a topic config could make all topics disappear. :(

data/lib/deimos/backends/kafka.rb

@@ -6,7 +6,7 @@ module Deimos
     class Kafka < Base
       # :nodoc:
       def self.execute(producer_class:, messages:)
-        Karafka.producer.produce_many_sync(messages)
+        Deimos.producer_for(producer_class.topic).produce_many_sync(messages)
       end
     end
   end

data/lib/deimos/backends/kafka_async.rb

@@ -4,9 +4,10 @@ module Deimos
   module Backends
     # Backend which produces to Kafka via an async producer.
     class KafkaAsync < Base
+
       # :nodoc:
       def self.execute(producer_class:, messages:)
-        Karafka.producer.produce_many_async(messages)
+        Deimos.producer_for(producer_class.topic).produce_many_async(messages)
       end
     end
   end

data/lib/deimos/metrics/datadog.rb

@@ -58,7 +58,9 @@ module Deimos
           end
         end
         Karafka::Setup::Config.setup if Karafka.producer.nil?
-        Karafka.producer.monitor.subscribe(waterdrop_listener)
+        Deimos.waterdrop_producers.each do |producer|
+          producer.monitor.subscribe(waterdrop_listener)
+        end
       end
 
       # :nodoc:

data/lib/deimos/test_helpers.rb

@@ -16,6 +16,11 @@ module Deimos
     def self.included(base)
       super
       base.include Karafka::Testing::RSpec::Helpers
+      RSpec.configure do |config|
+        config.before(:each) do
+          allow(Deimos).to receive(:producer_for).and_return(Karafka.producer)
+        end
+      end
     end
 
     # @return [Array<Hash>]

data/lib/deimos/utils/outbox_producer.rb

@@ -207,7 +207,7 @@ module Deimos
 
         batch[current_index..-1].in_groups_of(batch_size, false).each do |group|
           @logger.debug("Publishing #{group.size} messages to #{@current_topic}")
-          Karafka.producer.produce_many_sync(group)
+          Deimos.producer_for(@current_topic).produce_many_sync(group)
           current_index += group.size
           @logger.info("Sent #{group.size} messages to #{@current_topic}")
         end

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '2.2.2'
+  VERSION = '2.2.3-beta2'
 end

data/lib/deimos.rb

@@ -142,23 +142,34 @@ module Deimos
       signal_handler.run!
     end
 
-    def setup_karafka
-      Karafka.producer.middleware.append(Deimos::ProducerMiddleware)
-      # for multiple setup calls
-      Karafka.producer.config.kafka =
-        Karafka::Setup::AttributesMap.producer(Karafka::Setup::Config.config.kafka.dup)
-      EVENT_TYPES.each { |type| Karafka.monitor.notifications_bus.register_event(type) }
-
-      Karafka.producer.monitor.subscribe(ProducerMetricsListener.new)
+    def setup_producers
+      @producers = {}
+      producers_by_broker = {}
+      Deimos.karafka_configs.each do |topic|
+        broker = topic.kafka[:'bootstrap.servers']
+        producers_by_broker[broker] ||= ::WaterDrop::Producer.new do |p_config|
+          config_hash = Karafka::Setup::Config.config.kafka.merge(topic.kafka)
+          p_config.kafka = Karafka::Setup::AttributesMap.producer(config_hash)
+        end
+        @producers[topic.name] = producers_by_broker[broker]
+      end
+    end
 
-      Karafka.producer.monitor.subscribe('error.occurred') do |event|
-        if event.payload.key?(:messages)
-          topic = event[:messages].first[:topic]
-          config = Deimos.karafka_config_for(topic: topic)
-          message = Deimos::Logging.messages_log_text(config&.payload_log, event[:messages])
-          Karafka.logger.error("Error producing messages: #{event[:error].message} #{message.to_json}")
+    def setup_karafka
+      setup_producers
+      waterdrop_producers.each do |producer|
+        producer.middleware.append(Deimos::ProducerMiddleware)
+        producer.monitor.subscribe(ProducerMetricsListener.new)
+        producer.monitor.subscribe('error.occurred') do |event|
+          if event.payload.key?(:messages)
+            topic = event[:messages].first[:topic]
+            config = Deimos.karafka_config_for(topic: topic)
+            message = Deimos::Logging.messages_log_text(config&.payload_log, event[:messages])
+            Karafka.logger.error("Error producing messages: #{event[:error].message} #{message.to_json}")
+          end
         end
       end
+      EVENT_TYPES.each { |type| Karafka.monitor.notifications_bus.register_event(type) }
     end
 
     # @return [Array<Karafka::Routing::Topic]
@@ -176,7 +187,16 @@ module Deimos
       end
     end
 
-    # @param handler_class [Class]
+    # @return [Array<::WaterDrop::Producer>]
+    def waterdrop_producers
+      (@producers.values + [Karafka.producer]).uniq
+    end
+
+    # @param topic [String]
+    def producer_for(topic)
+      @producers[topic] || Karafka.producer
+    end
+
     # @return [String,nil]
     def topic_for_consumer(handler_class)
       Deimos.karafka_configs.each do |topic|

data/spec/deimos_spec.rb

@@ -50,4 +50,36 @@ describe Deimos do
     end
   end
 
+  specify '#producer_for' do
+    allow(described_class).to receive(:producer_for).and_call_original
+    Karafka::App.routes.redraw do
+      topic 'main-broker' do
+        active false
+        kafka({
+                'bootstrap.servers': 'broker1:9092'
+              })
+      end
+      topic 'main-broker2' do
+        active false
+        kafka({
+                'bootstrap.servers': 'broker1:9092'
+              })
+      end
+      topic 'other-broker' do
+        active false
+        kafka({
+                'bootstrap.servers': 'broker2:9092'
+              })
+      end
+    end
+    described_class.setup_producers
+
+    producer1 = described_class.producer_for('main-broker')
+    producer2 = described_class.producer_for('main-broker2')
+    producer3 = described_class.producer_for('other-broker')
+    expect(producer1).to eq(producer2)
+    expect(producer1.config.kafka[:'bootstrap.servers']).to eq('broker1:9092')
+    expect(producer3.config.kafka[:'bootstrap.servers']).to eq('broker2:9092')
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 2.2.2
+  version: 2.2.3.pre.beta2
 platform: ruby
 authors:
 - Daniel Orner
@@ -698,7 +698,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.9
+rubygems_version: 4.0.3
 specification_version: 4
 summary: Kafka libraries for Ruby.
 test_files: []