deimos-ruby 2.3.0.pre.beta4 → 2.3.0

data/lib/generators/deimos/schema_class_generator.rb

@@ -13,7 +13,7 @@ module Deimos
       # @return [Array<Symbol>]
       SPECIAL_TYPES = %i(record enum).freeze
       # @return [String]
-      INITIALIZE_WHITESPACE = "\n#{' ' * 19}".freeze
+      INITIALIZE_WHITESPACE = "\n#{' ' * 19}"
       # @return [Array<String>]
       IGNORE_DEFAULTS = %w(message_id timestamp).freeze
       # @return [String]
@@ -192,7 +192,7 @@ module Deimos
 
       def generate_from_schema_files(found_schemas)
         path = Deimos.config.schema.path || Deimos.config.schema.paths[:avro].first
-        schema_store = SchemaRegistry::AvroSchemaStore.new(path: path)
+        schema_store = AvroTurf::MutableSchemaStore.new(path: path)
         schema_store.load_schemas!
         schema_store.schemas.values.sort_by { |s| "#{s.namespace}#{s.name}" }.each do |schema|
           name = "#{schema.namespace}.#{schema.name}"
@@ -265,7 +265,7 @@ module Deimos
         end
 
         result = "def initialize(_from_message: false, #{arguments.first}"
-        arguments[1..].each_with_index do |arg, _i|
+        arguments[1..-1].each_with_index do |arg, _i|
           result += ",#{INITIALIZE_WHITESPACE}#{arg}"
         end
         "#{result})"

data/lib/generators/deimos/v2_generator.rb

@@ -119,7 +119,7 @@ module Deimos
       end
 
       def consumer_configs
-        deimos_config.consumer_objects.group_by(&:group_id).to_h do |group_id, consumers|
+        deimos_config.consumer_objects.group_by(&:group_id).map { |group_id, consumers|
           [group_id, consumers.map do |consumer|
             kafka_configs = {}
             kafka_configs['auto.offset.reset'] = consumer.start_from_beginning ? 'earliest' : 'latest'
@@ -159,7 +159,7 @@ module Deimos
             configs[:each_message] = true unless consumer.delivery.to_s == 'inline_batch'
             configs
           end]
-        end
+        }.to_h
       end
 
       def producer_configs

data/spec/deimos_spec.rb

@@ -50,4 +50,36 @@ describe Deimos do
     end
   end
 
+  specify '#producer_for' do
+    allow(described_class).to receive(:producer_for).and_call_original
+    Karafka::App.routes.redraw do
+      topic 'main-broker' do
+        active false
+        kafka({
+                'bootstrap.servers': 'broker1:9092'
+              })
+      end
+      topic 'main-broker2' do
+        active false
+        kafka({
+                'bootstrap.servers': 'broker1:9092'
+              })
+      end
+      topic 'other-broker' do
+        active false
+        kafka({
+                'bootstrap.servers': 'broker2:9092'
+              })
+      end
+    end
+    described_class.setup_producers
+
+    producer1 = described_class.producer_for('main-broker')
+    producer2 = described_class.producer_for('main-broker2')
+    producer3 = described_class.producer_for('other-broker')
+    expect(producer1).to eq(producer2)
+    expect(producer1.config.kafka[:'bootstrap.servers']).to eq('broker1:9092')
+    expect(producer3.config.kafka[:'bootstrap.servers']).to eq('broker2:9092')
+  end
+
 end

data/spec/generators/schema_class_generator_spec.rb

@@ -20,7 +20,7 @@ end
 
 RSpec.describe Deimos::Generators::SchemaClassGenerator do
   let(:schema_class_path) { 'spec/app/lib/schema_classes' }
-  let(:files) { Dir["#{schema_class_path}/**/*.rb"].to_h { |f| [f, File.read(f)]} }
+  let(:files) { Dir["#{schema_class_path}/**/*.rb"].map { |f| [f, File.read(f)]}.to_h }
 
   before(:each) do
     Deimos.config.reset!
@@ -37,10 +37,9 @@ RSpec.describe Deimos::Generators::SchemaClassGenerator do
 
   context 'with a Consumers Schema' do
     before(:each) do
-      Deimos.configure do
-        consumer do
-          class_name 'ConsumerTest::MyConsumer'
-          topic 'MyTopic'
+      Karafka::App.routes.redraw do
+        topic 'MyTopic' do
+          consumer ConsumerTest::MyConsumer
           schema 'Generated'
           namespace 'com.my-namespace'
           key_config field: :a_string

data/spec/schema_backends/avro_base_shared.rb

@@ -66,7 +66,7 @@ RSpec.shared_examples_for('an Avro backend') do
     expect(backend.encode_key('test_id', 1, topic: 'topic')).to eq('itsme')
     expect(backend).to have_received(:encode).
       with({ 'test_id' => 1 }, { schema: 'MySchema_key', topic: 'topic' })
-    expect(backend.schema_store.find('com.my-namespace.MySchema_key').to_avro).
+    expect(backend.schema_store.find('MySchema_key', 'com.my-namespace').to_avro).
       to eq(
         'doc' => 'Key for com.my-namespace.MySchema - autogenerated by Deimos',
         'fields' => [

data/spec/schema_backends/avro_local_spec.rb

@@ -15,10 +15,17 @@ RSpec.describe Deimos::SchemaBackends::AvroLocal do
   it_should_behave_like 'an Avro backend'
 
   it 'should encode and decode correctly' do
+    avro_turf = instance_double(AvroTurf)
+    allow(avro_turf).to receive_messages(encode: 'encoded-payload', decode: payload)
+    allow(backend).to receive(:avro_turf).and_return(avro_turf)
     results = backend.encode(payload)
-    expect(results).to start_with("Obj\u0001\u0004\u0014avro.codec\bnull\u0016avro.schema\x94")
+    expect(results).to eq('encoded-payload')
     results = backend.decode(results)
     expect(results).to eq(payload)
+    expect(avro_turf).to have_received(:encode).
+      with(payload, schema_name: 'MySchema', namespace: 'com.my-namespace')
+    expect(avro_turf).to have_received(:decode).
+      with('encoded-payload', schema_name: 'MySchema', namespace: 'com.my-namespace')
   end
 
 end

data/spec/schema_backends/avro_schema_registry_spec.rb

@@ -15,17 +15,17 @@ RSpec.describe Deimos::SchemaBackends::AvroSchemaRegistry do
   it_should_behave_like 'an Avro backend'
 
   it 'should encode and decode correctly' do
-    schema_registry = instance_double(SchemaRegistry::Client)
-    allow(schema_registry).to receive_messages(encode: 'encoded-payload', decode: payload)
-    allow(backend).to receive(:schema_registry).and_return(schema_registry)
+    avro_turf = instance_double(AvroTurf::Messaging)
+    allow(avro_turf).to receive_messages(encode: 'encoded-payload', decode: payload)
+    allow(backend).to receive(:avro_turf_messaging).and_return(avro_turf)
     results = backend.encode(payload, topic: 'topic')
     expect(results).to eq('encoded-payload')
     results = backend.decode(results)
     expect(results).to eq(payload)
-    expect(schema_registry).to have_received(:encode).
-      with(payload, schema_name: 'com.my-namespace.MySchema', subject: 'topic-value')
-    expect(schema_registry).to have_received(:decode).
-      with('encoded-payload')
+    expect(avro_turf).to have_received(:encode).
+      with(payload, schema_name: 'MySchema', subject: 'topic')
+    expect(avro_turf).to have_received(:decode).
+      with('encoded-payload', schema_name: 'MySchema')
   end
 
 end

data/spec/schema_backends/base_spec.rb

@@ -6,13 +6,13 @@ describe Deimos::SchemaBackends::Base do
 
   it 'should validate on encode' do
     expect(backend).to receive(:validate).with(payload, schema: 'schema')
-    expect(backend).to receive(:encode_payload).with(payload, schema: 'schema', subject: 'topic-value')
+    expect(backend).to receive(:encode_payload).with(payload, schema: 'schema', topic: 'topic')
     backend.encode(payload, topic: 'topic')
   end
 
   it 'should validate and encode a passed schema' do
     expect(backend).to receive(:validate).with(payload, schema: 'schema2')
-    expect(backend).to receive(:encode_payload).with(payload, schema: 'schema2', subject: 'topic-value')
+    expect(backend).to receive(:encode_payload).with(payload, schema: 'schema2', topic: 'topic')
     backend.encode(payload, schema: 'schema2', topic: 'topic')
   end
 

data/spec/schema_backends/proto_schema_registry_spec.rb

@@ -2,7 +2,6 @@
 
 require 'deimos/schema_backends/proto_schema_registry'
 require_relative "#{__dir__}/../gen/sample/v1/sample_pb"
-require_relative "#{__dir__}/../gen/sample/v1/sample_key_pb"
 
 RSpec.describe Deimos::SchemaBackends::ProtoSchemaRegistry do
   let(:payload) do
@@ -17,234 +16,32 @@ RSpec.describe Deimos::SchemaBackends::ProtoSchemaRegistry do
       str_map: { 'foo' => 'bar' }
     )
   end
-  let(:schema_registry) { instance_double(SchemaRegistry::Client) }
-  let(:key_schema_registry) { instance_double(SchemaRegistry::Client) }
 
   let(:backend) { described_class.new(schema: 'sample.v1.SampleMessage') }
 
-  before(:each) do
-    allow(described_class).to receive_messages(schema_registry: schema_registry,
-                                               key_schema_registry: key_schema_registry)
+  specify('#encode_key') do
+    expect(backend.encode_key(nil, 789)).to eq('789')
+    expect(backend.encode_key(nil, 'string')).to eq('string')
+    expect(backend.encode_key(nil, { foo: 'bar' })).to eq('{"foo":"bar"}')
+    expect(backend.encode_key(:foo, 'bar')).to eq('bar')
   end
 
-  describe 'payload encoding and decoding' do
-    it 'should encode and decode payloads correctly' do
-      allow(schema_registry).to receive_messages(encode: 'encoded-payload', decode: payload)
-
-      # Encode
-      encoded = backend.encode(payload, topic: 'topic')
-      expect(encoded).to eq('encoded-payload')
-      expect(schema_registry).to have_received(:encode).with(payload, subject: 'topic-value')
-
-      # Decode
-      decoded = backend.decode(encoded)
-      expect(decoded).to eq(payload)
-      expect(schema_registry).to have_received(:decode).with('encoded-payload')
-    end
-
-    it 'should encode hash payloads by converting to protobuf message' do
-      hash_payload = {
-        str: 'string',
-        num: 123,
-        flag: true
-      }
-      allow(schema_registry).to receive(:encode).and_return('encoded-payload')
-
-      backend.encode(hash_payload, topic: 'topic')
-
-      # Should have converted hash to protobuf message before encoding
-      expect(schema_registry).to have_received(:encode) do |arg, **kwargs|
-        expect(arg).to be_a(Sample::V1::SampleMessage)
-        expect(arg.str).to eq('string')
-        expect(arg.num).to eq(123)
-        expect(arg.flag).to be(true)
-        expect(kwargs).to eq(subject: 'topic-value')
-      end
-    end
+  specify('#decode_key') do
+    expect(backend.decode_key('789', nil)).to eq(789)
+    expect(backend.decode_key('{"foo":"bar"}', :foo)).to eq('bar')
+    expect(backend.decode_key('{"foo":"bar"}', nil)).to eq({ 'foo' => 'bar' })
   end
 
-  describe 'key encoding with key_config field (auto-generated JSON schema)' do
-    it 'should encode a simple field key using JSON schema' do
-      key_hash = { 'str' => 'test-key' }
-
-      allow(key_schema_registry).to receive(:encode).and_return('encoded-key')
-
-      # encode_key is called from encode_proto_key with the hash
-      encoded = backend.encode_key('str', 'test-key', topic: 'my-topic')
-
-      expect(key_schema_registry).to have_received(:encode).with(
-        key_hash,
-        subject: 'my-topic-key',
-        schema_text: anything
-      )
-      expect(encoded).to eq('encoded-key')
-    end
-
-    it 'should encode a hash key with field extraction using JSON schema' do
-      full_payload = { 'str' => 'test-key', 'num' => 123 }
-      key_hash = { 'str' => 'test-key' }
-
-      allow(key_schema_registry).to receive(:encode).and_return('encoded-key')
-
-      encoded = backend.encode_key('str', full_payload, topic: 'my-topic')
-
-      expect(key_schema_registry).to have_received(:encode).with(
-        key_hash,
-        subject: 'my-topic-key',
-        schema_text: anything
-      )
-      expect(encoded).to eq('encoded-key')
-    end
-
-    it 'should decode a JSON schema key' do
-      decoded_json = { 'str' => 'test-key' }
-      allow(key_schema_registry).to receive(:decode).and_return(decoded_json)
-
-      result = backend.decode_key('encoded-key', 'str')
-
-      expect(key_schema_registry).to have_received(:decode).with('encoded-key')
-      expect(result).to eq('test-key')
-    end
-
-    it 'should decode a JSON schema key without field extraction' do
-      decoded_json = { 'str' => 'test-key', 'num' => 123 }
-      allow(key_schema_registry).to receive(:decode).and_return(decoded_json)
-
-      result = backend.decode_key('encoded-key', nil)
-
-      expect(key_schema_registry).to have_received(:decode).with('encoded-key')
-      expect(result).to eq(decoded_json)
-    end
+  it 'should encode and decode correctly' do
+    proto_turf = instance_double(ProtoTurf)
+    allow(proto_turf).to receive_messages(encode: 'encoded-payload', decode: payload)
+    allow(described_class).to receive(:proto_turf).and_return(proto_turf)
+    results = backend.encode(payload, topic: 'topic')
+    expect(results).to eq('encoded-payload')
+    results = backend.decode(results)
+    expect(results).to eq(payload)
+    expect(proto_turf).to have_received(:encode).with(payload, subject: 'topic')
+    expect(proto_turf).to have_received(:decode).with('encoded-payload')
   end
 
-  describe 'key encoding with key_config schema (separate Protobuf schema)' do
-    let(:key_backend) { described_class.new(schema: 'sample.v1.SampleMessageKey') }
-    let(:key_schema_registry_for_test) { instance_double(SchemaRegistry::Client) }
-    let(:schema_registry_for_test) { instance_double(SchemaRegistry::Client) }
-
-    before(:each) do
-      # For key_backend tests, mock both schema_registry instances
-      allow(described_class).to receive_messages(schema_registry: schema_registry_for_test,
-                                                 key_schema_registry: key_schema_registry_for_test)
-    end
-
-    it 'should encode a protobuf key message using the key schema' do
-      key_msg = Sample::V1::SampleMessageKey.new(str: 'test-key')
-      # Since subject ends with '-key', encode_payload uses key_schema_registry
-      allow(key_schema_registry_for_test).to receive(:encode).and_return('encoded-key')
-
-      encoded = key_backend.encode(key_msg, topic: 'my-topic', is_key: true)
-
-      expect(key_schema_registry_for_test).to have_received(:encode).with(key_msg, subject: 'my-topic-key')
-      expect(encoded).to eq('encoded-key')
-    end
-
-    it 'should encode a hash key using the key schema' do
-      key_hash = { str: 'test-key' }
-      allow(key_schema_registry_for_test).to receive(:encode).and_return('encoded-key')
-
-      encoded = key_backend.encode(key_hash, topic: 'my-topic', is_key: true)
-
-      expect(key_schema_registry_for_test).to have_received(:encode) do |arg, **kwargs|
-        expect(arg).to be_a(Sample::V1::SampleMessageKey)
-        expect(arg.str).to eq('test-key')
-        expect(kwargs).to eq(subject: 'my-topic-key')
-      end
-      expect(encoded).to eq('encoded-key')
-    end
-
-    it 'should decode a protobuf key' do
-      key_msg = Sample::V1::SampleMessageKey.new(str: 'test-key')
-      allow(schema_registry_for_test).to receive(:decode).and_return(key_msg)
-
-      decoded = key_backend.decode('encoded-key')
-
-      expect(schema_registry_for_test).to have_received(:decode).with('encoded-key')
-      expect(decoded).to eq(key_msg)
-    end
-  end
-
-  describe 'backward compatibility with plain string/JSON keys' do
-    it 'should encode plain string keys as strings' do
-      expect(backend.encode_key(nil, 'simple-string', topic: 'my-topic')).to eq('simple-string')
-    end
-
-    it 'should encode integer keys as strings' do
-      expect(backend.encode_key(nil, 789, topic: 'my-topic')).to eq('789')
-    end
-
-    it 'should encode hash keys using JSON schema when no field specified' do
-      key_hash = { foo: 'bar', baz: 'qux' }
-      allow(key_schema_registry).to receive(:encode).and_return('encoded-json')
-
-      backend.encode_key(nil, key_hash, topic: 'my-topic')
-
-      # Should use encode_proto_key which calls key_schema_registry.encode
-      expect(key_schema_registry).to have_received(:encode)
-    end
-
-    it 'should decode JSON string keys to hashes' do
-      decoded_hash = { 'foo' => 'bar' }
-      allow(key_schema_registry).to receive(:decode).and_return(decoded_hash)
-
-      result = backend.decode_key('{"foo":"bar"}', nil)
-      expect(result).to eq(decoded_hash)
-    end
-
-    it 'should decode JSON keys with field extraction' do
-      decoded_hash = { 'foo' => 'bar', 'baz' => 'qux' }
-      allow(key_schema_registry).to receive(:decode).and_return(decoded_hash)
-
-      result = backend.decode_key('encoded-key', :foo)
-      expect(result).to eq('bar')
-    end
-  end
-
-  describe 'edge cases' do
-    it 'should handle nested field extraction for keys' do
-      # Use nested field from the actual schema: non_union_nested.nested_str
-      nested_payload = { 'non_union_nested' => { 'nested_str' => 'value', 'nested_num' => 123 }, 'str' => 'other' }
-      allow(key_schema_registry).to receive(:encode).and_return('encoded-key')
-
-      backend.encode_key('non_union_nested.nested_str', nested_payload, topic: 'my-topic')
-
-      # Should encode with just the nested_str field extracted from the nested message
-      expect(key_schema_registry).to have_received(:encode).with(
-        { 'nested_str' => 'value' },
-        subject: 'my-topic-key',
-        schema_text: anything
-      )
-    end
-
-    it 'should handle encoding payloads with subject suffix detection' do
-      allow(schema_registry).to receive(:encode).and_return('encoded-payload')
-      allow(key_schema_registry).to receive(:encode).and_return('encoded-payload')
-
-      # When subject ends with '-key', should use key_schema_registry
-      backend.encode_payload(payload, subject: 'my-topic-key')
-      expect(key_schema_registry).to have_received(:encode).with(payload, subject: 'my-topic-key')
-
-      # When subject doesn't end with '-key', should use schema_registry
-      backend.encode_payload(payload, subject: 'my-topic-value')
-      expect(schema_registry).to have_received(:encode).with(payload, subject: 'my-topic-value')
-    end
-
-    it 'should handle nil payloads gracefully' do
-      # encode with nil payload - schema_registry will handle the nil
-      allow(schema_registry).to receive(:encode).with(nil, subject: 'topic-value').and_return(nil)
-      expect(backend.encode(nil, topic: 'topic')).to be_nil
-
-      # decode returns nil for nil payload (base class checks this)
-      expect(backend.decode(nil)).to be_nil
-    end
-  end
-
-  describe 'schema field introspection' do
-    it 'should return schema fields from protobuf descriptor' do
-      fields = backend.schema_fields
-
-      expect(fields).to be_an(Array)
-      expect(fields.map(&:name)).to include('str', 'num', 'str_arr', 'flag', 'timestamp')
-    end
-  end
 end

data/spec/snapshots/consumers-no-nest.snap

@@ -217,6 +217,13 @@ module Schemas
       'com.my-namespace'
     end
 
+    def self.tombstone(key)
+      record = self.allocate
+      record.tombstone_key = key
+      record.a_string = key
+      record
+    end
+
     # @override
     def as_json(_opts={})
       {
@@ -437,13 +444,6 @@ module Schemas
       'com.my-namespace'
     end
 
-    def self.tombstone(key)
-      record = self.allocate
-      record.tombstone_key = key
-      record.test_id = key
-      record
-    end
-
     # @override
     def as_json(_opts={})
       {

data/spec/snapshots/consumers.snap

@@ -177,6 +177,13 @@ module Schemas
       'com.my-namespace'
     end
 
+    def self.tombstone(key)
+      record = self.allocate
+      record.tombstone_key = key
+      record.a_string = key
+      record
+    end
+
     # @override
     def as_json(_opts={})
       {
@@ -391,13 +398,6 @@ module Schemas
       'com.my-namespace'
     end
 
-    def self.tombstone(key)
-      record = self.allocate
-      record.tombstone_key = key
-      record.test_id = key
-      record
-    end
-
     # @override
     def as_json(_opts={})
       {

data/spec/snapshots/consumers_and_producers-no-nest.snap

@@ -217,6 +217,13 @@ module Schemas
       'com.my-namespace'
     end
 
+    def self.tombstone(key)
+      record = self.allocate
+      record.tombstone_key = key
+      record.a_string = key
+      record
+    end
+
     # @override
     def as_json(_opts={})
       {
@@ -437,13 +444,6 @@ module Schemas
       'com.my-namespace'
     end
 
-    def self.tombstone(key)
-      record = self.allocate
-      record.tombstone_key = key
-      record.test_id = key
-      record
-    end
-
     # @override
     def as_json(_opts={})
       {

data/spec/snapshots/consumers_and_producers.snap

@@ -177,6 +177,13 @@ module Schemas
       'com.my-namespace'
     end
 
+    def self.tombstone(key)
+      record = self.allocate
+      record.tombstone_key = key
+      record.a_string = key
+      record
+    end
+
     # @override
     def as_json(_opts={})
       {
@@ -391,13 +398,6 @@ module Schemas
       'com.my-namespace'
     end
 
-    def self.tombstone(key)
-      record = self.allocate
-      record.tombstone_key = key
-      record.test_id = key
-      record
-    end
-
     # @override
     def as_json(_opts={})
       {

data/spec/snapshots/consumers_circular-no-nest.snap

@@ -217,6 +217,13 @@ module Schemas
       'com.my-namespace'
     end
 
+    def self.tombstone(key)
+      record = self.allocate
+      record.tombstone_key = key
+      record.a_string = key
+      record
+    end
+
     # @override
     def as_json(_opts={})
       {
@@ -437,13 +444,6 @@ module Schemas
       'com.my-namespace'
     end
 
-    def self.tombstone(key)
-      record = self.allocate
-      record.tombstone_key = key
-      record.test_id = key
-      record
-    end
-
     # @override
     def as_json(_opts={})
       {

data/spec/snapshots/consumers_circular.snap

@@ -177,6 +177,13 @@ module Schemas
       'com.my-namespace'
     end
 
+    def self.tombstone(key)
+      record = self.allocate
+      record.tombstone_key = key
+      record.a_string = key
+      record
+    end
+
     # @override
     def as_json(_opts={})
       {
@@ -391,13 +398,6 @@ module Schemas
       'com.my-namespace'
     end
 
-    def self.tombstone(key)
-      record = self.allocate
-      record.tombstone_key = key
-      record.test_id = key
-      record
-    end
-
     # @override
     def as_json(_opts={})
       {

data/spec/snapshots/consumers_complex_types-no-nest.snap

@@ -217,6 +217,13 @@ module Schemas
       'com.my-namespace'
     end
 
+    def self.tombstone(key)
+      record = self.allocate
+      record.tombstone_key = key
+      record.a_string = key
+      record
+    end
+
     # @override
     def as_json(_opts={})
       {
@@ -437,13 +444,6 @@ module Schemas
       'com.my-namespace'
     end
 
-    def self.tombstone(key)
-      record = self.allocate
-      record.tombstone_key = key
-      record.test_id = key
-      record
-    end
-
     # @override
     def as_json(_opts={})
       {

data/spec/snapshots/consumers_complex_types.snap

@@ -177,6 +177,13 @@ module Schemas
       'com.my-namespace'
     end
 
+    def self.tombstone(key)
+      record = self.allocate
+      record.tombstone_key = key
+      record.a_string = key
+      record
+    end
+
     # @override
     def as_json(_opts={})
       {
@@ -391,13 +398,6 @@ module Schemas
       'com.my-namespace'
     end
 
-    def self.tombstone(key)
-      record = self.allocate
-      record.tombstone_key = key
-      record.test_id = key
-      record
-    end
-
     # @override
     def as_json(_opts={})
       {