deimos-ruby 2.2.2 → 2.3.0.pre.beta2

data/lib/deimos/schema_backends/proto_local.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'proto_base'
-require 'proto_turf'
+require 'schema_registry_client'
 
 module Deimos
   module SchemaBackends
@@ -14,14 +14,14 @@ module Deimos
       end
 
       # @override
-      def encode_payload(payload, schema: nil, topic: nil)
+      def encode_payload(payload, schema: nil, subject: nil)
         msg = payload.is_a?(Hash) ? proto_schema.msgclass.new(**payload) : payload
         proto_schema.msgclass.encode(msg)
       end
 
-      # @return [ProtoTurf]
-      def self.proto_turf
-        @proto_turf ||= ProtoTurf.new(
+      # @return [SchemaRegistry::Client]
+      def self.schema_registry
+        @schema_registry ||= SchemaRegistry::Client.new(
           registry_url: Deimos.config.schema.registry_url,
           logger: Karafka.logger
         )

data/lib/deimos/schema_backends/proto_schema_registry.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'proto_base'
-require 'proto_turf'
+require 'schema_registry_client'
 
 module Deimos
   module SchemaBackends
@@ -10,22 +10,43 @@ module Deimos
 
       # @override
       def decode_payload(payload, schema:)
-        self.class.proto_turf.decode(payload)
+        self.class.schema_registry.decode(payload)
       end
 
       # @override
-      def encode_payload(payload, schema: nil, topic: nil)
+      def encode_payload(payload, schema: nil, subject: nil)
         msg = payload.is_a?(Hash) ? proto_schema.msgclass.new(**payload) : payload
-        self.class.proto_turf.encode(msg, subject: topic)
+        encoder = subject&.ends_with?('-key') ? self.class.key_schema_registry : self.class.schema_registry
+        encoder.encode(msg, subject: subject)
       end
 
-      # @return [ProtoTurf]
-      def self.proto_turf
-        @proto_turf ||= ProtoTurf.new(
+      # @override
+      def encode_proto_key(key, topic: nil, field: nil)
+        schema_text = SchemaRegistry::Output::JsonSchema.output(proto_schema.to_proto, path: field)
+        self.class.key_schema_registry.encode(key, subject: "#{topic}-key", schema_text: schema_text)
+      end
+
+      # @override
+      def decode_proto_key(payload)
+        self.class.key_schema_registry.decode(payload)
+      end
+
+      # @return [SchemaRegistry::Client]
+      def self.schema_registry
+        @schema_registry ||= SchemaRegistry::Client.new(
           registry_url: Deimos.config.schema.registry_url,
           logger: Karafka.logger
         )
       end
+
+      def self.key_schema_registry
+        @key_schema_registry ||= SchemaRegistry::Client.new(
+          registry_url: Deimos.config.schema.registry_url,
+          logger: Karafka.logger,
+          schema_type: SchemaRegistry::Schema::ProtoJsonSchema
+        )
+      end
+
     end
   end
 end

data/lib/deimos/transcoder.rb

@@ -36,7 +36,7 @@ module Deimos
       if self.key_field
         self.backend.encode_key(self.key_field, key, topic: @topic)
       else
-        self.backend.encode(key, topic: @topic)
+        self.backend.encode(key, topic: @topic, is_key: true)
       end
     end
 

data/lib/deimos/utils/outbox_producer.rb

@@ -205,7 +205,7 @@ module Deimos
         batch_size = batch.size
         current_index = 0
 
-        batch[current_index..-1].in_groups_of(batch_size, false).each do |group|
+        batch[current_index..].in_groups_of(batch_size, false).each do |group|
           @logger.debug("Publishing #{group.size} messages to #{@current_topic}")
           Karafka.producer.produce_many_sync(group)
           current_index += group.size

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '2.2.2'
+  VERSION = '2.3.0-beta2'
 end

data/lib/generators/deimos/active_record_generator.rb

@@ -48,7 +48,7 @@ module Deimos
         # @return [String]
         def schema
           last_dot = self.full_schema.rindex('.')
-          self.full_schema[(last_dot + 1)..-1]
+          self.full_schema[(last_dot + 1)..]
         end
 
         # @return [String]

data/lib/generators/deimos/schema_class_generator.rb

@@ -13,7 +13,7 @@ module Deimos
       # @return [Array<Symbol>]
       SPECIAL_TYPES = %i(record enum).freeze
       # @return [String]
-      INITIALIZE_WHITESPACE = "\n#{' ' * 19}"
+      INITIALIZE_WHITESPACE = "\n#{' ' * 19}".freeze
       # @return [Array<String>]
       IGNORE_DEFAULTS = %w(message_id timestamp).freeze
       # @return [String]
@@ -192,7 +192,7 @@ module Deimos
 
       def generate_from_schema_files(found_schemas)
         path = Deimos.config.schema.path || Deimos.config.schema.paths[:avro].first
-        schema_store = AvroTurf::MutableSchemaStore.new(path: path)
+        schema_store = SchemaRegistry::AvroSchemaStore.new(path: path)
         schema_store.load_schemas!
         schema_store.schemas.values.sort_by { |s| "#{s.namespace}#{s.name}" }.each do |schema|
           name = "#{schema.namespace}.#{schema.name}"
@@ -265,7 +265,7 @@ module Deimos
         end
 
         result = "def initialize(_from_message: false, #{arguments.first}"
-        arguments[1..-1].each_with_index do |arg, _i|
+        arguments[1..].each_with_index do |arg, _i|
           result += ",#{INITIALIZE_WHITESPACE}#{arg}"
         end
         "#{result})"

data/lib/generators/deimos/v2_generator.rb

@@ -119,7 +119,7 @@ module Deimos
       end
 
       def consumer_configs
-        deimos_config.consumer_objects.group_by(&:group_id).map { |group_id, consumers|
+        deimos_config.consumer_objects.group_by(&:group_id).to_h do |group_id, consumers|
           [group_id, consumers.map do |consumer|
             kafka_configs = {}
             kafka_configs['auto.offset.reset'] = consumer.start_from_beginning ? 'earliest' : 'latest'
@@ -159,7 +159,7 @@ module Deimos
             configs[:each_message] = true unless consumer.delivery.to_s == 'inline_batch'
             configs
           end]
-        }.to_h
+        end
       end
 
       def producer_configs

data/spec/gen/sample/v1/sample_key_pb.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: sample/v1/sample_key.proto
+
+require 'google/protobuf'
+
+
+descriptor_data = "\n\x1asample/v1/sample_key.proto\x12\tsample.v1\"\x1f\n\x10SampleMessageKey\x12\x0b\n\x03str\x18\x01 \x01(\tb\x06proto3"
+
+pool = ::Google::Protobuf::DescriptorPool.generated_pool
+pool.add_serialized_file(descriptor_data)
+
+module Sample
+  module V1
+    SampleMessageKey = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("sample.v1.SampleMessageKey").msgclass
+  end
+end

data/spec/generators/schema_class_generator_spec.rb

@@ -20,7 +20,7 @@ end
 
 RSpec.describe Deimos::Generators::SchemaClassGenerator do
   let(:schema_class_path) { 'spec/app/lib/schema_classes' }
-  let(:files) { Dir["#{schema_class_path}/**/*.rb"].map { |f| [f, File.read(f)]}.to_h }
+  let(:files) { Dir["#{schema_class_path}/**/*.rb"].to_h { |f| [f, File.read(f)]} }
 
   before(:each) do
     Deimos.config.reset!

data/spec/protos/sample/v1/sample_key.proto

@@ -0,0 +1,7 @@
+syntax = "proto3";
+
+package sample.v1;
+
+message SampleMessageKey {
+  string str = 1;
+}

data/spec/schema_backends/avro_base_shared.rb

@@ -66,7 +66,7 @@ RSpec.shared_examples_for('an Avro backend') do
     expect(backend.encode_key('test_id', 1, topic: 'topic')).to eq('itsme')
     expect(backend).to have_received(:encode).
       with({ 'test_id' => 1 }, { schema: 'MySchema_key', topic: 'topic' })
-    expect(backend.schema_store.find('MySchema_key', 'com.my-namespace').to_avro).
+    expect(backend.schema_store.find('com.my-namespace.MySchema_key').to_avro).
       to eq(
         'doc' => 'Key for com.my-namespace.MySchema - autogenerated by Deimos',
         'fields' => [

data/spec/schema_backends/avro_local_spec.rb

@@ -15,17 +15,10 @@ RSpec.describe Deimos::SchemaBackends::AvroLocal do
   it_should_behave_like 'an Avro backend'
 
   it 'should encode and decode correctly' do
-    avro_turf = instance_double(AvroTurf)
-    allow(avro_turf).to receive_messages(encode: 'encoded-payload', decode: payload)
-    allow(backend).to receive(:avro_turf).and_return(avro_turf)
     results = backend.encode(payload)
-    expect(results).to eq('encoded-payload')
+    expect(results).to start_with("Obj\u0001\u0004\u0014avro.codec\bnull\u0016avro.schema\x94")
     results = backend.decode(results)
     expect(results).to eq(payload)
-    expect(avro_turf).to have_received(:encode).
-      with(payload, schema_name: 'MySchema', namespace: 'com.my-namespace')
-    expect(avro_turf).to have_received(:decode).
-      with('encoded-payload', schema_name: 'MySchema', namespace: 'com.my-namespace')
   end
 
 end

data/spec/schema_backends/avro_schema_registry_spec.rb

@@ -15,17 +15,17 @@ RSpec.describe Deimos::SchemaBackends::AvroSchemaRegistry do
   it_should_behave_like 'an Avro backend'
 
   it 'should encode and decode correctly' do
-    avro_turf = instance_double(AvroTurf::Messaging)
-    allow(avro_turf).to receive_messages(encode: 'encoded-payload', decode: payload)
-    allow(backend).to receive(:avro_turf_messaging).and_return(avro_turf)
+    schema_registry = instance_double(SchemaRegistry::Client)
+    allow(schema_registry).to receive_messages(encode: 'encoded-payload', decode: payload)
+    allow(backend).to receive(:schema_registry).and_return(schema_registry)
     results = backend.encode(payload, topic: 'topic')
     expect(results).to eq('encoded-payload')
     results = backend.decode(results)
     expect(results).to eq(payload)
-    expect(avro_turf).to have_received(:encode).
-      with(payload, schema_name: 'MySchema', subject: 'topic')
-    expect(avro_turf).to have_received(:decode).
-      with('encoded-payload', schema_name: 'MySchema')
+    expect(schema_registry).to have_received(:encode).
+      with(payload, schema_name: 'com.my-namespace.MySchema', subject: 'topic-value')
+    expect(schema_registry).to have_received(:decode).
+      with('encoded-payload')
   end
 
 end

data/spec/schema_backends/base_spec.rb

@@ -6,13 +6,13 @@ describe Deimos::SchemaBackends::Base do
 
   it 'should validate on encode' do
     expect(backend).to receive(:validate).with(payload, schema: 'schema')
-    expect(backend).to receive(:encode_payload).with(payload, schema: 'schema', topic: 'topic')
+    expect(backend).to receive(:encode_payload).with(payload, schema: 'schema', subject: 'topic-value')
     backend.encode(payload, topic: 'topic')
   end
 
   it 'should validate and encode a passed schema' do
     expect(backend).to receive(:validate).with(payload, schema: 'schema2')
-    expect(backend).to receive(:encode_payload).with(payload, schema: 'schema2', topic: 'topic')
+    expect(backend).to receive(:encode_payload).with(payload, schema: 'schema2', subject: 'topic-value')
     backend.encode(payload, schema: 'schema2', topic: 'topic')
   end
 

data/spec/schema_backends/proto_schema_registry_spec.rb

@@ -2,6 +2,7 @@
 
 require 'deimos/schema_backends/proto_schema_registry'
 require_relative "#{__dir__}/../gen/sample/v1/sample_pb"
+require_relative "#{__dir__}/../gen/sample/v1/sample_key_pb"
 
 RSpec.describe Deimos::SchemaBackends::ProtoSchemaRegistry do
   let(:payload) do
@@ -16,32 +17,234 @@ RSpec.describe Deimos::SchemaBackends::ProtoSchemaRegistry do
       str_map: { 'foo' => 'bar' }
     )
   end
+  let(:schema_registry) { instance_double(SchemaRegistry::Client) }
+  let(:key_schema_registry) { instance_double(SchemaRegistry::Client) }
 
   let(:backend) { described_class.new(schema: 'sample.v1.SampleMessage') }
 
-  specify('#encode_key') do
-    expect(backend.encode_key(nil, 789)).to eq('789')
-    expect(backend.encode_key(nil, 'string')).to eq('string')
-    expect(backend.encode_key(nil, { foo: 'bar' })).to eq('{"foo":"bar"}')
-    expect(backend.encode_key(:foo, 'bar')).to eq('bar')
+  before(:each) do
+    allow(described_class).to receive_messages(schema_registry: schema_registry,
+                                               key_schema_registry: key_schema_registry)
   end
 
-  specify('#decode_key') do
-    expect(backend.decode_key('789', nil)).to eq(789)
-    expect(backend.decode_key('{"foo":"bar"}', :foo)).to eq('bar')
-    expect(backend.decode_key('{"foo":"bar"}', nil)).to eq({ 'foo' => 'bar' })
+  describe 'payload encoding and decoding' do
+    it 'should encode and decode payloads correctly' do
+      allow(schema_registry).to receive_messages(encode: 'encoded-payload', decode: payload)
+
+      # Encode
+      encoded = backend.encode(payload, topic: 'topic')
+      expect(encoded).to eq('encoded-payload')
+      expect(schema_registry).to have_received(:encode).with(payload, subject: 'topic-value')
+
+      # Decode
+      decoded = backend.decode(encoded)
+      expect(decoded).to eq(payload)
+      expect(schema_registry).to have_received(:decode).with('encoded-payload')
+    end
+
+    it 'should encode hash payloads by converting to protobuf message' do
+      hash_payload = {
+        str: 'string',
+        num: 123,
+        flag: true
+      }
+      allow(schema_registry).to receive(:encode).and_return('encoded-payload')
+
+      backend.encode(hash_payload, topic: 'topic')
+
+      # Should have converted hash to protobuf message before encoding
+      expect(schema_registry).to have_received(:encode) do |arg, **kwargs|
+        expect(arg).to be_a(Sample::V1::SampleMessage)
+        expect(arg.str).to eq('string')
+        expect(arg.num).to eq(123)
+        expect(arg.flag).to be(true)
+        expect(kwargs).to eq(subject: 'topic-value')
+      end
+    end
   end
 
-  it 'should encode and decode correctly' do
-    proto_turf = instance_double(ProtoTurf)
-    allow(proto_turf).to receive_messages(encode: 'encoded-payload', decode: payload)
-    allow(described_class).to receive(:proto_turf).and_return(proto_turf)
-    results = backend.encode(payload, topic: 'topic')
-    expect(results).to eq('encoded-payload')
-    results = backend.decode(results)
-    expect(results).to eq(payload)
-    expect(proto_turf).to have_received(:encode).with(payload, subject: 'topic')
-    expect(proto_turf).to have_received(:decode).with('encoded-payload')
+  describe 'key encoding with key_config field (auto-generated JSON schema)' do
+    it 'should encode a simple field key using JSON schema' do
+      key_hash = { 'str' => 'test-key' }
+
+      allow(key_schema_registry).to receive(:encode).and_return('encoded-key')
+
+      # encode_key is called from encode_proto_key with the hash
+      encoded = backend.encode_key('str', 'test-key', topic: 'my-topic')
+
+      expect(key_schema_registry).to have_received(:encode).with(
+        key_hash,
+        subject: 'my-topic-key',
+        schema_text: anything
+      )
+      expect(encoded).to eq('encoded-key')
+    end
+
+    it 'should encode a hash key with field extraction using JSON schema' do
+      full_payload = { 'str' => 'test-key', 'num' => 123 }
+      key_hash = { 'str' => 'test-key' }
+
+      allow(key_schema_registry).to receive(:encode).and_return('encoded-key')
+
+      encoded = backend.encode_key('str', full_payload, topic: 'my-topic')
+
+      expect(key_schema_registry).to have_received(:encode).with(
+        key_hash,
+        subject: 'my-topic-key',
+        schema_text: anything
+      )
+      expect(encoded).to eq('encoded-key')
+    end
+
+    it 'should decode a JSON schema key' do
+      decoded_json = { 'str' => 'test-key' }
+      allow(key_schema_registry).to receive(:decode).and_return(decoded_json)
+
+      result = backend.decode_key('encoded-key', 'str')
+
+      expect(key_schema_registry).to have_received(:decode).with('encoded-key')
+      expect(result).to eq('test-key')
+    end
+
+    it 'should decode a JSON schema key without field extraction' do
+      decoded_json = { 'str' => 'test-key', 'num' => 123 }
+      allow(key_schema_registry).to receive(:decode).and_return(decoded_json)
+
+      result = backend.decode_key('encoded-key', nil)
+
+      expect(key_schema_registry).to have_received(:decode).with('encoded-key')
+      expect(result).to eq(decoded_json)
+    end
   end
 
+  describe 'key encoding with key_config schema (separate Protobuf schema)' do
+    let(:key_backend) { described_class.new(schema: 'sample.v1.SampleMessageKey') }
+    let(:key_schema_registry_for_test) { instance_double(SchemaRegistry::Client) }
+    let(:schema_registry_for_test) { instance_double(SchemaRegistry::Client) }
+
+    before(:each) do
+      # For key_backend tests, mock both schema_registry instances
+      allow(described_class).to receive_messages(schema_registry: schema_registry_for_test,
+                                                 key_schema_registry: key_schema_registry_for_test)
+    end
+
+    it 'should encode a protobuf key message using the key schema' do
+      key_msg = Sample::V1::SampleMessageKey.new(str: 'test-key')
+      # Since subject ends with '-key', encode_payload uses key_schema_registry
+      allow(key_schema_registry_for_test).to receive(:encode).and_return('encoded-key')
+
+      encoded = key_backend.encode(key_msg, topic: 'my-topic', is_key: true)
+
+      expect(key_schema_registry_for_test).to have_received(:encode).with(key_msg, subject: 'my-topic-key')
+      expect(encoded).to eq('encoded-key')
+    end
+
+    it 'should encode a hash key using the key schema' do
+      key_hash = { str: 'test-key' }
+      allow(key_schema_registry_for_test).to receive(:encode).and_return('encoded-key')
+
+      encoded = key_backend.encode(key_hash, topic: 'my-topic', is_key: true)
+
+      expect(key_schema_registry_for_test).to have_received(:encode) do |arg, **kwargs|
+        expect(arg).to be_a(Sample::V1::SampleMessageKey)
+        expect(arg.str).to eq('test-key')
+        expect(kwargs).to eq(subject: 'my-topic-key')
+      end
+      expect(encoded).to eq('encoded-key')
+    end
+
+    it 'should decode a protobuf key' do
+      key_msg = Sample::V1::SampleMessageKey.new(str: 'test-key')
+      allow(schema_registry_for_test).to receive(:decode).and_return(key_msg)
+
+      decoded = key_backend.decode('encoded-key')
+
+      expect(schema_registry_for_test).to have_received(:decode).with('encoded-key')
+      expect(decoded).to eq(key_msg)
+    end
+  end
+
+  describe 'backward compatibility with plain string/JSON keys' do
+    it 'should encode plain string keys as strings' do
+      expect(backend.encode_key(nil, 'simple-string', topic: 'my-topic')).to eq('simple-string')
+    end
+
+    it 'should encode integer keys as strings' do
+      expect(backend.encode_key(nil, 789, topic: 'my-topic')).to eq('789')
+    end
+
+    it 'should encode hash keys using JSON schema when no field specified' do
+      key_hash = { foo: 'bar', baz: 'qux' }
+      allow(key_schema_registry).to receive(:encode).and_return('encoded-json')
+
+      backend.encode_key(nil, key_hash, topic: 'my-topic')
+
+      # Should use encode_proto_key which calls key_schema_registry.encode
+      expect(key_schema_registry).to have_received(:encode)
+    end
+
+    it 'should decode JSON string keys to hashes' do
+      decoded_hash = { 'foo' => 'bar' }
+      allow(key_schema_registry).to receive(:decode).and_return(decoded_hash)
+
+      result = backend.decode_key('{"foo":"bar"}', nil)
+      expect(result).to eq(decoded_hash)
+    end
+
+    it 'should decode JSON keys with field extraction' do
+      decoded_hash = { 'foo' => 'bar', 'baz' => 'qux' }
+      allow(key_schema_registry).to receive(:decode).and_return(decoded_hash)
+
+      result = backend.decode_key('encoded-key', :foo)
+      expect(result).to eq('bar')
+    end
+  end
+
+  describe 'edge cases' do
+    it 'should handle nested field extraction for keys' do
+      # Use nested field from the actual schema: non_union_nested.nested_str
+      nested_payload = { 'non_union_nested' => { 'nested_str' => 'value', 'nested_num' => 123 }, 'str' => 'other' }
+      allow(key_schema_registry).to receive(:encode).and_return('encoded-key')
+
+      backend.encode_key('non_union_nested.nested_str', nested_payload, topic: 'my-topic')
+
+      # Should encode with just the nested_str field extracted from the nested message
+      expect(key_schema_registry).to have_received(:encode).with(
+        { 'nested_str' => 'value' },
+        subject: 'my-topic-key',
+        schema_text: anything
+      )
+    end
+
+    it 'should handle encoding payloads with subject suffix detection' do
+      allow(schema_registry).to receive(:encode).and_return('encoded-payload')
+      allow(key_schema_registry).to receive(:encode).and_return('encoded-payload')
+
+      # When subject ends with '-key', should use key_schema_registry
+      backend.encode_payload(payload, subject: 'my-topic-key')
+      expect(key_schema_registry).to have_received(:encode).with(payload, subject: 'my-topic-key')
+
+      # When subject doesn't end with '-key', should use schema_registry
+      backend.encode_payload(payload, subject: 'my-topic-value')
+      expect(schema_registry).to have_received(:encode).with(payload, subject: 'my-topic-value')
+    end
+
+    it 'should handle nil payloads gracefully' do
+      # encode with nil payload - schema_registry will handle the nil
+      allow(schema_registry).to receive(:encode).with(nil, subject: 'topic-value').and_return(nil)
+      expect(backend.encode(nil, topic: 'topic')).to be_nil
+
+      # decode returns nil for nil payload (base class checks this)
+      expect(backend.decode(nil)).to be_nil
+    end
+  end
+
+  describe 'schema field introspection' do
+    it 'should return schema fields from protobuf descriptor' do
+      fields = backend.schema_fields
+
+      expect(fields).to be_an(Array)
+      expect(fields.map(&:name)).to include('str', 'num', 'str_arr', 'flag', 'timestamp')
+    end
+  end
 end

data/spec/spec_helper.rb

@@ -16,7 +16,7 @@ require 'handlers/my_consumer'
 require 'rspec/rails'
 require 'rspec/snapshot'
 require 'karafka/testing/rspec/helpers'
-Dir['./spec/schemas/**/*.rb'].sort.each { |f| require f }
+Dir['./spec/schemas/**/*.rb'].each { |f| require f }
 
 # Constants used for consumer specs
 SCHEMA_CLASS_SETTINGS = { off: false, on: true }.freeze