deimos-ruby 2.2.3.pre.beta2 → 2.3.0.pre.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bf4ddbfc7afab6d397364f0108d2eb845e2b646e8615783d2376b92edc34d380
-  data.tar.gz: 5f3a404983d2dfa8f0c5324d8ba4d728b1f976940b9242756d81fcc53a6a6cab
+  metadata.gz: fa14b17ba7b1cef1f42545415c749c2ae6681aaae7d65497a80820ede0d7d506
+  data.tar.gz: 238d530e1be139a8985a27f29dde91572dc361fc43ab1e0a8a3fc479a2eaf2ab
 SHA512:
-  metadata.gz: db87d1ed05ccae6430adfd1749902b319e53b7039f29512fb8335cceef4395e4ad26ad0a6437b87ec622f301923a3b27f9855471f3b8a1b5c5a86e4991d51dc0
-  data.tar.gz: 996d61652323b29bf40152ccb9fb71c74b050948bc1363b0d1a26191e8595240a20344b6256a0cfa6821ae31c7f1ee64501b54bfb071564bb06abc81e831a790
+  metadata.gz: 3ce9fa60268a647866e0f88253e300bf8a02ff675f8b05380815a593d79911e22864648c53602eab493e3900f96d0403e11db54a1af873f45347a06e77a32821
+  data.tar.gz: 8a14f011128bfc9c5ed1693a012dc6aad3dfe2a344f56e6a46c0c18e77d9f8f3e3d0163704ffa562e9feee4417865a81c5a126dc6147832259b87a240a600993

data/.rubocop.yml

@@ -4,7 +4,7 @@ plugins:
   - rubocop-rspec
 
 AllCops:
-  TargetRubyVersion: 2.5
+  TargetRubyVersion: 3.0
   Exclude:
     - lib/deimos/monkey_patches/*.rb
     - spec/gen/**/*.rb

data/CHANGELOG.md

@@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
-- Feature: Support broker setting per topic in producer configs.
+- Major change: Switch from using `avro_turf` and `proto_turf` to use `schema_registry_client`, which handles both Avro and Protobuf.
 
 # 2.2.2 - 2025-11-7
 

data/CLAUDE.md

@@ -0,0 +1,270 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Overview
+
+Deimos is a Ruby framework for Kafka development that marries Kafka with schema definitions (Avro/Protobuf), ActiveRecord, and provides a comprehensive toolbox. Built on top of Karafka (which itself builds on RDKafka), it provides schema encoding/decoding, database integration, metrics, tracing, and test helpers.
+
+## Development Commands
+
+### Testing
+
+```bash
+# Run all tests
+bundle exec rspec
+
+# Run a single test file
+bundle exec rspec spec/path/to/file_spec.rb
+
+# Run a specific test
+bundle exec rspec spec/path/to/file_spec.rb:LINE_NUMBER
+```
+
+### Linting
+
+```bash
+# Run Rubocop linter
+bundle exec rubocop
+
+# Auto-correct issues
+bundle exec rubocop -a
+```
+
+### Schema Class Generation
+
+When modifying schema-related code, you may need to regenerate test schema classes:
+
+```bash
+# Regenerate test schema classes (used after schema changes)
+bundle exec ./regenerate_test_schema_classes.rb
+
+# Generate Protobuf classes
+protoc -I spec/protos --ruby_out=spec/gen --ruby_opt=paths=source_relative spec/protos/**/*.proto
+```
+
+### Rake Tasks
+
+```bash
+# Start Deimos consumer (in Rails environment)
+rake deimos:start
+
+# Start outbox backend producer
+rake deimos:outbox
+
+# Start database poller
+rake deimos:db_poller
+
+# Generate schema classes
+rake deimos:generate_schema_classes
+```
+
+## Architecture
+
+### Layer Structure
+
+Deimos is built in layers:
+1. **RDKafka** - Low-level Kafka client providing producer/consumer APIs
+2. **Karafka** - Threaded consumers wrapper with lifecycle management
+3. **Deimos** - Full framework with schema integration, database features, metrics, and utilities
+
+### Key Directories
+
+- `lib/deimos/` - Core Deimos code
+- `lib/deimos/backends/` - Producer backends (kafka, kafka_async, outbox, test)
+- `lib/deimos/schema_backends/` - Schema handlers (Avro local/registry/validation, Protobuf, plain, mock)
+- `lib/deimos/metrics/` - Metrics providers (DataDog, mock)
+- `lib/deimos/tracing/` - Tracing providers (DataDog, mock)
+- `lib/deimos/utils/` - Utilities (DB poller, outbox producer, etc.)
+- `lib/deimos/config/` - Configuration classes
+- `lib/deimos/ext/` - Extensions to Karafka routing
+- `lib/generators/` - Rails generators for migrations
+- `lib/tasks/` - Rake tasks
+
+### Core Concepts
+
+#### Schema Backends
+
+Schema backends encode/decode payloads. All backends must implement:
+- `encode(payload, topic:)` - Encode payload to binary/string format
+- `decode(payload)` - Decode binary/string to hash
+- `validate(payload)` - Validate payload against schema
+- `coerce(payload)` - Coerce payload to match schema types
+- `schema_fields` - List fields in schema (used with ActiveRecord)
+- Define a `mock` backend for testing
+
+Available backends: `:avro_local`, `:avro_schema_registry`, `:avro_validation`, `:proto_schema_registry`, `:proto_local`, `:mock`, `:plain`
+
+#### Producer Backends
+
+Producer backends determine how messages are sent. All backends inherit from `Deimos::Backends::Base` and implement `execute(messages)`.
+
+Available backends:
+- `:kafka` - Send directly to Kafka (default)
+- `:kafka_async` - Async variant of kafka backend
+- `:outbox` - Transactional outbox pattern (save to DB, send async)
+- `:test` - For testing (stores messages in memory)
+
+#### Consumer Types
+
+- `Deimos::Consumer` - Base consumer class
+  - Per-message: Override `consume_message(message)` and set `each_message true`
+  - Batch: Override `consume_batch` (receives `messages` collection)
+
+- `Deimos::ActiveRecordConsumer` - Automatically saves/updates ActiveRecord models
+  - Per-message mode: Uses `fetch_record`, `assign_key`, `destroy_record`
+  - Batch mode: Uses `activerecord-import` for bulk operations
+  - Override `record_attributes(payload, key)` to customize attributes
+
+#### Producers
+
+- `Deimos::Producer` - Base producer class
+  - Call `self.produce([{payload: ..., key: ..., topic: ...}])`
+  - Override `partition_key(payload)` for custom partitioning
+
+- `Deimos::ActiveRecordProducer` - Produces from ActiveRecord models
+  - Set `record_class Widget, refetch: false`
+  - Override `generate_payload(attributes, record)` to customize payload
+  - Override `watched_attributes(record)` to add non-schema fields
+
+#### Key Configuration
+
+Every producer must define `key_config`:
+- `key_config none: true` - No keys (events)
+- `key_config plain: true` - Unencoded keys (legacy)
+- `key_config schema: 'MySchema-key'` - Use existing key schema
+- `key_config field: 'my_id'` - Auto-generate key schema from value field
+
+#### KafkaSource Mixin
+
+The `Deimos::KafkaSource` mixin adds callbacks to ActiveRecord models to automatically send Kafka messages on save/destroy:
+
+```ruby
+class Widget < ActiveRecord::Base
+  include Deimos::KafkaSource
+
+  def self.kafka_producers
+    [MyProducer]
+  end
+
+  def self.kafka_config
+    { update: true, delete: true, import: true, create: true }
+  end
+end
+```
+
+#### Outbox Pattern (Database Backend)
+
+The outbox pattern provides transactional guarantees:
+1. Messages are validated, encoded, and saved to `kafka_messages` table
+2. Separate thread pool (via `Deimos::Utils::OutboxProducer`) reads from DB and sends to Kafka
+3. Uses `kafka_topic_info` table for topic-level locking
+4. Runs via `rake deimos:outbox` or `Deimos.start_outbox_backend!(thread_count: N)`
+
+#### Database Poller
+
+Polls database tables and produces messages:
+1. Configure with `Deimos.configure { db_poller { producer_class 'MyProducer' } }`
+2. Two modes:
+   - Time-based (default): Uses `updated_at` and `id` columns
+   - State-based: Updates state column after publishing
+3. Tracks progress in `poll_info` table
+4. Runs via `rake deimos:db_poller`
+
+### Configuration
+
+Configuration uses the `fig_tree` gem. See `lib/deimos/config/configuration.rb` for the schema. Configure via:
+
+```ruby
+Deimos.configure do |config|
+  config.producers.backend = :outbox
+  config.schema.backend = :avro_schema_registry
+  config.schema.registry_url = 'http://localhost:8081'
+end
+```
+
+### Karafka Integration
+
+Deimos extends Karafka routing with schema configuration:
+
+```ruby
+Karafka::App.routes.draw do
+  topic 'my-topic' do
+    consumer MyConsumer
+    schema 'MySchema'
+    namespace 'com.my-namespace'
+    key_config field: 'test_id'
+  end
+end
+```
+
+The `lib/deimos/ext/` directory contains the routing extensions that add schema-aware capabilities to Karafka's routing DSL.
+
+### Shared Configuration
+
+Both producers and consumers use the `SharedConfig` module to standardize schema settings, topic configuration, key handling, etc.
+
+### Testing
+
+Test helpers in `lib/deimos/test_helpers.rb` provide:
+- `test_consume_message(consumer, payload)` - Test message consumption
+- `test_consume_batch(consumer, payloads)` - Test batch consumption
+- `expect(topic).to have_sent(payload, key, partition_key, headers)` - Assert messages sent
+- `Deimos::TestHelpers.sent_messages` - Inspect sent messages
+
+Configure for tests:
+```ruby
+Deimos.config.schema.backend = :avro_validation  # Validates but doesn't encode
+Deimos.config.producers.backend = :test  # Stores in memory
+```
+
+### Instrumentation
+
+Deimos sends events through Karafka's instrumentation monitor:
+- `deimos.encode_message` - Schema encoding
+- `deimos.outbox.produce` - Outbox messages processed
+- `deimos.ar_consumer.consume_batch` - ActiveRecord batch consumption
+- `deimos.batch_consumption.valid_records` - Valid records upserted
+- `deimos.batch_consumption.invalid_records` - Invalid records rejected
+
+Subscribe with: `Karafka.monitor.subscribe('event.name') { |event| ... }`
+
+## Database Integration
+
+Deimos uses two primary database tables (created via generators):
+
+- `kafka_messages` - Stores outbox messages (topic, key, payload, partition_key)
+- `kafka_topic_info` - Locks topics for outbox processing
+- `poll_info` - Tracks database poller progress
+
+## Special Considerations
+
+### Schema Changes
+
+When adding/modifying schemas:
+1. Update schema files in configured schema path
+2. Run `rake deimos:generate_schema_classes` if using schema classes
+3. Regenerate test schemas with `./regenerate_test_schema_classes.rb`
+
+### Protobuf
+
+- Requires `schema_registry_client` gem in Gemfile
+- Protobuf payloads must be Protobuf message objects, not hashes
+- Protobuf should NOT be used for keys (unstable binary encoding)
+
+### ActiveRecord Batch Consumers
+
+- Only supports primary keys as identifiers by default
+- Skips ActiveRecord callbacks (uses `activerecord-import` for bulk SQL)
+- Set `compacted false` to process all messages vs. only last per key
+- Requires `bulk_import_id_column` config when saving to multiple tables
+
+### Error Handling
+
+- Set `config.consumers.reraise_errors = false` to swallow non-fatal errors
+- Define fatal errors via `config.fatal_error` (global) or `fatal_error?(exception, payload, metadata)` (per-consumer)
+- Prevents consumer from getting stuck on bad messages
+
+## Integration Tests
+
+Integration tests run against real databases (PostgreSQL, MySQL/Trilogy, SQLite). They are marked with `:integration` metadata and use different database configs from `DbConfigs::DB_OPTIONS`.

data/README.md

@@ -94,8 +94,6 @@ Currently we have the following possible schema backends:
 * Protobuf Schema Registry (use Protobuf with the Confluent Schema Registry)
 * Mock (no actual encoding/decoding).
 
-Note that to use Protobuf, you must include the [proto_turf](https://github.com/flipp-oss/proto_turf) gem in your Gemfile.
-
 Other possible schemas could [JSONSchema](https://json-schema.org/), etc. Feel free to
 contribute!
 

data/deimos-ruby.gemspec

@@ -17,10 +17,10 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_dependency('avro_turf', '>= 1.4', '< 2')
   spec.add_dependency('benchmark', '~> 0.5')
   spec.add_dependency('fig_tree', '~> 0.2.0')
   spec.add_dependency('karafka', '~> 2.0')
+  spec.add_dependency('schema_registry_client')
   spec.add_dependency('sigurd', '>= 0.1.0', '< 1.0')
 
   spec.add_development_dependency('activerecord-import')
@@ -33,7 +33,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency('guard-rubocop', '~> 1')
   spec.add_development_dependency('karafka-testing', '~> 2.0')
   spec.add_development_dependency('pg', '~> 1.1')
-  spec.add_development_dependency('proto_turf')
   spec.add_development_dependency('rails', '~> 8.0')
   spec.add_development_dependency('rake', '~> 13')
   spec.add_development_dependency('rspec', '~> 3')
@@ -42,6 +41,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency('rspec-snapshot', '~> 2.0')
   spec.add_development_dependency('rubocop', '~> 1.0')
   spec.add_development_dependency('rubocop-rspec', '3.8')
+  spec.add_development_dependency('schema_registry_client')
   spec.add_development_dependency('sord', '>= 5.0')
   spec.add_development_dependency('sqlite3', '~> 2.7')
   spec.add_development_dependency('steep', '~> 1.0')

data/lib/deimos/backends/kafka.rb

@@ -6,7 +6,7 @@ module Deimos
     class Kafka < Base
       # :nodoc:
       def self.execute(producer_class:, messages:)
-        Deimos.producer_for(producer_class.topic).produce_many_sync(messages)
+        Karafka.producer.produce_many_sync(messages)
       end
     end
   end

data/lib/deimos/backends/kafka_async.rb

@@ -4,10 +4,9 @@ module Deimos
   module Backends
     # Backend which produces to Kafka via an async producer.
     class KafkaAsync < Base
-
       # :nodoc:
       def self.execute(producer_class:, messages:)
-        Deimos.producer_for(producer_class.topic).produce_many_async(messages)
+        Karafka.producer.produce_many_async(messages)
       end
     end
   end

data/lib/deimos/config/configuration.rb

@@ -43,7 +43,7 @@ module Deimos
               'Please provide a directory.'
       end
 
-      Dir["./#{Deimos.config.schema.generated_class_path}/**/*.rb"].sort.
+      Dir["./#{Deimos.config.schema.generated_class_path}/**/*.rb"].
         each { |f| require f }
     rescue LoadError
       raise 'Cannot load schema classes. Please regenerate classes with' \

data/lib/deimos/ext/producer_middleware.rb

@@ -83,7 +83,7 @@ module Deimos
                                     nil
                                   else
                                     encoder.encode(message.payload,
-                                                   topic: "#{Deimos.config.producers.topic_prefix}#{config.name}-value")
+                                                   topic: "#{Deimos.config.producers.topic_prefix}#{config.name}")
                                   end
       end
 
@@ -96,7 +96,7 @@ module Deimos
         if config.deserializers[:key].respond_to?(:encode_key)
           config.deserializers[:key].encode_key(key)
         elsif key
-          config.deserializers[:payload].encode(key)
+          config.deserializers[:payload].encode(key, is_key: true)
         else
           key
         end

data/lib/deimos/kafka_source.rb

@@ -118,7 +118,7 @@ end
           unique_columns = column_names.map(&:to_s) -
                            options[:on_duplicate_key_update].map(&:to_s) - %w(id created_at)
           records = hashes_without_id.map do |hash|
-            self.where(unique_columns.map { |c| [c, hash[c]] }.to_h).first
+            self.where(unique_columns.to_h { |c| [c, hash[c]] }).first
           end
           self.kafka_producers.each { |p| p.send_events(records) }
         else

data/lib/deimos/metrics/datadog.rb

@@ -58,9 +58,7 @@ module Deimos
           end
         end
         Karafka::Setup::Config.setup if Karafka.producer.nil?
-        Deimos.waterdrop_producers.each do |producer|
-          producer.monitor.subscribe(waterdrop_listener)
-        end
+        Karafka.producer.monitor.subscribe(waterdrop_listener)
       end
 
       # :nodoc:

data/lib/deimos/schema_backends/avro_base.rb

@@ -1,9 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'base'
-require 'avro'
-require 'avro_turf'
-require 'avro_turf/mutable_schema_store'
+require 'schema_registry_client'
 require_relative 'avro_schema_coercer'
 
 module Deimos
@@ -15,7 +13,7 @@ module Deimos
       # @override
       def initialize(schema:, namespace:)
         super
-        @schema_store = AvroTurf::MutableSchemaStore.new(path: Deimos.config.schema.path)
+        @schema_store = SchemaRegistry::AvroSchemaStore.new(path: Deimos.config.schema.path)
       end
 
       def supports_key_schemas?
@@ -31,7 +29,7 @@ module Deimos
       def encode_key(key_id, key, topic: nil)
         begin
           @key_schema ||= @schema_store.find("#{@schema}_key")
-        rescue AvroTurf::SchemaNotFoundError
+        rescue SchemaRegistry::SchemaNotFoundError
           @key_schema = generate_key_schema(key_id)
         end
         field_name = _field_name_from_schema(@key_schema)
@@ -184,7 +182,7 @@ module Deimos
       # @return [Avro::Schema]
       def avro_schema(schema=nil)
         schema ||= @schema
-        @schema_store.find(schema, @namespace)
+        @schema_store.find("#{@namespace}.#{schema}")
       end
 
       # @param value_schema [Hash]

data/lib/deimos/schema_backends/avro_local.rb

@@ -8,23 +8,24 @@ module Deimos
     class AvroLocal < AvroBase
       # @override
       def decode_payload(payload, schema:)
-        avro_turf.decode(payload, schema_name: schema, namespace: @namespace)
+        stream = StringIO.new(payload)
+        schema = @schema_store.find("#{@namespace}.#{schema}")
+        reader = Avro::IO::DatumReader.new(nil, schema)
+        Avro::DataFile::Reader.new(stream, reader).first
       end
 
       # @override
-      def encode_payload(payload, schema: nil, topic: nil)
-        avro_turf.encode(payload, schema_name: schema, namespace: @namespace)
-      end
-
-    private
+      def encode_payload(payload, schema: nil, subject: nil)
+        stream = StringIO.new
+        schema = schema_store.find("#{@namespace}.#{schema}")
+        writer = Avro::IO::DatumWriter.new(schema)
 
-      # @return [AvroTurf]
-      def avro_turf
-        @avro_turf ||= AvroTurf.new(
-          schemas_path: Deimos.config.schema.path,
-          schema_store: @schema_store
-        )
+        dw = Avro::DataFile::Writer.new(stream, writer, schema)
+        dw << payload.to_h
+        dw.close
+        stream.string
       end
+
     end
   end
 end

data/lib/deimos/schema_backends/avro_schema_registry.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require_relative 'avro_base'
-require 'avro_turf/messaging'
 
 module Deimos
   module SchemaBackends
@@ -9,27 +8,27 @@ module Deimos
     class AvroSchemaRegistry < AvroBase
       # @override
       def decode_payload(payload, schema:)
-        avro_turf_messaging.decode(payload.to_s, schema_name: schema)
+        schema_registry.decode(payload.to_s)
       end
 
       # @override
-      def encode_payload(payload, schema: nil, topic: nil)
-        avro_turf_messaging.encode(payload, schema_name: schema, subject: topic || schema)
+      def encode_payload(payload, schema: nil, subject: nil)
+        schema_registry.encode(payload, subject: subject || schema, schema_name: "#{@namespace}.#{schema}")
       end
 
     private
 
-      # @return [AvroTurf::Messaging]
-      def avro_turf_messaging
-         @avro_turf_messaging ||= AvroTurf::Messaging.new(
-           schema_store: @schema_store,
-           registry_url: Deimos.config.schema.registry_url,
-           schemas_path: Deimos.config.schema.path,
-           user: Deimos.config.schema.user,
-           password: Deimos.config.schema.password,
-           namespace: @namespace,
-           logger: Karafka.logger
-         )
+      # @return [SchemaRegistry::Client]
+      def schema_registry
+        @schema_registry ||= SchemaRegistry::Client.new(
+          registry_url: Deimos.config.schema.registry_url,
+          logger: Karafka.logger,
+          user: Deimos.config.schema.user,
+          password: Deimos.config.schema.password,
+          schema_type: SchemaRegistry::Schema::Avro
+        )
+        SchemaRegistry.avro_schema_path = Deimos.config.schema.path
+        @schema_registry
       end
     end
   end

data/lib/deimos/schema_backends/avro_validation.rb

@@ -13,7 +13,7 @@ module Deimos
       end
 
       # @override
-      def encode_payload(payload, schema: nil, topic: nil)
+      def encode_payload(payload, schema: nil, subject: nil)
         payload.to_h.with_indifferent_access.to_json
       end
     end

data/lib/deimos/schema_backends/base.rb

@@ -56,9 +56,10 @@ module Deimos
       # @param schema [String,Symbol]
       # @param topic [String]
       # @return [String]
-      def encode(payload, schema: nil, topic: nil)
+      def encode(payload, schema: nil, topic: nil, is_key: false)
         validate(payload, schema: schema || @schema)
-        encode_payload(payload, schema: schema || @schema, topic: topic)
+        subject = is_key ? "#{topic}-key" : "#{topic}-value"
+        encode_payload(payload, schema: schema || @schema, subject: subject)
       end
 
       # Decode a payload with a schema. Public method.
@@ -114,9 +115,9 @@ module Deimos
       # Encode a payload. To be defined by subclass.
       # @param payload [Hash]
       # @param schema [String,Symbol]
-      # @param topic [String]
+      # @param subject [String]
       # @return [String]
-      def encode_payload(_payload, schema:, topic: nil)
+      def encode_payload(_payload, schema:, subject: nil)
         raise MissingImplementationError
       end
 

data/lib/deimos/schema_backends/mock.rb

@@ -15,7 +15,7 @@ module Deimos
       end
 
       # @override
-      def encode_payload(payload, schema:, topic: nil)
+      def encode_payload(payload, schema:, subject: nil)
         payload.is_a?(String) ? 'payload-encoded' : payload.map { |k, v| [k, "encoded-#{v}"] }.to_json
       end
 

data/lib/deimos/schema_backends/plain.rb

@@ -15,7 +15,7 @@ module Deimos
       end
 
       # @override
-      def encode_payload(payload, schema:, topic: nil)
+      def encode_payload(payload, schema:, subject: nil)
         payload.to_s
       end
 

data/lib/deimos/schema_backends/proto_base.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'base'
-require 'proto_turf'
+require 'schema_registry_client'
 
 module Deimos
   module SchemaBackends
@@ -24,26 +24,46 @@ module Deimos
         float: :float,
         message: :record
       }.freeze
+
       def proto_schema(schema=@schema)
         Google::Protobuf::DescriptorPool.generated_pool.lookup(schema)
       end
 
       # @override
       def encode_key(key_id, key, topic: nil)
-        if key.is_a?(Hash)
-          key_id ? key.with_indifferent_access[key_id].to_s : key.sort.to_h.to_json
+        if key.respond_to?(:to_h)
+          hash = if key_id
+                   key_id.to_s.split('.')[...-1].each do |k|
+                     key = key.with_indifferent_access[k]
+                   end
+                   key.to_h.with_indifferent_access.slice(key_id.split('.').last)
+                 else
+                   key.to_h.sort.to_h
+                 end
+          self.encode_proto_key(hash, topic: topic, field: key_id)
+        elsif key_id
+          hash = { key_id.to_s.split('.').last => key }
+          self.encode_proto_key(hash, topic: topic, field: key_id)
         else
           key.to_s
         end
       end
 
+      # @param hash [Hash]
+      # @return [String]
+      def encode_proto_key(hash, topic: nil)
+        hash.sort.to_h.to_json
+      end
+
+      def decode_proto_key(payload)
+        JSON.parse(payload)
+      rescue StandardError
+        payload
+      end
+
       # @override
       def decode_key(payload, key_id)
-        val = begin
-                JSON.parse(payload)
-        rescue StandardError
-                payload
-        end
+        val = decode_proto_key(payload)
         key_id ? val[key_id.to_s] : val
       end
 
@@ -85,8 +105,12 @@ module Deimos
         :mock
       end
 
-      def generate_key_schema(_field_name)
-        raise 'Protobuf cannot generate key schemas! Please use field_config :plain'
+      def generate_key_schema(field_name)
+        SchemaRegistry::Output::JsonSchema.output(proto_schema, field: field_name)
+      end
+
+      def supports_key_schemas?
+        true
       end
 
     end