deimos-ruby 2.3.0 → 2.4.0.pre.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9811f05eec0d9b6777e2ed047a2e81b0a23bd770b52866bf8c8a79f84c8ec586
-  data.tar.gz: c84cba2ecf750f4349ba86e47ddab2d01a4dd8e5f8fd9ab008b3fcfeb14b1425
+  metadata.gz: bc23c10fe732ab1eb2b6c0e2daf5bdb943a939510624b821456d0d383224f579
+  data.tar.gz: 84fdb7e495d76bb27b0592572f13249a7c8e28c3d123c68d15ac5d30ff883421
 SHA512:
-  metadata.gz: b8319ada2f2b715e750e7de5caf6c6117d6154da6277988e12851f420c69a2f3adc0ab30594a23bca86ec8ffa57ef2cc3e5ab9758421e26fbe91cf682ad28e04
-  data.tar.gz: 2174b8d776b547689422d6af381f423680d61df368be813e53f213eb8d084971b338a3382d8775e07ef324b9443fc1f5427fed834c54629d52c88728d016fd49
+  metadata.gz: 9b47d42b33e3c66ed3fd852570490e6a719d55c6709b706b38b58144a192b40a5edef6c6b5be516c89c3798069ea22f6be2134e1000f41ba3110f92b5face15d
+  data.tar.gz: 3fea215d836fbcc1e10df7971e7112194ada61a95dab4be1524cafc275d523b7ee186887298a0a89e162945aeaf2b83d5826031844e9b26dd25039d7797d6c75

data/.rubocop.yml

@@ -4,7 +4,7 @@ plugins:
   - rubocop-rspec
 
 AllCops:
-  TargetRubyVersion: 2.5
+  TargetRubyVersion: 3.0
   Exclude:
     - lib/deimos/monkey_patches/*.rb
     - spec/gen/**/*.rb

data/CHANGELOG.md

@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+- Major change: Switch from using `avro_turf` and `proto_turf` to use `schema_registry_client`, which handles both Avro and Protobuf.
+
 # 2.3.0 - 2026-01-13
 
 - Feature: Support broker setting per topic in producer configs.

data/CLAUDE.md

@@ -0,0 +1,270 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Overview
+
+Deimos is a Ruby framework for Kafka development that marries Kafka with schema definitions (Avro/Protobuf), ActiveRecord, and provides a comprehensive toolbox. Built on top of Karafka (which itself builds on RDKafka), it provides schema encoding/decoding, database integration, metrics, tracing, and test helpers.
+
+## Development Commands
+
+### Testing
+
+```bash
+# Run all tests
+bundle exec rspec
+
+# Run a single test file
+bundle exec rspec spec/path/to/file_spec.rb
+
+# Run a specific test
+bundle exec rspec spec/path/to/file_spec.rb:LINE_NUMBER
+```
+
+### Linting
+
+```bash
+# Run Rubocop linter
+bundle exec rubocop
+
+# Auto-correct issues
+bundle exec rubocop -a
+```
+
+### Schema Class Generation
+
+When modifying schema-related code, you may need to regenerate test schema classes:
+
+```bash
+# Regenerate test schema classes (used after schema changes)
+bundle exec ./regenerate_test_schema_classes.rb
+
+# Generate Protobuf classes
+protoc -I spec/protos --ruby_out=spec/gen --ruby_opt=paths=source_relative spec/protos/**/*.proto
+```
+
+### Rake Tasks
+
+```bash
+# Start Deimos consumer (in Rails environment)
+rake deimos:start
+
+# Start outbox backend producer
+rake deimos:outbox
+
+# Start database poller
+rake deimos:db_poller
+
+# Generate schema classes
+rake deimos:generate_schema_classes
+```
+
+## Architecture
+
+### Layer Structure
+
+Deimos is built in layers:
+1. **RDKafka** - Low-level Kafka client providing producer/consumer APIs
+2. **Karafka** - Threaded consumers wrapper with lifecycle management
+3. **Deimos** - Full framework with schema integration, database features, metrics, and utilities
+
+### Key Directories
+
+- `lib/deimos/` - Core Deimos code
+- `lib/deimos/backends/` - Producer backends (kafka, kafka_async, outbox, test)
+- `lib/deimos/schema_backends/` - Schema handlers (Avro local/registry/validation, Protobuf, plain, mock)
+- `lib/deimos/metrics/` - Metrics providers (DataDog, mock)
+- `lib/deimos/tracing/` - Tracing providers (DataDog, mock)
+- `lib/deimos/utils/` - Utilities (DB poller, outbox producer, etc.)
+- `lib/deimos/config/` - Configuration classes
+- `lib/deimos/ext/` - Extensions to Karafka routing
+- `lib/generators/` - Rails generators for migrations
+- `lib/tasks/` - Rake tasks
+
+### Core Concepts
+
+#### Schema Backends
+
+Schema backends encode/decode payloads. All backends must implement:
+- `encode(payload, topic:)` - Encode payload to binary/string format
+- `decode(payload)` - Decode binary/string to hash
+- `validate(payload)` - Validate payload against schema
+- `coerce(payload)` - Coerce payload to match schema types
+- `schema_fields` - List fields in schema (used with ActiveRecord)
+- Define a `mock` backend for testing
+
+Available backends: `:avro_local`, `:avro_schema_registry`, `:avro_validation`, `:proto_schema_registry`, `:proto_local`, `:mock`, `:plain`
+
+#### Producer Backends
+
+Producer backends determine how messages are sent. All backends inherit from `Deimos::Backends::Base` and implement `execute(messages)`.
+
+Available backends:
+- `:kafka` - Send directly to Kafka (default)
+- `:kafka_async` - Async variant of kafka backend
+- `:outbox` - Transactional outbox pattern (save to DB, send async)
+- `:test` - For testing (stores messages in memory)
+
+#### Consumer Types
+
+- `Deimos::Consumer` - Base consumer class
+  - Per-message: Override `consume_message(message)` and set `each_message true`
+  - Batch: Override `consume_batch` (receives `messages` collection)
+
+- `Deimos::ActiveRecordConsumer` - Automatically saves/updates ActiveRecord models
+  - Per-message mode: Uses `fetch_record`, `assign_key`, `destroy_record`
+  - Batch mode: Uses `activerecord-import` for bulk operations
+  - Override `record_attributes(payload, key)` to customize attributes
+
+#### Producers
+
+- `Deimos::Producer` - Base producer class
+  - Call `self.produce([{payload: ..., key: ..., topic: ...}])`
+  - Override `partition_key(payload)` for custom partitioning
+
+- `Deimos::ActiveRecordProducer` - Produces from ActiveRecord models
+  - Set `record_class Widget, refetch: false`
+  - Override `generate_payload(attributes, record)` to customize payload
+  - Override `watched_attributes(record)` to add non-schema fields
+
+#### Key Configuration
+
+Every producer must define `key_config`:
+- `key_config none: true` - No keys (events)
+- `key_config plain: true` - Unencoded keys (legacy)
+- `key_config schema: 'MySchema-key'` - Use existing key schema
+- `key_config field: 'my_id'` - Auto-generate key schema from value field
+
+#### KafkaSource Mixin
+
+The `Deimos::KafkaSource` mixin adds callbacks to ActiveRecord models to automatically send Kafka messages on save/destroy:
+
+```ruby
+class Widget < ActiveRecord::Base
+  include Deimos::KafkaSource
+
+  def self.kafka_producers
+    [MyProducer]
+  end
+
+  def self.kafka_config
+    { update: true, delete: true, import: true, create: true }
+  end
+end
+```
+
+#### Outbox Pattern (Database Backend)
+
+The outbox pattern provides transactional guarantees:
+1. Messages are validated, encoded, and saved to `kafka_messages` table
+2. Separate thread pool (via `Deimos::Utils::OutboxProducer`) reads from DB and sends to Kafka
+3. Uses `kafka_topic_info` table for topic-level locking
+4. Runs via `rake deimos:outbox` or `Deimos.start_outbox_backend!(thread_count: N)`
+
+#### Database Poller
+
+Polls database tables and produces messages:
+1. Configure with `Deimos.configure { db_poller { producer_class 'MyProducer' } }`
+2. Two modes:
+   - Time-based (default): Uses `updated_at` and `id` columns
+   - State-based: Updates state column after publishing
+3. Tracks progress in `poll_info` table
+4. Runs via `rake deimos:db_poller`
+
+### Configuration
+
+Configuration uses the `fig_tree` gem. See `lib/deimos/config/configuration.rb` for the schema. Configure via:
+
+```ruby
+Deimos.configure do |config|
+  config.producers.backend = :outbox
+  config.schema.backend = :avro_schema_registry
+  config.schema.registry_url = 'http://localhost:8081'
+end
+```
+
+### Karafka Integration
+
+Deimos extends Karafka routing with schema configuration:
+
+```ruby
+Karafka::App.routes.draw do
+  topic 'my-topic' do
+    consumer MyConsumer
+    schema 'MySchema'
+    namespace 'com.my-namespace'
+    key_config field: 'test_id'
+  end
+end
+```
+
+The `lib/deimos/ext/` directory contains the routing extensions that add schema-aware capabilities to Karafka's routing DSL.
+
+### Shared Configuration
+
+Both producers and consumers use the `SharedConfig` module to standardize schema settings, topic configuration, key handling, etc.
+
+### Testing
+
+Test helpers in `lib/deimos/test_helpers.rb` provide:
+- `test_consume_message(consumer, payload)` - Test message consumption
+- `test_consume_batch(consumer, payloads)` - Test batch consumption
+- `expect(topic).to have_sent(payload, key, partition_key, headers)` - Assert messages sent
+- `Deimos::TestHelpers.sent_messages` - Inspect sent messages
+
+Configure for tests:
+```ruby
+Deimos.config.schema.backend = :avro_validation  # Validates but doesn't encode
+Deimos.config.producers.backend = :test  # Stores in memory
+```
+
+### Instrumentation
+
+Deimos sends events through Karafka's instrumentation monitor:
+- `deimos.encode_message` - Schema encoding
+- `deimos.outbox.produce` - Outbox messages processed
+- `deimos.ar_consumer.consume_batch` - ActiveRecord batch consumption
+- `deimos.batch_consumption.valid_records` - Valid records upserted
+- `deimos.batch_consumption.invalid_records` - Invalid records rejected
+
+Subscribe with: `Karafka.monitor.subscribe('event.name') { |event| ... }`
+
+## Database Integration
+
+Deimos uses two primary database tables (created via generators):
+
+- `kafka_messages` - Stores outbox messages (topic, key, payload, partition_key)
+- `kafka_topic_info` - Locks topics for outbox processing
+- `poll_info` - Tracks database poller progress
+
+## Special Considerations
+
+### Schema Changes
+
+When adding/modifying schemas:
+1. Update schema files in configured schema path
+2. Run `rake deimos:generate_schema_classes` if using schema classes
+3. Regenerate test schemas with `./regenerate_test_schema_classes.rb`
+
+### Protobuf
+
+- Requires `schema_registry_client` gem in Gemfile
+- Protobuf payloads must be Protobuf message objects, not hashes
+- Protobuf should NOT be used for keys (unstable binary encoding)
+
+### ActiveRecord Batch Consumers
+
+- Only supports primary keys as identifiers by default
+- Skips ActiveRecord callbacks (uses `activerecord-import` for bulk SQL)
+- Set `compacted false` to process all messages vs. only last per key
+- Requires `bulk_import_id_column` config when saving to multiple tables
+
+### Error Handling
+
+- Set `config.consumers.reraise_errors = false` to swallow non-fatal errors
+- Define fatal errors via `config.fatal_error` (global) or `fatal_error?(exception, payload, metadata)` (per-consumer)
+- Prevents consumer from getting stuck on bad messages
+
+## Integration Tests
+
+Integration tests run against real databases (PostgreSQL, MySQL/Trilogy, SQLite). They are marked with `:integration` metadata and use different database configs from `DbConfigs::DB_OPTIONS`.

data/README.md

@@ -91,12 +91,11 @@ Currently we have the following possible schema backends:
 * Avro Schema Registry (use the Confluent Schema Registry)
 * Avro Validation (validate using an Avro schema but leave decoded - this is useful
   for unit testing and development)
+* Protobuf Local (use pure Protobuf)
 * Protobuf Schema Registry (use Protobuf with the Confluent Schema Registry)
 * Mock (no actual encoding/decoding).
 
-Note that to use Protobuf, you must include the [proto_turf](https://github.com/flipp-oss/proto_turf) gem in your Gemfile.
-
-Other possible schemas could [JSONSchema](https://json-schema.org/), etc. Feel free to
+Other possible schemas could include [JSONSchema](https://json-schema.org/), etc. Feel free to
 contribute!
 
 To create a new schema backend, please see the existing examples [here](lib/deimos/schema_backends).
@@ -283,7 +282,7 @@ MyProducer.publish({
 ```
 
 > [!IMPORTANT]
-> Protobuf should *not* be used as a key schema, since the binary encoding is [unstable](https://protobuf.dev/programming-guides/encoding/#implications) and may break partitioning. Deimos will automatically convert key fields to plain values and key hashes to JSON.
+> Protobuf should *not* be used as a key schema, since the binary encoding is [unstable](https://protobuf.dev/programming-guides/encoding/#implications) and may break partitioning. Deimos will automatically convert keys to sorted JSON, and will use JSON Schema in the schema registry.
 
 ## Instrumentation
 
@@ -1012,7 +1011,13 @@ end
 # test can have the same settings every time it is run
 after(:each) do
   Deimos.config.reset!
-  Deimos.config.schema.backend = :avro_validation
+  # set specific settings here
+  Deimos.config.schema.path = 'my/schema/path'
+end
+
+around(:each) do |ex|
+  # replace e.g. avro_schema_registry with avro_validation, proto_schema_registry with proto_local
+  Deimos::TestHelpers.with_mock_backends { ex.run }
 end
 ```
 

data/deimos-ruby.gemspec

@@ -17,10 +17,10 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_dependency('avro_turf', '>= 1.4', '< 2')
   spec.add_dependency('benchmark', '~> 0.5')
   spec.add_dependency('fig_tree', '~> 0.2.0')
   spec.add_dependency('karafka', '~> 2.0')
+  spec.add_dependency('schema_registry_client')
   spec.add_dependency('sigurd', '>= 0.1.0', '< 1.0')
 
   spec.add_development_dependency('activerecord-import')
@@ -33,7 +33,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency('guard-rubocop', '~> 1')
   spec.add_development_dependency('karafka-testing', '~> 2.0')
   spec.add_development_dependency('pg', '~> 1.1')
-  spec.add_development_dependency('proto_turf')
   spec.add_development_dependency('rails', '~> 8.0')
   spec.add_development_dependency('rake', '~> 13')
   spec.add_development_dependency('rspec', '~> 3')
@@ -42,6 +41,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency('rspec-snapshot', '~> 2.0')
   spec.add_development_dependency('rubocop', '~> 1.0')
   spec.add_development_dependency('rubocop-rspec', '3.8')
+  spec.add_development_dependency('schema_registry_client')
   spec.add_development_dependency('sord', '>= 5.0')
   spec.add_development_dependency('sqlite3', '~> 2.7')
   spec.add_development_dependency('steep', '~> 1.0')

data/lib/deimos/config/configuration.rb

@@ -43,7 +43,7 @@ module Deimos
               'Please provide a directory.'
       end
 
-      Dir["./#{Deimos.config.schema.generated_class_path}/**/*.rb"].sort.
+      Dir["./#{Deimos.config.schema.generated_class_path}/**/*.rb"].
         each { |f| require f }
     rescue LoadError
       raise 'Cannot load schema classes. Please regenerate classes with' \

data/lib/deimos/ext/producer_middleware.rb

@@ -83,7 +83,7 @@ module Deimos
                                     nil
                                   else
                                     encoder.encode(message.payload,
-                                                   topic: "#{Deimos.config.producers.topic_prefix}#{config.name}-value")
+                                                   topic: "#{Deimos.config.producers.topic_prefix}#{config.name}")
                                   end
       end
 
@@ -96,7 +96,7 @@ module Deimos
         if config.deserializers[:key].respond_to?(:encode_key)
           config.deserializers[:key].encode_key(key)
         elsif key
-          config.deserializers[:payload].encode(key)
+          config.deserializers[:payload].encode(key, is_key: true)
         else
           key
         end

data/lib/deimos/kafka_source.rb

@@ -118,7 +118,7 @@ end
           unique_columns = column_names.map(&:to_s) -
                            options[:on_duplicate_key_update].map(&:to_s) - %w(id created_at)
           records = hashes_without_id.map do |hash|
-            self.where(unique_columns.map { |c| [c, hash[c]] }.to_h).first
+            self.where(unique_columns.to_h { |c| [c, hash[c]] }).first
           end
           self.kafka_producers.each { |p| p.send_events(records) }
         else

data/lib/deimos/message.rb

@@ -38,10 +38,19 @@ module Deimos
     def add_fields(fields)
       return if @payload.to_h.with_indifferent_access.except(:payload_key, :partition_key).blank?
 
-      if fields.include?('message_id')
+      if @payload.respond_to?(:message_id)
+        if fields.include?('message_id') && @payload.message_id.blank?
+          @payload.message_id = SecureRandom.uuid
+        end
+      elsif fields.include?('message_id')
         @payload['message_id'] ||= SecureRandom.uuid
       end
-      if fields.include?('timestamp')
+
+      if @payload.respond_to?(:timestamp)
+        if fields.include?('timestamp') && @payload.timestamp.blank?
+          @payload.timestamp = Time.now.in_time_zone.to_s
+        end
+      elsif fields.include?('timestamp')
         @payload['timestamp'] ||= Time.now.in_time_zone.to_s
       end
     end

data/lib/deimos/schema_backends/avro_base.rb

@@ -1,9 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'base'
-require 'avro'
-require 'avro_turf'
-require 'avro_turf/mutable_schema_store'
+require 'schema_registry_client'
 require_relative 'avro_schema_coercer'
 
 module Deimos
@@ -15,7 +13,7 @@ module Deimos
       # @override
       def initialize(schema:, namespace:)
         super
-        @schema_store = AvroTurf::MutableSchemaStore.new(path: Deimos.config.schema.path)
+        @schema_store = SchemaRegistry::AvroSchemaStore.new(path: Deimos.config.schema.path)
       end
 
       def supports_key_schemas?
@@ -31,7 +29,7 @@ module Deimos
       def encode_key(key_id, key, topic: nil)
         begin
           @key_schema ||= @schema_store.find("#{@schema}_key")
-        rescue AvroTurf::SchemaNotFoundError
+        rescue SchemaRegistry::SchemaNotFoundError
           @key_schema = generate_key_schema(key_id)
         end
         field_name = _field_name_from_schema(@key_schema)
@@ -184,7 +182,7 @@ module Deimos
       # @return [Avro::Schema]
       def avro_schema(schema=nil)
         schema ||= @schema
-        @schema_store.find(schema, @namespace)
+        @schema_store.find("#{@namespace}.#{schema}")
       end
 
       # @param value_schema [Hash]

data/lib/deimos/schema_backends/avro_local.rb

@@ -8,23 +8,24 @@ module Deimos
     class AvroLocal < AvroBase
       # @override
       def decode_payload(payload, schema:)
-        avro_turf.decode(payload, schema_name: schema, namespace: @namespace)
+        stream = StringIO.new(payload)
+        schema = @schema_store.find("#{@namespace}.#{schema}")
+        reader = Avro::IO::DatumReader.new(nil, schema)
+        Avro::DataFile::Reader.new(stream, reader).first
       end
 
       # @override
-      def encode_payload(payload, schema: nil, topic: nil)
-        avro_turf.encode(payload, schema_name: schema, namespace: @namespace)
-      end
-
-    private
+      def encode_payload(payload, schema: nil, subject: nil)
+        stream = StringIO.new
+        schema = schema_store.find("#{@namespace}.#{schema}")
+        writer = Avro::IO::DatumWriter.new(schema)
 
-      # @return [AvroTurf]
-      def avro_turf
-        @avro_turf ||= AvroTurf.new(
-          schemas_path: Deimos.config.schema.path,
-          schema_store: @schema_store
-        )
+        dw = Avro::DataFile::Writer.new(stream, writer, schema)
+        dw << payload.to_h
+        dw.close
+        stream.string
       end
+
     end
   end
 end

data/lib/deimos/schema_backends/avro_schema_registry.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require_relative 'avro_base'
-require 'avro_turf/messaging'
 
 module Deimos
   module SchemaBackends
@@ -9,27 +8,27 @@ module Deimos
     class AvroSchemaRegistry < AvroBase
       # @override
       def decode_payload(payload, schema:)
-        avro_turf_messaging.decode(payload.to_s, schema_name: schema)
+        schema_registry.decode(payload.to_s)
       end
 
       # @override
-      def encode_payload(payload, schema: nil, topic: nil)
-        avro_turf_messaging.encode(payload, schema_name: schema, subject: topic || schema)
+      def encode_payload(payload, schema: nil, subject: nil)
+        schema_registry.encode(payload, subject: subject || schema, schema_name: "#{@namespace}.#{schema}")
       end
 
     private
 
-      # @return [AvroTurf::Messaging]
-      def avro_turf_messaging
-         @avro_turf_messaging ||= AvroTurf::Messaging.new(
-           schema_store: @schema_store,
-           registry_url: Deimos.config.schema.registry_url,
-           schemas_path: Deimos.config.schema.path,
-           user: Deimos.config.schema.user,
-           password: Deimos.config.schema.password,
-           namespace: @namespace,
-           logger: Karafka.logger
-         )
+      # @return [SchemaRegistry::Client]
+      def schema_registry
+        @schema_registry ||= SchemaRegistry::Client.new(
+          registry_url: Deimos.config.schema.registry_url,
+          logger: Karafka.logger,
+          user: Deimos.config.schema.user,
+          password: Deimos.config.schema.password,
+          schema_type: SchemaRegistry::Schema::Avro
+        )
+        SchemaRegistry.avro_schema_path = Deimos.config.schema.path
+        @schema_registry
       end
     end
   end

data/lib/deimos/schema_backends/avro_validation.rb

@@ -13,7 +13,7 @@ module Deimos
       end
 
       # @override
-      def encode_payload(payload, schema: nil, topic: nil)
+      def encode_payload(payload, schema: nil, subject: nil)
         payload.to_h.with_indifferent_access.to_json
       end
     end

data/lib/deimos/schema_backends/base.rb

@@ -56,9 +56,10 @@ module Deimos
       # @param schema [String,Symbol]
       # @param topic [String]
       # @return [String]
-      def encode(payload, schema: nil, topic: nil)
+      def encode(payload, schema: nil, topic: nil, is_key: false)
         validate(payload, schema: schema || @schema)
-        encode_payload(payload, schema: schema || @schema, topic: topic)
+        subject = is_key ? "#{topic}-key" : "#{topic}-value"
+        encode_payload(payload, schema: schema || @schema, subject: subject)
       end
 
       # Decode a payload with a schema. Public method.
@@ -114,9 +115,9 @@ module Deimos
       # Encode a payload. To be defined by subclass.
       # @param payload [Hash]
       # @param schema [String,Symbol]
-      # @param topic [String]
+      # @param subject [String]
       # @return [String]
-      def encode_payload(_payload, schema:, topic: nil)
+      def encode_payload(_payload, schema:, subject: nil)
         raise MissingImplementationError
       end
 

data/lib/deimos/schema_backends/mock.rb

@@ -15,7 +15,7 @@ module Deimos
       end
 
       # @override
-      def encode_payload(payload, schema:, topic: nil)
+      def encode_payload(payload, schema:, subject: nil)
         payload.is_a?(String) ? 'payload-encoded' : payload.map { |k, v| [k, "encoded-#{v}"] }.to_json
       end
 

data/lib/deimos/schema_backends/plain.rb

@@ -15,7 +15,7 @@ module Deimos
       end
 
       # @override
-      def encode_payload(payload, schema:, topic: nil)
+      def encode_payload(payload, schema:, subject: nil)
         payload.to_s
       end