deimos-ruby 2.3.5 → 2.4.0.pre.beta1

data/lib/deimos/schema_backends/avro_schema_registry.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require_relative 'avro_base'
-require 'avro_turf/messaging'
 
 module Deimos
   module SchemaBackends
@@ -9,27 +8,27 @@ module Deimos
     class AvroSchemaRegistry < AvroBase
       # @override
       def decode_payload(payload, schema:)
-        avro_turf_messaging.decode(payload.to_s, schema_name: schema)
+        schema_registry.decode(payload.to_s)
       end
 
       # @override
-      def encode_payload(payload, schema: nil, topic: nil)
-        avro_turf_messaging.encode(payload, schema_name: schema, subject: topic || schema)
+      def encode_payload(payload, schema: nil, subject: nil)
+        schema_registry.encode(payload, subject: subject || schema, schema_name: "#{@namespace}.#{schema}")
       end
 
     private
 
-      # @return [AvroTurf::Messaging]
-      def avro_turf_messaging
-         @avro_turf_messaging ||= AvroTurf::Messaging.new(
-           schema_store: @schema_store,
-           registry_url: Deimos.config.schema.registry_url,
-           schemas_path: Deimos.config.schema.path,
-           user: Deimos.config.schema.user,
-           password: Deimos.config.schema.password,
-           namespace: @namespace,
-           logger: Karafka.logger
-         )
+      # @return [SchemaRegistry::Client]
+      def schema_registry
+        @schema_registry ||= SchemaRegistry::Client.new(
+          registry_url: Deimos.config.schema.registry_url,
+          logger: Karafka.logger,
+          user: Deimos.config.schema.user,
+          password: Deimos.config.schema.password,
+          schema_type: SchemaRegistry::Schema::Avro
+        )
+        SchemaRegistry.avro_schema_path = Deimos.config.schema.path
+        @schema_registry
       end
     end
   end

data/lib/deimos/schema_backends/avro_validation.rb

@@ -13,7 +13,7 @@ module Deimos
       end
 
       # @override
-      def encode_payload(payload, schema: nil, topic: nil)
+      def encode_payload(payload, schema: nil, subject: nil)
         payload.to_h.with_indifferent_access.to_json
       end
     end

data/lib/deimos/schema_backends/base.rb

@@ -56,9 +56,10 @@ module Deimos
       # @param schema [String,Symbol]
       # @param topic [String]
       # @return [String]
-      def encode(payload, schema: nil, topic: nil)
+      def encode(payload, schema: nil, topic: nil, is_key: false)
         validate(payload, schema: schema || @schema)
-        encode_payload(payload, schema: schema || @schema, topic: topic)
+        subject = is_key ? "#{topic}-key" : "#{topic}-value"
+        encode_payload(payload, schema: schema || @schema, subject: subject)
       end
 
       # Decode a payload with a schema. Public method.
@@ -114,9 +115,9 @@ module Deimos
       # Encode a payload. To be defined by subclass.
       # @param payload [Hash]
       # @param schema [String,Symbol]
-      # @param topic [String]
+      # @param subject [String]
       # @return [String]
-      def encode_payload(_payload, schema:, topic: nil)
+      def encode_payload(_payload, schema:, subject: nil)
         raise MissingImplementationError
       end
 

data/lib/deimos/schema_backends/mock.rb

@@ -15,7 +15,7 @@ module Deimos
       end
 
       # @override
-      def encode_payload(payload, schema:, topic: nil)
+      def encode_payload(payload, schema:, subject: nil)
         payload.is_a?(String) ? 'payload-encoded' : payload.map { |k, v| [k, "encoded-#{v}"] }.to_json
       end
 

data/lib/deimos/schema_backends/plain.rb

@@ -15,7 +15,7 @@ module Deimos
       end
 
       # @override
-      def encode_payload(payload, schema:, topic: nil)
+      def encode_payload(payload, schema:, subject: nil)
         payload.to_s
       end
 

data/lib/deimos/schema_backends/proto_base.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'base'
-require 'proto_turf'
+require 'schema_registry_client'
 
 module Deimos
   module SchemaBackends
@@ -24,26 +24,51 @@ module Deimos
         float: :float,
         message: :record
       }.freeze
+
       def proto_schema(schema=@schema)
-        Google::Protobuf::DescriptorPool.generated_pool.lookup(schema)
+        proto = Google::Protobuf::DescriptorPool.generated_pool.lookup(schema)
+        if proto.nil?
+          raise "Could not find Protobuf schema '#{schema}'."
+        end
+
+        proto
       end
 
       # @override
       def encode_key(key_id, key, topic: nil)
-        if key.is_a?(Hash)
-          key_id ? key.with_indifferent_access[key_id].to_s : key.sort.to_h.to_json
+        if key.respond_to?(:to_h)
+          hash = if key_id
+                   key_id.to_s.split('.')[...-1].each do |k|
+                     key = key.with_indifferent_access[k]
+                   end
+                   key.to_h.with_indifferent_access.slice(key_id.split('.').last)
+                 else
+                   key.to_h.sort.to_h
+                 end
+          self.encode_proto_key(hash, topic: topic, field: key_id)
+        elsif key_id
+          hash = { key_id.to_s.split('.').last => key }
+          self.encode_proto_key(hash, topic: topic, field: key_id)
         else
           key.to_s
         end
       end
 
+      # @param hash [Hash]
+      # @return [String]
+      def encode_proto_key(hash, topic: nil)
+        hash.sort.to_h.to_json
+      end
+
+      def decode_proto_key(payload)
+        JSON.parse(payload)
+      rescue StandardError
+        payload
+      end
+
       # @override
       def decode_key(payload, key_id)
-        val = begin
-                JSON.parse(payload)
-        rescue StandardError
-                payload
-        end
+        val = decode_proto_key(payload)
         key_id ? val[key_id.to_s] : val
       end
 
@@ -85,8 +110,8 @@ module Deimos
         :mock
       end
 
-      def generate_key_schema(_field_name)
-        raise 'Protobuf cannot generate key schemas! Please use field_config :plain'
+      def supports_key_schemas?
+        false
       end
 
     end

data/lib/deimos/schema_backends/proto_local.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'proto_base'
-require 'proto_turf'
+require 'schema_registry_client'
 
 module Deimos
   module SchemaBackends
@@ -14,14 +14,14 @@ module Deimos
       end
 
       # @override
-      def encode_payload(payload, schema: nil, topic: nil)
+      def encode_payload(payload, schema: nil, subject: nil)
         msg = payload.is_a?(Hash) ? proto_schema.msgclass.new(**payload) : payload
         proto_schema.msgclass.encode(msg)
       end
 
-      # @return [ProtoTurf]
-      def self.proto_turf
-        @proto_turf ||= ProtoTurf.new(
+      # @return [SchemaRegistry::Client]
+      def self.schema_registry
+        @schema_registry ||= SchemaRegistry::Client.new(
           registry_url: Deimos.config.schema.registry_url,
           logger: Karafka.logger
         )

data/lib/deimos/schema_backends/proto_schema_registry.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'proto_base'
-require 'proto_turf'
+require 'schema_registry_client'
 
 module Deimos
   module SchemaBackends
@@ -10,22 +10,47 @@ module Deimos
 
       # @override
       def decode_payload(payload, schema:)
-        self.class.proto_turf.decode(payload)
+        self.class.schema_registry.decode(payload)
       end
 
       # @override
-      def encode_payload(payload, schema: nil, topic: nil)
+      def encode_payload(payload, schema: nil, subject: nil)
         msg = payload.is_a?(Hash) ? proto_schema.msgclass.new(**payload) : payload
-        self.class.proto_turf.encode(msg, subject: topic)
+        encoder = subject&.ends_with?('-key') ? self.class.key_schema_registry : self.class.schema_registry
+        encoder.encode(msg, subject: subject)
       end
 
-      # @return [ProtoTurf]
-      def self.proto_turf
-        @proto_turf ||= ProtoTurf.new(
+      # @override
+      def encode_proto_key(key, topic: nil, field: nil)
+        schema_text = SchemaRegistry::Output::JsonSchema.output(proto_schema.to_proto, path: field)
+        self.class.key_schema_registry.encode(key, subject: "#{topic}-key", schema_text: schema_text)
+      end
+
+      # @override
+      def decode_proto_key(payload)
+        self.class.key_schema_registry.decode(payload)
+      end
+
+      # @return [SchemaRegistry::Client]
+      def self.schema_registry
+        @schema_registry ||= SchemaRegistry::Client.new(
           registry_url: Deimos.config.schema.registry_url,
+          user: Deimos.config.schema.user,
+          password: Deimos.config.schema.password,
           logger: Karafka.logger
         )
       end
+
+      def self.key_schema_registry
+        @key_schema_registry ||= SchemaRegistry::Client.new(
+          registry_url: Deimos.config.schema.registry_url,
+          user: Deimos.config.schema.user,
+          password: Deimos.config.schema.password,
+          logger: Karafka.logger,
+          schema_type: SchemaRegistry::Schema::ProtoJsonSchema
+        )
+      end
+
     end
   end
 end

data/lib/deimos/transcoder.rb

@@ -36,7 +36,7 @@ module Deimos
       if self.key_field
         self.backend.encode_key(self.key_field, key, topic: @topic)
       else
-        self.backend.encode(key, topic: @topic)
+        self.backend.encode(key, topic: @topic, is_key: true)
       end
     end
 

data/lib/deimos/utils/outbox_producer.rb

@@ -205,7 +205,7 @@ module Deimos
         batch_size = batch.size
         current_index = 0
 
-        batch[current_index..-1].in_groups_of(batch_size, false).each do |group|
+        batch[current_index..].in_groups_of(batch_size, false).each do |group|
           @logger.debug("Publishing #{group.size} messages to #{@current_topic}")
           Deimos.producer_for(@current_topic).produce_many_sync(group)
           current_index += group.size

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '2.3.5'
+  VERSION = '2.4.0-beta1'
 end

data/lib/generators/deimos/active_record_generator.rb

@@ -48,7 +48,7 @@ module Deimos
         # @return [String]
         def schema
           last_dot = self.full_schema.rindex('.')
-          self.full_schema[(last_dot + 1)..-1]
+          self.full_schema[(last_dot + 1)..]
         end
 
         # @return [String]

data/lib/generators/deimos/schema_class_generator.rb

@@ -13,7 +13,7 @@ module Deimos
       # @return [Array<Symbol>]
       SPECIAL_TYPES = %i(record enum).freeze
       # @return [String]
-      INITIALIZE_WHITESPACE = "\n#{' ' * 19}"
+      INITIALIZE_WHITESPACE = "\n#{' ' * 19}".freeze
       # @return [Array<String>]
       IGNORE_DEFAULTS = %w(message_id timestamp).freeze
       # @return [String]
@@ -192,7 +192,7 @@ module Deimos
 
       def generate_from_schema_files(found_schemas)
         path = Deimos.config.schema.path || Deimos.config.schema.paths[:avro].first
-        schema_store = AvroTurf::MutableSchemaStore.new(path: path)
+        schema_store = SchemaRegistry::AvroSchemaStore.new(path: path)
         schema_store.load_schemas!
         schema_store.schemas.values.sort_by { |s| "#{s.namespace}#{s.name}" }.each do |schema|
           name = "#{schema.namespace}.#{schema.name}"
@@ -265,7 +265,7 @@ module Deimos
         end
 
         result = "def initialize(_from_message: false, #{arguments.first}"
-        arguments[1..-1].each_with_index do |arg, _i|
+        arguments[1..].each_with_index do |arg, _i|
           result += ",#{INITIALIZE_WHITESPACE}#{arg}"
         end
         "#{result})"

data/lib/generators/deimos/v2_generator.rb

@@ -119,7 +119,7 @@ module Deimos
       end
 
       def consumer_configs
-        deimos_config.consumer_objects.group_by(&:group_id).map { |group_id, consumers|
+        deimos_config.consumer_objects.group_by(&:group_id).to_h do |group_id, consumers|
           [group_id, consumers.map do |consumer|
             kafka_configs = {}
             kafka_configs['auto.offset.reset'] = consumer.start_from_beginning ? 'earliest' : 'latest'
@@ -159,7 +159,7 @@ module Deimos
             configs[:each_message] = true unless consumer.delivery.to_s == 'inline_batch'
             configs
           end]
-        }.to_h
+        end
       end
 
       def producer_configs

data/spec/active_record_batch_consumer_association_spec.rb

@@ -70,7 +70,7 @@ module ActiveRecordBatchConsumerTest # rubocop:disable Metrics/ModuleLength
     end
 
     before(:each) do
-      ActiveRecord::Base.connection.truncate_tables(:widgets, :details, :locales)
+      ActiveRecord::Base.connection.truncate_tables(%i(widgets details locales))
       Widget.create!(test_id: 'bad_id', some_int: 100) # should not show up
     end
 

data/spec/active_record_batch_consumer_spec.rb

@@ -57,12 +57,6 @@ module ActiveRecordBatchConsumerTest
       Class.new(described_class) do
         record_class Widget
         compacted false
-
-        def process_message?(payload)
-          return true if payload.nil?
-
-          payload['test_id'] != 'skipme'
-        end
       end
     end
 
@@ -109,27 +103,6 @@ module ActiveRecordBatchConsumerTest
               )
           end
 
-          it 'should not create records for messages that return false from process_message?' do
-            publish_batch(
-              [
-                { key: 1,
-                  payload: { test_id: 'abc', some_int: 3 } },
-                { key: 2,
-                  payload: { test_id: 'skipme', some_int: 4 } },
-                { key: 3,
-                  payload: { test_id: 'ghi', some_int: 5 } }
-              ]
-            )
-
-            expect(all_widgets).
-              to contain_exactly(
-                have_attributes(id: 1, test_id: 'abc', some_int: 3, updated_at: start,
-                                created_at: start),
-                have_attributes(id: 3, test_id: 'ghi', some_int: 5, updated_at: start,
-                                created_at: start)
-              )
-          end
-
           it 'should handle deleting a record that doesn\'t exist' do
             publish_batch(
               [
@@ -832,64 +805,6 @@ module ActiveRecordBatchConsumerTest
 
     end
 
-    describe 'post_process_batch' do
-      before(:each) do
-        register_consumer(consumer_class,
-                          'MySchema',
-                          key_config: { plain: true })
-        MyBatchConsumer.last_post_process_batch = nil
-      end
-
-      let(:consumer_class) do
-        Class.new(described_class) do
-          class << self
-            attr_accessor :last_post_process_batch
-          end
-
-          record_class Widget
-          compacted false
-
-          def post_process_batch(messages)
-            self.class.last_post_process_batch = messages
-          end
-
-          def process_message?(payload)
-            payload['test_id'] != 'skipme'
-          end
-        end
-      end
-
-      it 'is called after the batch is processed with the processed messages' do
-        publish_batch(
-          [
-            { key: 1,
-              payload: { test_id: 'abc', some_int: 3 } },
-            { key: 2,
-              payload: { test_id: 'def', some_int: 4 } }
-          ]
-        )
-        expect(MyBatchConsumer.last_post_process_batch.map(&:payload).map do |p|
- p[:test_id]
-        end).to contain_exactly('abc', 'def')
-      end
-
-      it 'is called with filtered messages when process_message? excludes some' do
-        publish_batch(
-          [
-            { key: 1,
-              payload: { test_id: 'abc', some_int: 3 } },
-            { key: 2,
-              payload: { test_id: 'skipme', some_int: 4 } },
-            { key: 3,
-              payload: { test_id: 'ghi', some_int: 5 } }
-          ]
-        )
-
-        expect(MyBatchConsumer.last_post_process_batch.size).to eq(2)
-        expect(MyBatchConsumer.last_post_process_batch.map(&:key).map(&:to_i)).to contain_exactly(1, 3)
-      end
-    end
-
   end
 end
 # rubocop:enable Lint/ConstantDefinitionInBlock

data/spec/active_record_consumer_spec.rb

@@ -255,29 +255,6 @@ module ActiveRecordConsumerTest
             expect(Widget.find_by_test_id('id2').some_int).to eq(4)
           end
 
-          it 'should handle record_class as a string' do
-            consumer_class = Class.new(Deimos::ActiveRecordConsumer) do
-              record_class 'ActiveRecordConsumerTest::Widget'
-            end
-            stub_const('MyStringRecordClassConsumer', consumer_class)
-            Karafka::App.routes.redraw do
-              topic 'my-topic-string' do
-                consumer MyStringRecordClassConsumer
-                schema 'MySchema'
-                namespace 'com.my-namespace'
-                key_config plain: true
-              end
-            end
-
-            expect(Widget.count).to eq(0)
-            test_consume_message(MyStringRecordClassConsumer, {
-                                   test_id: 'abc',
-                                   some_int: 3
-                                 }, call_original: true, key: 5)
-            expect(Widget.count).to eq(1)
-            expect(Widget.last.test_id).to eq('abc')
-          end
-
           it 'should not create record of process_message returns false' do
             allow_any_instance_of(MyConsumer).to receive(:process_message?).and_return(false)
             expect(Widget.count).to eq(0)

data/spec/gen/sample/v1/sample_key_pb.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: sample/v1/sample_key.proto
+
+require 'google/protobuf'
+
+
+descriptor_data = "\n\x1asample/v1/sample_key.proto\x12\tsample.v1\"\x1f\n\x10SampleMessageKey\x12\x0b\n\x03str\x18\x01 \x01(\tb\x06proto3"
+
+pool = ::Google::Protobuf::DescriptorPool.generated_pool
+pool.add_serialized_file(descriptor_data)
+
+module Sample
+  module V1
+    SampleMessageKey = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("sample.v1.SampleMessageKey").msgclass
+  end
+end

data/spec/generators/schema_class_generator_spec.rb

@@ -20,7 +20,7 @@ end
 
 RSpec.describe Deimos::Generators::SchemaClassGenerator do
   let(:schema_class_path) { 'spec/app/lib/schema_classes' }
-  let(:files) { Dir["#{schema_class_path}/**/*.rb"].map { |f| [f, File.read(f)]}.to_h }
+  let(:files) { Dir["#{schema_class_path}/**/*.rb"].to_h { |f| [f, File.read(f)]} }
 
   before(:each) do
     Deimos.config.reset!

data/spec/protos/sample/v1/sample_key.proto

@@ -0,0 +1,7 @@
+syntax = "proto3";
+
+package sample.v1;
+
+message SampleMessageKey {
+  string str = 1;
+}

data/spec/schema_backends/avro_base_shared.rb

@@ -66,7 +66,7 @@ RSpec.shared_examples_for('an Avro backend') do
     expect(backend.encode_key('test_id', 1, topic: 'topic')).to eq('itsme')
     expect(backend).to have_received(:encode).
       with({ 'test_id' => 1 }, { schema: 'MySchema_key', topic: 'topic' })
-    expect(backend.schema_store.find('MySchema_key', 'com.my-namespace').to_avro).
+    expect(backend.schema_store.find('com.my-namespace.MySchema_key').to_avro).
       to eq(
         'doc' => 'Key for com.my-namespace.MySchema - autogenerated by Deimos',
         'fields' => [

data/spec/schema_backends/avro_local_spec.rb

@@ -15,17 +15,10 @@ RSpec.describe Deimos::SchemaBackends::AvroLocal do
   it_should_behave_like 'an Avro backend'
 
   it 'should encode and decode correctly' do
-    avro_turf = instance_double(AvroTurf)
-    allow(avro_turf).to receive_messages(encode: 'encoded-payload', decode: payload)
-    allow(backend).to receive(:avro_turf).and_return(avro_turf)
     results = backend.encode(payload)
-    expect(results).to eq('encoded-payload')
+    expect(results).to start_with("Obj\u0001\u0004\u0014avro.codec\bnull\u0016avro.schema\x94")
     results = backend.decode(results)
     expect(results).to eq(payload)
-    expect(avro_turf).to have_received(:encode).
-      with(payload, schema_name: 'MySchema', namespace: 'com.my-namespace')
-    expect(avro_turf).to have_received(:decode).
-      with('encoded-payload', schema_name: 'MySchema', namespace: 'com.my-namespace')
   end
 
 end

data/spec/schema_backends/avro_schema_registry_spec.rb

@@ -15,17 +15,17 @@ RSpec.describe Deimos::SchemaBackends::AvroSchemaRegistry do
   it_should_behave_like 'an Avro backend'
 
   it 'should encode and decode correctly' do
-    avro_turf = instance_double(AvroTurf::Messaging)
-    allow(avro_turf).to receive_messages(encode: 'encoded-payload', decode: payload)
-    allow(backend).to receive(:avro_turf_messaging).and_return(avro_turf)
+    schema_registry = instance_double(SchemaRegistry::Client)
+    allow(schema_registry).to receive_messages(encode: 'encoded-payload', decode: payload)
+    allow(backend).to receive(:schema_registry).and_return(schema_registry)
     results = backend.encode(payload, topic: 'topic')
     expect(results).to eq('encoded-payload')
     results = backend.decode(results)
     expect(results).to eq(payload)
-    expect(avro_turf).to have_received(:encode).
-      with(payload, schema_name: 'MySchema', subject: 'topic')
-    expect(avro_turf).to have_received(:decode).
-      with('encoded-payload', schema_name: 'MySchema')
+    expect(schema_registry).to have_received(:encode).
+      with(payload, schema_name: 'com.my-namespace.MySchema', subject: 'topic-value')
+    expect(schema_registry).to have_received(:decode).
+      with('encoded-payload')
   end
 
 end

data/spec/schema_backends/base_spec.rb

@@ -6,13 +6,13 @@ describe Deimos::SchemaBackends::Base do
 
   it 'should validate on encode' do
     expect(backend).to receive(:validate).with(payload, schema: 'schema')
-    expect(backend).to receive(:encode_payload).with(payload, schema: 'schema', topic: 'topic')
+    expect(backend).to receive(:encode_payload).with(payload, schema: 'schema', subject: 'topic-value')
     backend.encode(payload, topic: 'topic')
   end
 
   it 'should validate and encode a passed schema' do
     expect(backend).to receive(:validate).with(payload, schema: 'schema2')
-    expect(backend).to receive(:encode_payload).with(payload, schema: 'schema2', topic: 'topic')
+    expect(backend).to receive(:encode_payload).with(payload, schema: 'schema2', subject: 'topic-value')
     backend.encode(payload, schema: 'schema2', topic: 'topic')
   end