deimos-ruby 2.3.0 → 2.4.0.pre.beta2

data/lib/deimos/schema_backends/proto_base.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'base'
-require 'proto_turf'
+require 'schema_registry_client'
 
 module Deimos
   module SchemaBackends
@@ -24,26 +24,51 @@ module Deimos
         float: :float,
         message: :record
       }.freeze
+
       def proto_schema(schema=@schema)
-        Google::Protobuf::DescriptorPool.generated_pool.lookup(schema)
+        proto = Google::Protobuf::DescriptorPool.generated_pool.lookup(schema)
+        if proto.nil?
+          raise "Could not find Protobuf schema '#{schema}'."
+        end
+
+        proto
       end
 
       # @override
       def encode_key(key_id, key, topic: nil)
-        if key.is_a?(Hash)
-          key_id ? key.with_indifferent_access[key_id].to_s : key.sort.to_h.to_json
+        if key.respond_to?(:to_h)
+          hash = if key_id
+                   key_id.to_s.split('.')[...-1].each do |k|
+                     key = key.with_indifferent_access[k]
+                   end
+                   key.to_h.with_indifferent_access.slice(key_id.split('.').last)
+                 else
+                   key.to_h.sort.to_h
+                 end
+          self.encode_proto_key(hash, topic: topic, field: key_id)
+        elsif key_id
+          hash = { key_id.to_s.split('.').last => key }
+          self.encode_proto_key(hash, topic: topic, field: key_id)
         else
           key.to_s
         end
       end
 
+      # @param hash [Hash]
+      # @return [String]
+      def encode_proto_key(hash, topic: nil, field: nil)
+        hash.sort.to_h.to_json
+      end
+
+      def decode_proto_key(payload)
+        JSON.parse(payload)
+      rescue StandardError
+        payload
+      end
+
       # @override
       def decode_key(payload, key_id)
-        val = begin
-                JSON.parse(payload)
-        rescue StandardError
-                payload
-        end
+        val = decode_proto_key(payload)
         key_id ? val[key_id.to_s] : val
       end
 
@@ -85,8 +110,8 @@ module Deimos
         :mock
       end
 
-      def generate_key_schema(_field_name)
-        raise 'Protobuf cannot generate key schemas! Please use field_config :plain'
+      def supports_key_schemas?
+        false
       end
 
     end

data/lib/deimos/schema_backends/proto_local.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'proto_base'
-require 'proto_turf'
+require 'schema_registry_client'
 
 module Deimos
   module SchemaBackends
@@ -14,14 +14,14 @@ module Deimos
       end
 
       # @override
-      def encode_payload(payload, schema: nil, topic: nil)
+      def encode_payload(payload, schema: nil, subject: nil)
         msg = payload.is_a?(Hash) ? proto_schema.msgclass.new(**payload) : payload
         proto_schema.msgclass.encode(msg)
       end
 
-      # @return [ProtoTurf]
-      def self.proto_turf
-        @proto_turf ||= ProtoTurf.new(
+      # @return [SchemaRegistry::Client]
+      def self.schema_registry
+        @schema_registry ||= SchemaRegistry::Client.new(
           registry_url: Deimos.config.schema.registry_url,
           logger: Karafka.logger
         )

data/lib/deimos/schema_backends/proto_schema_registry.rb

@@ -1,31 +1,61 @@
 # frozen_string_literal: true
 
 require_relative 'proto_base'
-require 'proto_turf'
+require 'schema_registry_client'
 
 module Deimos
   module SchemaBackends
     # Encode / decode using the Protobuf schema registry.
     class ProtoSchemaRegistry < ProtoBase
 
+      # @override
+      def self.mock_backend
+        :proto_local
+      end
+
       # @override
       def decode_payload(payload, schema:)
-        self.class.proto_turf.decode(payload)
+        self.class.schema_registry.decode(payload)
       end
 
       # @override
-      def encode_payload(payload, schema: nil, topic: nil)
+      def encode_payload(payload, schema: nil, subject: nil)
         msg = payload.is_a?(Hash) ? proto_schema.msgclass.new(**payload) : payload
-        self.class.proto_turf.encode(msg, subject: topic)
+        encoder = subject&.ends_with?('-key') ? self.class.key_schema_registry : self.class.schema_registry
+        encoder.encode(msg, subject: subject)
+      end
+
+      # @override
+      def encode_proto_key(key, topic: nil, field: nil)
+        schema_text = SchemaRegistry::Output::JsonSchema.output(proto_schema.to_proto, path: field)
+        self.class.key_schema_registry.encode(key, subject: "#{topic}-key", schema_text: schema_text)
+      end
+
+      # @override
+      def decode_proto_key(payload)
+        self.class.key_schema_registry.decode(payload)
       end
 
-      # @return [ProtoTurf]
-      def self.proto_turf
-        @proto_turf ||= ProtoTurf.new(
+      # @return [SchemaRegistry::Client]
+      def self.schema_registry
+        @schema_registry ||= SchemaRegistry::Client.new(
           registry_url: Deimos.config.schema.registry_url,
+          user: Deimos.config.schema.user,
+          password: Deimos.config.schema.password,
           logger: Karafka.logger
         )
       end
+
+      def self.key_schema_registry
+        @key_schema_registry ||= SchemaRegistry::Client.new(
+          registry_url: Deimos.config.schema.registry_url,
+          user: Deimos.config.schema.user,
+          password: Deimos.config.schema.password,
+          logger: Karafka.logger,
+          schema_type: SchemaRegistry::Schema::ProtoJsonSchema
+        )
+      end
+
     end
   end
 end

data/lib/deimos/test_helpers.rb

@@ -50,6 +50,13 @@ module Deimos
         warn("unit_test! is deprecated and can be replaced by setting Deimos's schema backend " \
              'to `:avro_validation`. All other test behavior is provided by Karafka.')
       end
+
+      def with_mock_backends
+        Deimos.mock_backends = true
+        yield
+        Deimos.mock_backends = false
+      end
+
     end
 
     # get the difference of 2 hashes.

data/lib/deimos/transcoder.rb

@@ -36,7 +36,7 @@ module Deimos
       if self.key_field
         self.backend.encode_key(self.key_field, key, topic: @topic)
       else
-        self.backend.encode(key, topic: @topic)
+        self.backend.encode(key, topic: @topic, is_key: true)
       end
     end
 

data/lib/deimos/utils/outbox_producer.rb

@@ -205,7 +205,7 @@ module Deimos
         batch_size = batch.size
         current_index = 0
 
-        batch[current_index..-1].in_groups_of(batch_size, false).each do |group|
+        batch[current_index..].in_groups_of(batch_size, false).each do |group|
           @logger.debug("Publishing #{group.size} messages to #{@current_topic}")
           Deimos.producer_for(@current_topic).produce_many_sync(group)
           current_index += group.size

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '2.3.0'
+  VERSION = '2.4.0-beta2'
 end

data/lib/deimos.rb

@@ -54,6 +54,9 @@ module Deimos
 
   class << self
 
+    # @return [Boolean] for use in unit tests
+    attr_accessor :mock_backends
+
     # @param backend [Symbol, nil]
     # @return [Class<Deimos::SchemaBackends::Base>]
     def schema_backend_class(backend: nil)
@@ -61,7 +64,12 @@ module Deimos
 
       require "deimos/schema_backends/#{backend}"
 
-      "Deimos::SchemaBackends::#{backend.to_s.classify}".constantize
+      klass = "Deimos::SchemaBackends::#{backend.to_s.classify}".constantize
+      if self.mock_backends
+        require "deimos/schema_backends/#{klass.mock_backend}"
+        klass = "Deimos::SchemaBackends::#{klass.mock_backend.to_s.classify}".constantize
+      end
+      klass
     end
 
     # @param schema [String, Symbol]

data/lib/generators/deimos/active_record_generator.rb

@@ -48,7 +48,7 @@ module Deimos
         # @return [String]
         def schema
           last_dot = self.full_schema.rindex('.')
-          self.full_schema[(last_dot + 1)..-1]
+          self.full_schema[(last_dot + 1)..]
         end
 
         # @return [String]

data/lib/generators/deimos/schema_class_generator.rb

@@ -13,7 +13,7 @@ module Deimos
       # @return [Array<Symbol>]
       SPECIAL_TYPES = %i(record enum).freeze
       # @return [String]
-      INITIALIZE_WHITESPACE = "\n#{' ' * 19}"
+      INITIALIZE_WHITESPACE = "\n#{' ' * 19}".freeze
       # @return [Array<String>]
       IGNORE_DEFAULTS = %w(message_id timestamp).freeze
       # @return [String]
@@ -192,7 +192,7 @@ module Deimos
 
       def generate_from_schema_files(found_schemas)
         path = Deimos.config.schema.path || Deimos.config.schema.paths[:avro].first
-        schema_store = AvroTurf::MutableSchemaStore.new(path: path)
+        schema_store = SchemaRegistry::AvroSchemaStore.new(path: path)
         schema_store.load_schemas!
         schema_store.schemas.values.sort_by { |s| "#{s.namespace}#{s.name}" }.each do |schema|
           name = "#{schema.namespace}.#{schema.name}"
@@ -265,7 +265,7 @@ module Deimos
         end
 
         result = "def initialize(_from_message: false, #{arguments.first}"
-        arguments[1..-1].each_with_index do |arg, _i|
+        arguments[1..].each_with_index do |arg, _i|
           result += ",#{INITIALIZE_WHITESPACE}#{arg}"
         end
         "#{result})"

data/lib/generators/deimos/v2_generator.rb

@@ -119,7 +119,7 @@ module Deimos
       end
 
       def consumer_configs
-        deimos_config.consumer_objects.group_by(&:group_id).map { |group_id, consumers|
+        deimos_config.consumer_objects.group_by(&:group_id).to_h do |group_id, consumers|
           [group_id, consumers.map do |consumer|
             kafka_configs = {}
             kafka_configs['auto.offset.reset'] = consumer.start_from_beginning ? 'earliest' : 'latest'
@@ -159,7 +159,7 @@ module Deimos
             configs[:each_message] = true unless consumer.delivery.to_s == 'inline_batch'
             configs
           end]
-        }.to_h
+        end
       end
 
       def producer_configs

data/spec/gen/sample/v1/sample_key_pb.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: sample/v1/sample_key.proto
+
+require 'google/protobuf'
+
+
+descriptor_data = "\n\x1asample/v1/sample_key.proto\x12\tsample.v1\"\x1f\n\x10SampleMessageKey\x12\x0b\n\x03str\x18\x01 \x01(\tb\x06proto3"
+
+pool = ::Google::Protobuf::DescriptorPool.generated_pool
+pool.add_serialized_file(descriptor_data)
+
+module Sample
+  module V1
+    SampleMessageKey = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("sample.v1.SampleMessageKey").msgclass
+  end
+end

data/spec/generators/schema_class_generator_spec.rb

@@ -20,7 +20,7 @@ end
 
 RSpec.describe Deimos::Generators::SchemaClassGenerator do
   let(:schema_class_path) { 'spec/app/lib/schema_classes' }
-  let(:files) { Dir["#{schema_class_path}/**/*.rb"].map { |f| [f, File.read(f)]}.to_h }
+  let(:files) { Dir["#{schema_class_path}/**/*.rb"].to_h { |f| [f, File.read(f)]} }
 
   before(:each) do
     Deimos.config.reset!

data/spec/protos/sample/v1/sample_key.proto

@@ -0,0 +1,7 @@
+syntax = "proto3";
+
+package sample.v1;
+
+message SampleMessageKey {
+  string str = 1;
+}

data/spec/schema_backends/avro_base_shared.rb

@@ -66,7 +66,7 @@ RSpec.shared_examples_for('an Avro backend') do
     expect(backend.encode_key('test_id', 1, topic: 'topic')).to eq('itsme')
     expect(backend).to have_received(:encode).
       with({ 'test_id' => 1 }, { schema: 'MySchema_key', topic: 'topic' })
-    expect(backend.schema_store.find('MySchema_key', 'com.my-namespace').to_avro).
+    expect(backend.schema_store.find('com.my-namespace.MySchema_key').to_avro).
       to eq(
         'doc' => 'Key for com.my-namespace.MySchema - autogenerated by Deimos',
         'fields' => [

data/spec/schema_backends/avro_local_spec.rb

@@ -15,17 +15,10 @@ RSpec.describe Deimos::SchemaBackends::AvroLocal do
   it_should_behave_like 'an Avro backend'
 
   it 'should encode and decode correctly' do
-    avro_turf = instance_double(AvroTurf)
-    allow(avro_turf).to receive_messages(encode: 'encoded-payload', decode: payload)
-    allow(backend).to receive(:avro_turf).and_return(avro_turf)
     results = backend.encode(payload)
-    expect(results).to eq('encoded-payload')
+    expect(results).to start_with("Obj\u0001\u0004\u0014avro.codec\bnull\u0016avro.schema\x94")
     results = backend.decode(results)
     expect(results).to eq(payload)
-    expect(avro_turf).to have_received(:encode).
-      with(payload, schema_name: 'MySchema', namespace: 'com.my-namespace')
-    expect(avro_turf).to have_received(:decode).
-      with('encoded-payload', schema_name: 'MySchema', namespace: 'com.my-namespace')
   end
 
 end

data/spec/schema_backends/avro_schema_registry_spec.rb

@@ -15,17 +15,17 @@ RSpec.describe Deimos::SchemaBackends::AvroSchemaRegistry do
   it_should_behave_like 'an Avro backend'
 
   it 'should encode and decode correctly' do
-    avro_turf = instance_double(AvroTurf::Messaging)
-    allow(avro_turf).to receive_messages(encode: 'encoded-payload', decode: payload)
-    allow(backend).to receive(:avro_turf_messaging).and_return(avro_turf)
+    schema_registry = instance_double(SchemaRegistry::Client)
+    allow(schema_registry).to receive_messages(encode: 'encoded-payload', decode: payload)
+    allow(backend).to receive(:schema_registry).and_return(schema_registry)
     results = backend.encode(payload, topic: 'topic')
     expect(results).to eq('encoded-payload')
     results = backend.decode(results)
     expect(results).to eq(payload)
-    expect(avro_turf).to have_received(:encode).
-      with(payload, schema_name: 'MySchema', subject: 'topic')
-    expect(avro_turf).to have_received(:decode).
-      with('encoded-payload', schema_name: 'MySchema')
+    expect(schema_registry).to have_received(:encode).
+      with(payload, schema_name: 'com.my-namespace.MySchema', subject: 'topic-value')
+    expect(schema_registry).to have_received(:decode).
+      with('encoded-payload')
   end
 
 end

data/spec/schema_backends/base_spec.rb

@@ -6,13 +6,13 @@ describe Deimos::SchemaBackends::Base do
 
   it 'should validate on encode' do
     expect(backend).to receive(:validate).with(payload, schema: 'schema')
-    expect(backend).to receive(:encode_payload).with(payload, schema: 'schema', topic: 'topic')
+    expect(backend).to receive(:encode_payload).with(payload, schema: 'schema', subject: 'topic-value')
     backend.encode(payload, topic: 'topic')
   end
 
   it 'should validate and encode a passed schema' do
     expect(backend).to receive(:validate).with(payload, schema: 'schema2')
-    expect(backend).to receive(:encode_payload).with(payload, schema: 'schema2', topic: 'topic')
+    expect(backend).to receive(:encode_payload).with(payload, schema: 'schema2', subject: 'topic-value')
     backend.encode(payload, schema: 'schema2', topic: 'topic')
   end