deimos-ruby 1.15.1 → 1.16.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 13d2028ca2c052ef2d64099e9ca135c7dac2ea76fe0ee995f8729c6e16b39f63
-  data.tar.gz: a539787e5311593f664280c7e26a0d73f345026418b12e50e8daf508fd97d61c
+  metadata.gz: 59c0f6801979649cc677cc485cfc1435d1ef235769784a864eabbc3e35e55573
+  data.tar.gz: 85d9adfa94010a7594a94e8770ee15922a893ecb53ccb600e7911ae9615ff574
 SHA512:
-  metadata.gz: 7554b95d79b8fdf1c1b262208737b9ebf1a10ba3acbb3ae96f7a9c47fa8762fb42f77ae416be82a49c0d058e1e4af9bceb3139523e10a6fb5d890eb2fb64430f
-  data.tar.gz: 76e540826895dbac336fc04bb8f67bf6dc29ee9582c2a39367112a453a8abea7129854a0b89bf8468631b6476f353944e2f23d23288c1faad44b4501ba8f7111
+  metadata.gz: c663849c130074d566875b278b4e6af667c814e1860a24ac531cc163e819aaac399381578a1d93130b9121402ec1f500108155ee20f37be6999df3d496869376
+  data.tar.gz: 6ba7849db0b98b80684381056be5aeb1f6d206ee65e742c8e33c1cdc9cb0f0f150d8f61f868f18e29837ac2e3279d1b2737c244c79201cbec9038da1c7128264

data/CHANGELOG.md

@@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+# 1.16.2 - 2022-09-07
+
+- Added support of post_process method in ActiveRecordProducer
+
+# 1.16.1 - 2022-08-03
+
+- Fix issues with `enum` schema classes (e.g. equality not working, `to_s` not working)
+- Changed assumption in the base schema class that it was a record (e.g. defining `to_h` instead of `as_json`). Moved record functionality to the record base class.
+- Added the `active_span` and `set_tag` methods to the tracing classes.
+- Added span tags for fields in SchemaControllerMixin.
+- Updated SchemaControllerMixin so it works with generated schema classes.
+- Fixed bug with previous release where the filename and constant names for `generate_namespace_folders` did not always match.
+
 # 1.15.0 - 2022-07-20
 
 - Update to `sigurd` 0.1.0 - DB producer should now exit when receiving a `SIGTERM` instead of throwing a `SignalException`

data/README.md

@@ -785,6 +785,21 @@ have one process running at a time. If a particular poll takes longer than
 the poll interval (i.e. interval is set at 1 minute but it takes 75 seconds)
 the next poll will begin immediately following the first one completing.
 
+To Post-Process records that are sent to Kafka:
+
+You need to define one additional method in your producer class to post-process the messages sent to Kafka.
+
+```ruby
+class MyProducer < Deimos::ActiveRecordProducer
+  ...
+  def post_process(batch)
+    # If you need to do some extra actions with
+    # the collection of elements you just sent to Kafka
+    # write some code here
+  end
+end
+```
+
 ## Running consumers
 
 Deimos includes a rake task. Once it's in your gemfile, just run

data/lib/deimos/active_record_producer.rb

@@ -44,6 +44,7 @@ module Deimos
           generate_payload(attrs, record).with_indifferent_access
         end
         self.publish_list(messages, force_send: force_send)
+        self.post_process(records)
       end
 
       # Generate the payload, given a list of attributes or a record..
@@ -60,7 +61,7 @@ module Deimos
         end
         return payload unless Utils::SchemaClass.use?(config.to_h)
 
-        Utils::SchemaClass.instance(payload, config[:schema])
+        Utils::SchemaClass.instance(payload, config[:schema], config[:namespace])
       end
 
       # Query to use when polling the database with the DbPoller. Add
@@ -85,6 +86,12 @@ module Deimos
           time_to
         )
       end
+
+      # Post process records after publishing
+      # @param records [Array<ActiveRecord::Base>]
+      def post_process(_records)
+      end
+
     end
   end
 end

data/lib/deimos/consumer.rb

@@ -58,7 +58,9 @@ module Deimos
       decoded_payload = payload.nil? ? nil : self.class.decoder.decode(payload)
       return decoded_payload unless Utils::SchemaClass.use?(self.class.config.to_h)
 
-      Utils::SchemaClass.instance(decoded_payload, self.class.config[:schema])
+      Utils::SchemaClass.instance(decoded_payload,
+                                  self.class.config[:schema],
+                                  self.class.config[:namespace])
     end
 
   private

data/lib/deimos/schema_class/base.rb

@@ -11,21 +11,9 @@ module Deimos
       def initialize(*_args)
       end
 
-      # Converts the object to a string that represents a JSON object
-      # @return [String] a JSON string
-      def to_json(*_args)
-        to_h.to_json
-      end
-
       # Converts the object to a hash which can be used for debugging or comparing objects.
       # @return [Hash] a hash representation of the payload
       def as_json(_opts={})
-        JSON.parse(to_json)
-      end
-
-      # Converts the object attributes to a hash which can be used for Kafka
-      # @return [Hash] the payload as a hash.
-      def to_h
         raise NotImplementedError
       end
 
@@ -35,17 +23,6 @@ module Deimos
         self.send("#{key}=", val)
       end
 
-      # Merge a hash or an identical schema object with this one and return a new object.
-      # @param other_hash [Hash|SchemaClasses::Base]
-      # @return [SchemaClasses::Base]
-      def merge(other_hash)
-        obj = self.class.new(**self.to_h.symbolize_keys)
-        other_hash.to_h.each do |k, v|
-          obj.send("#{k}=", v)
-        end
-        obj
-      end
-
       # :nodoc:
       def ==(other)
         comparison = other
@@ -57,9 +34,9 @@ module Deimos
       end
 
       # :nodoc:
-      def to_s
+      def inspect
         klass = self.class
-        "#{klass}(#{self.as_json.symbolize_keys.to_s[1..-2]})"
+        "#{klass}(#{self.as_json})"
       end
 
       # Initializes this class from a given value

data/lib/deimos/schema_class/enum.rb

@@ -7,12 +7,35 @@ module Deimos
   module SchemaClass
     # Base Class for Enum Classes generated from Avro.
     class Enum < Base
+
+      attr_accessor :value
+
+      # @param other [Deimos::SchemaClass::Enum]
+      def ==(other)
+        other.is_a?(self.class) ? other.value == @value : other == @value
+      end
+
+      # @return [String]
+      def to_s
+        @value.to_s
+      end
+
+      # @param value [String]
+      def initialize(value)
+        @value = value
+      end
+
       # Returns all the valid symbols for this enum.
       # @return [Array<String>]
       def symbols
         raise NotImplementedError
       end
 
+      # :nodoc:
+      def as_json(_opts={})
+        @value
+      end
+
       # :nodoc:
       def self.initialize_from_value(value)
         return nil if value.nil?

data/lib/deimos/schema_class/record.rb

@@ -8,10 +8,21 @@ module Deimos
     # Base Class of Record Classes generated from Avro.
     class Record < Base
 
-      # Converts the object to a hash which can be used for debugging or comparing objects.
-      # @return [Hash] a hash representation of the payload
-      def as_json(_opts={})
-        super.except('payload_key')
+      # Converts the object attributes to a hash which can be used for Kafka
+      # @return [Hash] the payload as a hash.
+      def to_h
+        self.as_json
+      end
+
+      # Merge a hash or an identical schema object with this one and return a new object.
+      # @param other_hash [Hash|SchemaClasses::Base]
+      # @return [SchemaClasses::Base]
+      def merge(other_hash)
+        obj = self.class.new(**self.to_h.symbolize_keys)
+        other_hash.to_h.each do |k, v|
+          obj.send("#{k}=", v)
+        end
+        obj
       end
 
       # Element access method as if this Object were a hash

data/lib/deimos/test_helpers.rb

@@ -373,6 +373,7 @@ module Deimos
                              call_original,
                              &block)
       schema_class = handler.class.config[:schema]
+      namespace = handler.class.config[:namespace]
       expected = input.dup
 
       config = handler.class.config
@@ -382,10 +383,10 @@ module Deimos
       if use_schema_classes && schema_class.present?
         expected = if input.is_a?(Array)
                      input.map do |payload|
-                       Utils::SchemaClass.instance(payload, schema_class)
+                       Utils::SchemaClass.instance(payload, schema_class, namespace)
                      end
                    else
-                     Utils::SchemaClass.instance(input, schema_class)
+                     Utils::SchemaClass.instance(input, schema_class, namespace)
                    end
       end
 

data/lib/deimos/tracing/datadog.rb

@@ -26,10 +26,21 @@ module Deimos
         span.finish
       end
 
+      # :nodoc:
+      def active_span
+        ::Datadog.tracer.active_span
+      end
+
       # :nodoc:
       def set_error(span, exception)
         span.set_error(exception)
       end
+
+      # :nodoc:
+      def set_tag(tag, value, span=nil)
+        (span || active_span).set_tag(tag, value)
+      end
+
     end
   end
 end

data/lib/deimos/tracing/mock.rb

@@ -29,6 +29,16 @@ module Deimos
         @logger.info("Mock span '#{name}' finished: #{start} to #{finish}")
       end
 
+      # :nodoc:
+      def active_span
+        nil
+      end
+
+      # :nodoc:
+      def set_tag(name, value)
+        nil
+      end
+
       # :nodoc:
       def set_error(span, exception)
         span[:exception] = exception

data/lib/deimos/tracing/provider.rb

@@ -24,6 +24,19 @@ module Deimos
       def set_error(span, exception)
         raise NotImplementedError
       end
+
+      # Get the currently activated span.
+      def active_span
+        raise NotImplementedError
+      end
+
+      # Set a tag to a span. Use the currently active span if not given.
+      # @param tag [String]
+      # @param value [String]
+      def set_tag(tag, value, span=nil)
+        raise NotImplementedError
+      end
+
     end
   end
 end

data/lib/deimos/utils/schema_class.rb

@@ -5,14 +5,28 @@ module Deimos
     # Class used by SchemaClassGenerator and Consumer/Producer interfaces
     module SchemaClass
       class << self
+
+        # @param namespace [String]
+        # @return [Array<String>]
+        def modules_for(namespace)
+          modules = ['Schemas']
+          namespace_folder = namespace.split('.').last
+          if Deimos.config.schema.generate_namespace_folders && namespace_folder
+            modules.push(namespace_folder.underscore.classify)
+          end
+          modules
+        end
+
         # Converts a raw payload into an instance of the Schema Class
         # @param payload [Hash, Deimos::SchemaClass::Base]
         # @param schema [String]
+        # @param namespace [String]
         # @return [Deimos::SchemaClass::Record]
-        def instance(payload, schema)
+        def instance(payload, schema, namespace='')
           return payload if payload.is_a?(Deimos::SchemaClass::Base)
 
-          klass = "Schemas::#{schema.underscore.camelize}".safe_constantize
+          constants = modules_for(namespace) + [schema.underscore.camelize.singularize]
+          klass = constants.join('::').safe_constantize
           return payload if klass.nil? || payload.nil?
 
           klass.new(**payload.symbolize_keys)

data/lib/deimos/utils/schema_controller_mixin.rb

@@ -98,6 +98,12 @@ module Deimos
         namespace, schema = parse_namespace(:request)
         decoder = Deimos.schema_backend(schema: schema, namespace: namespace)
         @payload = decoder.decode(request.body.read).with_indifferent_access
+        @payload.each do |key, value|
+          Deimos.config.tracer&.set_tag("body.#{key}", value)
+        end
+        if Deimos.config.schema.use_schema_classes
+          @payload = Utils::SchemaClass.instance(@payload, schema, namespace)
+        end
         request.body.rewind if request.body.respond_to?(:rewind)
       end
 
@@ -106,7 +112,7 @@ module Deimos
       def render_schema(payload, schema: nil, namespace: nil)
         namespace, schema = parse_namespace(:response) if !schema && !namespace
         encoder = Deimos.schema_backend(schema: schema, namespace: namespace)
-        encoded = encoder.encode(payload, topic: "#{namespace}.#{schema}")
+        encoded = encoder.encode(payload.to_h, topic: "#{namespace}.#{schema}")
         response.headers['Content-Type'] = encoder.class.content_type
         send_data(encoded)
       end

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.15.1'
+  VERSION = '1.16.2'
 end

data/lib/generators/deimos/schema_class/templates/schema_enum.rb.tt

@@ -3,19 +3,8 @@
     # @return ['<%= @current_schema.symbols.join("', '") %>']
     attr_accessor :<%= @current_schema.name.underscore %>
 
-    # :nodoc:
-    def initialize(<%= @current_schema.name.underscore %>)
-      super
-      self.<%= @current_schema.name.underscore %> = <%= @current_schema.name.underscore %>
-    end
-
     # @override
     def symbols
       %w(<%= @current_schema.symbols.join(' ') %>)
     end
-
-    # @override
-    def to_h
-      @<%= @current_schema.name.underscore %>
-    end
   end

data/lib/generators/deimos/schema_class/templates/schema_record.rb.tt

@@ -64,10 +64,10 @@
     end
 
     # @override
-    def to_h
+    def as_json(_opts={})
       {
 <%- @fields.each do |field| -%>
-        <%= field_to_h(field) %>
+        <%= field_as_json(field) %>
 <% end -%>
       }
     end

data/lib/generators/deimos/schema_class_generator.rb

@@ -108,16 +108,12 @@ module Deimos
         # @param key_schema_base [Avro::Schema::NamedSchema, nil]
         def write_file(schema, key_schema_base)
           class_template = _generate_class_template_from_schema(schema, key_schema_base)
-          @modules = ['Schemas']
-          namespace_folder = schema.namespace.split('.').last
-          if Deimos.config.schema.generate_namespace_folders
-            @modules.push(namespace_folder.underscore.classify)
-          end
+          @modules = Utils::SchemaClass.modules_for(schema.namespace)
           @main_class_definition = class_template
 
           file_prefix = schema.name.underscore.singularize
           if Deimos.config.schema.generate_namespace_folders
-            file_prefix = "#{namespace_folder}/#{file_prefix}"
+            file_prefix = "#{@modules.last.underscore.singularize}/#{file_prefix}"
           end
           filename = "#{Deimos.config.schema.generated_class_path}/#{file_prefix}.rb"
           template(SCHEMA_CLASS_FILE, filename, force: true)
@@ -126,18 +122,18 @@ module Deimos
         # Format a given field into its appropriate to_h representation.
         # @param field[Deimos::SchemaField]
         # @return [String]
-        def field_to_h(field)
+        def field_as_json(field)
           res = "'#{field.name}' => @#{field.name}"
           field_base_type = _schema_base_class(field.type).type_sym
 
           if %i(record enum).include?(field_base_type)
             res += case field.type.type_sym
                    when :array
-                     '.map { |v| v&.to_h }'
+                     '.map { |v| v&.as_json }'
                    when :map
-                     '.transform_values { |v| v&.to_h }'
+                     '.transform_values { |v| v&.as_json }'
                    else
-                     '&.to_h'
+                     '&.as_json'
                    end
           end
 

data/spec/active_record_producer_spec.rb

@@ -45,6 +45,24 @@ describe Deimos::ActiveRecordProducer do
       record_class Widget
     end
     stub_const('MyProducerWithUniqueID', producer_class)
+
+    producer_class = Class.new(Deimos::ActiveRecordProducer) do
+      schema 'MySchemaWithUniqueId'
+      namespace 'com.my-namespace'
+      topic 'my-topic-with-unique-id'
+      key_config field: :id
+      record_class Widget
+
+      # :nodoc:
+      def self.post_process(batch)
+        batch.each do |message|
+          message.test_id = 'post_processed'
+          message.save!
+        end
+      end
+    end
+
+    stub_const('MyProducerWithPostProcess', producer_class)
   end
 
   describe 'produce' do
@@ -85,6 +103,12 @@ describe Deimos::ActiveRecordProducer do
           )
         end
 
+        it 'should post process the batch of records in #send_events' do
+          widget = Widget.create!(test_id: 'abc3', some_int: 4)
+          MyProducerWithPostProcess.send_events([widget])
+          expect(widget.reload.test_id).to eq('post_processed')
+        end
+
       end
     end
   end

data/spec/batch_consumer_spec.rb

@@ -32,6 +32,13 @@ module ConsumerTest
     describe 'consume_batch' do
       SCHEMA_CLASS_SETTINGS.each do |setting, use_schema_classes|
         context "with Schema Class consumption #{setting}" do
+
+          let(:schema_class_batch) do
+            batch.map do |p|
+              Deimos::Utils::SchemaClass.instance(p, 'MySchema', 'com.my-namespace')
+            end
+          end
+
           before(:each) do
             Deimos.configure { |config| config.schema.use_schema_classes = use_schema_classes }
           end
@@ -49,26 +56,22 @@ module ConsumerTest
             end
             stub_const('ConsumerTest::MyOldBatchConsumer', consumer_class)
 
-            test_consume_batch(MyOldBatchConsumer, batch) do |received, _metadata|
-              expect(received).to eq(batch)
+            test_consume_batch(MyOldBatchConsumer, schema_class_batch) do |received, _metadata|
+              expect(received).to eq(schema_class_batch)
             end
           end
 
           it 'should consume a batch of messages' do
-            test_consume_batch(MyBatchConsumer, batch) do |received, _metadata|
-              expect(received).to eq(batch)
+            test_consume_batch(MyBatchConsumer, schema_class_batch) do |received, _metadata|
+              expect(received).to eq(schema_class_batch)
             end
           end
 
           it 'should consume a message on a topic' do
-            test_consume_batch('my_batch_consume_topic', batch) do |received, _metadata|
-              expect(received).to eq(batch)
+            test_consume_batch('my_batch_consume_topic', schema_class_batch) do |received, _metadata|
+              expect(received).to eq(schema_class_batch)
             end
           end
-
-          it 'should fail on an invalid message in the batch' do
-            test_consume_batch_invalid_message(MyBatchConsumer, batch.concat(invalid_payloads))
-          end
         end
       end
     end

data/spec/generators/schema_class/my_schema_with_circular_reference_spec.rb

@@ -6,23 +6,23 @@ RSpec.describe Schemas::MySchemaWithCircularReference do
     {
       properties: {
         a_boolean: {
-          property: true
+          'property' => true
         },
         an_integer: {
-          property: 1
+          'property' => 1
         },
         a_float: {
-          property: 4.5
+          'property' => 4.5
         },
         a_string: {
-          property: 'string'
+          'property' => 'string'
         },
         an_array: {
-          property: [1, 2, 3]
+          'property' => [1, 2, 3]
         },
         an_hash: {
-          property: {
-            a_key: 'a_value'
+          'property' => {
+            'a_key' => 'a_value'
           }
         }
       }
@@ -63,23 +63,23 @@ RSpec.describe Schemas::MySchemaWithCircularReference do
       described_class.new(**payload_hash)
       payload_h = {
         'properties' => {
-          'a_boolean' => {
-            'property' => true
+          a_boolean: {
+            'property' =>true
           },
-          'an_integer' => {
-            'property' => 1
+          an_integer: {
+            'property' =>1
           },
-          'a_float' => {
-            'property' => 4.5
+          a_float: {
+            'property' =>4.5
           },
-          'a_string' => {
-            'property' => 'string'
+          a_string: {
+            'property' =>'string'
           },
-          'an_array' => {
-            'property' => [1, 2, 3]
+          an_array: {
+            'property' =>[1, 2, 3]
           },
-          'an_hash' => {
-            'property' => {
+          an_hash: {
+            'property' =>{
               'a_key' => 'a_value'
             }
           }

data/spec/generators/schema_class/my_schema_with_complex_types_spec.rb

@@ -153,7 +153,7 @@ RSpec.describe Schemas::MySchemaWithComplexType do
       it 'should get the value of some_enum_array' do
         some_enum_array = klass.some_enum_array
         expect(some_enum_array.first).to be_instance_of(Schemas::MySchemaWithComplexType::AnEnum)
-        expect(some_enum_array.first.an_enum).to eq('sym1')
+        expect(some_enum_array.first.value).to eq('sym1')
       end
 
       it 'should get the value of some_record' do
@@ -197,7 +197,7 @@ RSpec.describe Schemas::MySchemaWithComplexType do
       end
 
       it 'should modify the value of some_enum_array' do
-        klass.some_enum_array.first.an_enum = 'new_sym'
+        klass.some_enum_array.first.value = 'new_sym'
         expect(klass.some_enum_array.first).
           to eq(Schemas::MySchemaWithComplexType::AnEnum.new('new_sym'))
 

data/spec/schemas/generated.rb

@@ -31,7 +31,7 @@ module Schemas
       end
 
       # @override
-      def to_h
+      def as_json(_opts={})
         {
           'a_record_field' => @a_record_field
         }
@@ -55,7 +55,7 @@ module Schemas
       end
 
       # @override
-      def to_h
+      def as_json(_opts={})
         @an_enum
       end
     end
@@ -139,7 +139,7 @@ module Schemas
     end
 
     # @override
-    def to_h
+    def as_json(_opts={})
       {
         'a_string' => @a_string,
         'a_int' => @a_int,
@@ -147,12 +147,12 @@ module Schemas
         'a_float' => @a_float,
         'a_double' => @a_double,
         'an_optional_int' => @an_optional_int,
-        'an_enum' => @an_enum&.to_h,
+        'an_enum' => @an_enum&.as_json,
         'an_array' => @an_array,
         'a_map' => @a_map,
         'timestamp' => @timestamp,
         'message_id' => @message_id,
-        'a_record' => @a_record&.to_h
+        'a_record' => @a_record&.as_json
       }
     end
   end