deimos-ruby 1.23.2 → 1.24.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b0c3a1e7084ccdf781b9a826de555c98661f564c3ecaaf8daf80427d59f91c6e
-  data.tar.gz: 995449f489eb5bc76e870e2666c53a48866d54202b98d508c278f393c604d412
+  metadata.gz: 391cad2b23e7a6ef37106dcc5403af4ef5b62a82c1a6189e712fb3bc339613d1
+  data.tar.gz: 2bbb4468ee975158489ea797ee67c636d1c5da88cda8b10b33923b4410459178
 SHA512:
-  metadata.gz: a203d1c8d44f90f512ea4a4c3fa03d524a6fde8afad3e1683a5a1c7c544e21e159bdd61cde92f0a256465b06bdb30ae6472ae415afe0d988afa9a8eb217bd1b8
-  data.tar.gz: c510ed78c61d2ed42de36ba631d69070f9d9a9406cedb15bb7c0182cb984c0a13c25887b633467c819432809bf1aa73b48e3ce81589f1b9f82d6519588e1c043
+  metadata.gz: 48f37dc3bf2d016bc8806ccf26def976af003a4a8ebd46dad37e080fdeb9489f0bd3993747feda26a6e37d997b24f169554bb587e8d7ca88af6de85cf41851dd
+  data.tar.gz: ccee77567b4599bacc341e25f8c653a3f4ce21252c39dfb9849ffc2005fe44dc58aaa23537bb636f1545b1a691156225f66e1a909949b0bafb8440efa5269c4e

data/CHANGELOG.md

@@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+# 1.24.0 - 2024-02-26
+
+***BREAKING CHANGE****
+
+- Feature: Rename configuration option `generate_namespace_folders` to `use_full_namespace`. This will now use the entire namespace when generating schema classes
+- Feature: Add configuration option `schema_namespace_map` to enable full control over the namespace for generated schema classes. Requires `use_full_namespace`
+
+# 1.23.3 - 2024-01-25
+- Feature: Add configuration to skip messages that are too large to publish via DB poller.
+
 # 1.23.2 - 2024-01-22
 - Fix: Send a `publish_error` metric for errors other than `DeliveryFailed`.
 

data/docs/CONFIGURATION.md

@@ -210,7 +210,8 @@ schema.path|nil|Local path to find your schemas.
 schema.use_schema_classes|false|Set this to true to use generated schema classes in your application.
 schema.generated_class_path|`app/lib/schema_classes`|Local path to generated schema classes.
 schema.nest_child_schemas|false|Set to true to nest subschemas within the generated class for the parent schema.
-schema.generate_namespace_folders|false|Set to true to generate folders for schemas matching the last part of the namespace.
+schema.use_full_namespace|false|Set to true to generate folders for schemas matching the full namespace.
+schema.schema_namespace_map|{}|A map of namespace prefixes to base module name(s). Example: { 'com.mycompany.suborg' => ['SchemaClasses'] }. Requires `use_full_namespace` to be true.
 
 ## Database Producer Configuration
 

data/lib/deimos/config/configuration.rb

@@ -380,7 +380,13 @@ module Deimos # rubocop:disable Metrics/ModuleLength
 
       # Set to true to generate folders matching the last part of the schema namespace.
       # @return [Boolean]
-      setting :generate_namespace_folders, false
+      setting :use_full_namespace, false
+
+      # Use this option to reduce nesting when using use_full_namespace.
+      # For example: { 'com.mycompany.suborg' => 'SchemaClasses' }
+      # would replace a prefixed with the given key with the module name SchemaClasses.
+      # @return [Hash]
+      setting :schema_namespace_map, {}
     end
 
     # The configured metrics provider.
@@ -500,6 +506,9 @@ module Deimos # rubocop:disable Metrics/ModuleLength
       # The number of times to retry production when encountering a *non-Kafka* error. Set to nil
       # for infinite retries.
       setting :retries, 1
+      # If true, rather than shutting down when finding a message that is too large, log an
+      # error and skip it.
+      setting :skip_too_large_messages, false
       # Amount of time, in seconds, to wait before catching updates, to allow transactions
       # to complete but still pick up the right records. Should only be set for time-based mode.
       setting :delay_time, 2

data/lib/deimos/instrumentation.rb

@@ -43,6 +43,32 @@ module Deimos
 
   # This module listens to events published by RubyKafka.
   module KafkaListener
+    # @param exception [Exception]
+    def self.handle_exception_with_messages(exception)
+      messages = exception.failed_messages
+      messages.group_by(&:topic).each do |topic, batch|
+        producer = Deimos::Producer.descendants.find { |c| c.topic == topic }
+        next if batch.empty? || !producer
+
+        decoder = Deimos.schema_backend(schema: producer.config[:schema],
+                                        namespace: producer.config[:namespace])
+        payloads = batch.map { |m| decoder.decode(m.value) }
+
+        Deimos.config.metrics&.increment(
+          'publish_error',
+          tags: %W(topic:#{topic}),
+          by: payloads.size
+        )
+        Deimos.instrument(
+          'produce_error',
+          producer: producer,
+          topic: topic,
+          exception_object: exception,
+          payloads: payloads
+        )
+      end
+    end
+
     # Listens for any exceptions that happen during publishing and re-publishes
     # as a Deimos event.
     # @param event [ActiveSupport::Notifications::Event]
@@ -52,28 +78,7 @@ module Deimos
       return unless exception
 
       if exception.respond_to?(:failed_messages)
-        messages = exception.failed_messages
-        messages.group_by(&:topic).each do |topic, batch|
-          producer = Deimos::Producer.descendants.find { |c| c.topic == topic }
-          next if batch.empty? || !producer
-
-          decoder = Deimos.schema_backend(schema: producer.config[:schema],
-                                          namespace: producer.config[:namespace])
-          payloads = batch.map { |m| decoder.decode(m.value) }
-
-          Deimos.config.metrics&.increment(
-            'publish_error',
-            tags: %W(topic:#{topic}),
-            by: payloads.size
-          )
-          Deimos.instrument(
-            'produce_error',
-            producer: producer,
-            topic: topic,
-            exception_object: exception,
-            payloads: payloads
-          )
-      end
+        handle_exception_with_messages(exception)
       else
         Deimos.config.metrics&.increment(
           'publish_error',

data/lib/deimos/utils/db_poller/base.rb

@@ -105,6 +105,25 @@ module Deimos
           raise Deimos::MissingImplementationError
         end
 
+        # @param exception [Exception]
+        # @param batch [Array<ActiveRecord::Base>]
+        # @param status [PollStatus]
+        # @param span [Object]
+        # @return [Boolean]
+        def handle_message_too_large(exception, batch, status, span)
+          Deimos.config.logger.error("Error publishing through DB Poller: #{exception.message}")
+          if @config.skip_too_large_messages
+            Deimos.config.logger.error("Skipping messages #{batch.map(&:id).join(', ')} since they are too large")
+            Deimos.config.tracer&.set_error(span, exception)
+            status.batches_errored += 1
+            true
+          else # do the same thing as regular Kafka::Error
+            sleep(0.5)
+            false
+          end
+        end
+
+        # rubocop:disable Metrics/AbcSize
         # @param batch [Array<ActiveRecord::Base>]
         # @param status [PollStatus]
         # @return [Boolean]
@@ -118,6 +137,9 @@ module Deimos
             process_batch(batch)
             Deimos.config.tracer&.finish(span)
             status.batches_processed += 1
+          rescue Kafka::BufferOverflow, Kafka::MessageSizeTooLarge,
+                 Kafka::RecordListTooLarge => e
+            retry unless handle_message_too_large(e, batch, status, span)
           rescue Kafka::Error => e # keep trying till it fixes itself
             Deimos.config.logger.error("Error publishing through DB Poller: #{e.message}")
             sleep(0.5)
@@ -139,6 +161,7 @@ module Deimos
           end
           true
         end
+        # rubocop:enable Metrics/AbcSize
 
         # Publish batch using the configured producers
         # @param batch [Array<ActiveRecord::Base>]

data/lib/deimos/utils/schema_class.rb

@@ -10,10 +10,25 @@ module Deimos
         # @return [Array<String>]
         def modules_for(namespace)
           modules = ['Schemas']
-          namespace_folder = namespace.split('.').last
-          if Deimos.config.schema.generate_namespace_folders && namespace_folder
-            modules.push(namespace_folder.underscore.classify)
+          namespace_override = nil
+          module_namespace = namespace
+
+          if Deimos.config.schema.use_full_namespace
+            if Deimos.config.schema.schema_namespace_map.present?
+              namespace_keys = Deimos.config.schema.schema_namespace_map.keys.sort_by { |k| -k.length }
+              namespace_override = namespace_keys.find { |k| module_namespace.include?(k) }
+            end
+
+            if namespace_override.present?
+              # override default module
+              modules = Array(Deimos.config.schema.schema_namespace_map[namespace_override])
+              module_namespace = module_namespace.gsub(/#{namespace_override}\.?/, '')
+            end
+
+            namespace_folders = module_namespace.split('.').map { |f| f.underscore.camelize }
+            modules.concat(namespace_folders) if namespace_folders.any?
           end
+
           modules
         end
 

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.23.2'
+  VERSION = '1.24.0'
 end

data/lib/generators/deimos/schema_class_generator.rb

@@ -121,9 +121,16 @@ module Deimos
           @main_class_definition = class_template
 
           file_prefix = schema.name.underscore.singularize
-          if Deimos.config.schema.generate_namespace_folders
-            file_prefix = "#{@modules.last.underscore.singularize}/#{file_prefix}"
+          if Deimos.config.schema.use_full_namespace
+            # Use entire namespace for folders
+            # but don't add directories that are already in the path
+            directories = @modules.map(&:underscore).select do |m|
+              Deimos.config.schema.generated_class_path.exclude?(m)
+            end
+
+            file_prefix = "#{directories.join('/')}/#{file_prefix}"
           end
+
           filename = "#{Deimos.config.schema.generated_class_path}/#{file_prefix}.rb"
           template(SCHEMA_CLASS_FILE, filename, force: true)
         end

data/regenerate_test_schema_classes.rb

@@ -21,8 +21,11 @@ Deimos.configure do |deimos_config|
   deimos_config.schema.path = "spec/schemas"
   deimos_config.schema.backend = :avro_validation
   deimos_config.schema.generated_class_path = './spec/schemas'
-  deimos_config.schema.generate_namespace_folders = true
-  deimos_config.schema.nest_child_schemas = true
+  deimos_config.schema.use_full_namespace = true
+  deimos_config.schema.schema_namespace_map = {
+    'com' => 'Schemas',
+    'com.my-namespace.my-suborg' => %w(Schemas MyNamespace)
+  }
 
   consumer do
     class_name 'MyConsumer'
@@ -56,6 +59,14 @@ Deimos.configure do |deimos_config|
     key_config field: :test_id
   end
 
+  consumer do
+    class_name 'MyConsumer'
+    topic 'MyTopic'
+    schema 'MyLongNamespaceSchema'
+    namespace 'com.my-namespace.my-suborg'
+    key_config field: :test_id
+  end
+
   producer do
     class_name 'MyConsumer'
     topic 'MyTopic'

data/spec/active_record_batch_consumer_spec.rb

@@ -78,7 +78,7 @@ module ActiveRecordBatchConsumerTest
           before(:each) do
             Deimos.configure do |config|
               config.schema.use_schema_classes = use_schema_classes
-              config.schema.generate_namespace_folders = true
+              config.schema.use_full_namespace = true
             end
           end
 

data/spec/active_record_consumer_spec.rb

@@ -143,7 +143,7 @@ module ActiveRecordConsumerTest
           before(:each) do
             Deimos.configure do |config|
               config.schema.use_schema_classes = use_schema_classes
-              config.schema.generate_namespace_folders = true
+              config.schema.use_full_namespace = true
             end
           end
 

data/spec/active_record_producer_spec.rb

@@ -71,7 +71,7 @@ describe Deimos::ActiveRecordProducer do
         before(:each) do
           Deimos.configure do |config|
             config.schema.use_schema_classes = use_schema_classes
-            config.schema.generate_namespace_folders = true
+            config.schema.use_full_namespace = true
           end
         end
 

data/spec/batch_consumer_spec.rb

@@ -42,7 +42,7 @@ module ConsumerTest
           before(:each) do
             Deimos.configure do |config|
               config.schema.use_schema_classes = use_schema_classes
-              config.schema.generate_namespace_folders = true
+              config.schema.use_full_namespace = true
             end
           end
 

data/spec/consumer_spec.rb

@@ -31,7 +31,7 @@ module ConsumerTest
           before(:each) do
             Deimos.configure do |config|
               config.schema.use_schema_classes = use_schema_classes
-              config.schema.generate_namespace_folders = true
+              config.schema.use_full_namespace = true
             end
           end
 
@@ -138,7 +138,7 @@ module ConsumerTest
         before(:each) do
           Deimos.configure do |config|
             config.schema.use_schema_classes = true
-            config.schema.generate_namespace_folders = true
+            config.schema.use_full_namespace = true
           end
         end
 

data/spec/generators/schema_class_generator_spec.rb

@@ -217,13 +217,30 @@ RSpec.describe Deimos::Generators::SchemaClassGenerator do
 
     context 'with namespace folders' do
       it 'should generate the correct classes' do
-        Deimos.with_config('schema.generate_namespace_folders' => true) do
+        Deimos.with_config('schema.use_full_namespace' => true) do
           described_class.start
           expect(files).to match_snapshot('namespace_folders', snapshot_serializer: MultiFileSerializer)
         end
       end
     end
 
+    context 'with namespace map' do
+      it 'should generate the correct classes' do
+        Deimos.with_config(
+          {
+            'schema.use_full_namespace' => true,
+            'schema.schema_namespace_map' => {
+              'com' => 'Schemas',
+              'com.my-namespace.my-suborg' => %w(Schemas MyNamespace)
+            }
+          }
+        ) do
+          described_class.start
+          expect(files).to match_snapshot('namespace_map', snapshot_serializer: MultiFileSerializer)
+        end
+      end
+    end
+
     context 'nested true' do
       it 'should generate the correct classes' do
         Deimos.with_config('schema.nest_child_schemas' => true) do

data/spec/schemas/com/my-namespace/my-suborg/MyLongNamespaceSchema.avsc

@@ -0,0 +1,18 @@
+{
+  "namespace": "com.my-namespace.my-suborg",
+  "name": "MyLongNamespaceSchema",
+  "type": "record",
+  "doc": "Test schema",
+  "fields": [
+    {
+      "name": "test_id",
+      "type": "string",
+      "doc": "test string"
+    },
+    {
+      "name": "some_int",
+      "type": "int",
+      "doc": "test int"
+    }
+  ]
+}

data/spec/schemas/my_namespace/generated.rb

@@ -128,7 +128,7 @@ module Schemas; module MyNamespace
     end
 
     def self.tombstone(key)
-      record = self.new
+      record = self.allocate
       record.tombstone_key = key
       record.a_string = key
       record

data/spec/schemas/my_namespace/my_long_namespace_schema.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+# This file is autogenerated by Deimos, Do NOT modify
+module Schemas; module MyNamespace
+  ### Primary Schema Class ###
+  # Autogenerated Schema for Record at com.my-namespace.my-suborg.MyLongNamespaceSchema
+  class MyLongNamespaceSchema < Deimos::SchemaClass::Record
+
+    ### Attribute Accessors ###
+    # @return [String]
+    attr_accessor :test_id
+    # @return [Integer]
+    attr_accessor :some_int
+
+    # @override
+    def initialize(test_id: nil,
+                   some_int: nil)
+      super
+      self.test_id = test_id
+      self.some_int = some_int
+    end
+
+    # @override
+    def schema
+      'MyLongNamespaceSchema'
+    end
+
+    # @override
+    def namespace
+      'com.my-namespace.my-suborg'
+    end
+
+    def self.tombstone(key)
+      record = self.allocate
+      record.tombstone_key = key
+      record.test_id = key
+      record
+    end
+
+    # @override
+    def as_json(_opts={})
+      {
+        'test_id' => @test_id,
+        'some_int' => @some_int
+      }
+    end
+  end
+end; end

data/spec/schemas/my_namespace/my_nested_schema.rb

@@ -104,7 +104,7 @@ module Schemas; module MyNamespace
     end
 
     def self.tombstone(key)
-      record = self.new
+      record = self.allocate
       record.tombstone_key = key
       record.test_id = key
       record

data/spec/schemas/my_namespace/my_schema.rb

@@ -43,7 +43,7 @@ module Schemas; module MyNamespace
     end
 
     def self.tombstone(key)
-      record = self.new
+      record = self.allocate
       record.tombstone_key = MySchemaKey.initialize_from_value(key)
       record.payload_key = key
       record

data/spec/schemas/my_namespace/my_schema_with_circular_reference.rb

@@ -68,7 +68,7 @@ module Schemas; module MyNamespace
     end
 
     def self.tombstone(key)
-      record = self.new
+      record = self.allocate
       record
     end
 

data/spec/schemas/my_namespace/my_schema_with_complex_type.rb

@@ -9,7 +9,7 @@ module Schemas; module MyNamespace
     ### Secondary Schema Classes ###
     # Autogenerated Schema for Record at com.my-namespace.ARecord
     class ARecord < Deimos::SchemaClass::Record
-
+  
       ### Attribute Accessors ###
       # @return [String]
       attr_accessor :a_record_field
@@ -185,7 +185,7 @@ module Schemas; module MyNamespace
     end
 
     def self.tombstone(key)
-      record = self.new
+      record = self.allocate
       record.tombstone_key = key
       record.test_id = key
       record

data/spec/schemas/my_namespace/my_updated_schema.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 # This file is autogenerated by Deimos, Do NOT modify
-module Schemas; module MyNamespace
+module Schemas; module Com; module MyNamespace
   ### Primary Schema Class ###
   # Autogenerated Schema for Record at com.my-namespace.MySchema
   class MyUpdatedSchema < Schemas::MyNamespace::MySchema
@@ -16,3 +16,4 @@ module Schemas; module MyNamespace
   end
 end
 end
+end

data/spec/schemas/{request → my_namespace/request}/create_topic.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 # This file is autogenerated by Deimos, Do NOT modify
-module Schemas; module Request
+module Schemas; module MyNamespace; module Request
   ### Primary Schema Class ###
   # Autogenerated Schema for Record at com.my-namespace.request.CreateTopic
   class CreateTopic < Deimos::SchemaClass::Record
@@ -33,4 +33,4 @@ module Schemas; module Request
       }
     end
   end
-end; end
+end; end; end

data/spec/schemas/{request → my_namespace/request}/index.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 # This file is autogenerated by Deimos, Do NOT modify
-module Schemas; module Request
+module Schemas; module MyNamespace; module Request
   ### Primary Schema Class ###
   # Autogenerated Schema for Record at com.my-namespace.request.Index
   class Index < Deimos::SchemaClass::Record
@@ -33,4 +33,4 @@ module Schemas; module Request
       }
     end
   end
-end; end
+end; end; end

data/spec/schemas/{request → my_namespace/request}/update_request.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 # This file is autogenerated by Deimos, Do NOT modify
-module Schemas; module Request
+module Schemas; module MyNamespace; module Request
   ### Primary Schema Class ###
   # Autogenerated Schema for Record at com.my-namespace.request.UpdateRequest
   class UpdateRequest < Deimos::SchemaClass::Record
@@ -33,4 +33,4 @@ module Schemas; module Request
       }
     end
   end
-end; end
+end; end; end

data/spec/schemas/{response → my_namespace/response}/create_topic.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 # This file is autogenerated by Deimos, Do NOT modify
-module Schemas; module Response
+module Schemas; module MyNamespace; module Response
   ### Primary Schema Class ###
   # Autogenerated Schema for Record at com.my-namespace.response.CreateTopic
   class CreateTopic < Deimos::SchemaClass::Record
@@ -33,4 +33,4 @@ module Schemas; module Response
       }
     end
   end
-end; end
+end; end; end

data/spec/schemas/{response → my_namespace/response}/index.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 # This file is autogenerated by Deimos, Do NOT modify
-module Schemas; module Response
+module Schemas; module MyNamespace; module Response
   ### Primary Schema Class ###
   # Autogenerated Schema for Record at com.my-namespace.response.Index
   class Index < Deimos::SchemaClass::Record
@@ -33,4 +33,4 @@ module Schemas; module Response
       }
     end
   end
-end; end
+end; end; end

data/spec/schemas/{response → my_namespace/response}/update_response.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 # This file is autogenerated by Deimos, Do NOT modify
-module Schemas; module Response
+module Schemas; module MyNamespace; module Response
   ### Primary Schema Class ###
   # Autogenerated Schema for Record at com.my-namespace.response.UpdateResponse
   class UpdateResponse < Deimos::SchemaClass::Record
@@ -33,4 +33,4 @@ module Schemas; module Response
       }
     end
   end
-end; end
+end; end; end

data/spec/snapshots/consumers-no-nest.snap

@@ -267,6 +267,50 @@ module Schemas
 end
 
 
+spec/app/lib/schema_classes/my_long_namespace_schema.rb:
+# frozen_string_literal: true
+
+# This file is autogenerated by Deimos, Do NOT modify
+module Schemas
+  ### Primary Schema Class ###
+  # Autogenerated Schema for Record at com.my-namespace.my-suborg.MyLongNamespaceSchema
+  class MyLongNamespaceSchema < Deimos::SchemaClass::Record
+
+    ### Attribute Accessors ###
+    # @return [String]
+    attr_accessor :test_id
+    # @return [Integer]
+    attr_accessor :some_int
+
+    # @override
+    def initialize(test_id: nil,
+                   some_int: nil)
+      super
+      self.test_id = test_id
+      self.some_int = some_int
+    end
+
+    # @override
+    def schema
+      'MyLongNamespaceSchema'
+    end
+
+    # @override
+    def namespace
+      'com.my-namespace.my-suborg'
+    end
+
+    # @override
+    def as_json(_opts={})
+      {
+        'test_id' => @test_id,
+        'some_int' => @some_int
+      }
+    end
+  end
+end
+
+
 spec/app/lib/schema_classes/my_nested_record.rb:
 # frozen_string_literal: true