deimos-ruby 1.14.2 → 1.14.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7052c11e5277eb8b2844e56f13c08921d1be7e3da28db5307b2b64e17b8b918a
-  data.tar.gz: 02fee15de11ebe50518ae96c91649d9d2341613967462348b4cfcebde3f8218d
+  metadata.gz: 872f449652ea05dfe9aabf1b144391b98d63fa03b193c7160931748b514ff534
+  data.tar.gz: cf6e3b54b909d1731a06d68fbfa1f7dbad5d78ce4e7de01bf712161061515ceb
 SHA512:
-  metadata.gz: 2f9f38797fbd74e5958f3910d4886144ca4ae329cbadaa5cb6474acdb4eeefdfd6edac59d79f3a1081df4fd24ae5766329b0d00649b613c13c44204a85e3b6b7
-  data.tar.gz: c5dd68694b23e7cdb10a2a2b6fe1163fe6f84f4c3b9de64eaea61c6b50f3ffc08642d72764e5550cf19f56bb14da7913611f45f6cd48a132e4707d5b6a201816
+  metadata.gz: e2737082fbada9724a9e94615308e4f7cdb17a36b8202dd1bf1f83c46e6378f5a6724eeb9e4cae5b86beabff94246ae90619b4b8e55f06064adf8dcd42e2ef14
+  data.tar.gz: 70b73b45dcb4b881bfa2c50aae105af62caf2d4aad2b103a807c1d776477c49621650434ae5aba9358b7cb1f977fe246e831f655bc7bfde902417bcf9a0b9ce5

data/CHANGELOG.md

@@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+# 1.14.5 - 2022-06-21
+
+- Fix crash with the tracer when error happens in decoding a message during batch consuming
+- Generate schema classes for all schemas, even without a consumer/producer set
+
+# 1.14.4 - 2022-06-18
+
+- Fix import in ActiveRecordConsumer on mysql
+
+# 1.14.3 - 2022-06-17
+
+- Fix issue with ActiveRecordConsumer double-decoding keys in batch mode.
+
 # 1.14.2 - 2022-05-26
 
 - Fix crash with `test_consume_message` when passing in an instance of a schema class instead of a hash.

data/README.md

@@ -818,7 +818,7 @@ Add the following configurations to start using generated schema classes in your
 Additionally, you can enable or disable the usage of schema classes for a particular consumer or producer with the
 `use_schema_classes` config. See [Configuration](./docs/CONFIGURATION.md#defining-producers).
 
-Note that if you have a schema in your repo but have not configured a producer or consumer, the generator will **not** generate a schema class for it. This is because you need to supply the `key_config` for the generator to know how to handle the schema key. 
+Note that if you have a schema in your repo but have not configured a producer or consumer, the generator will generate a schema class without a key schema.
 
 One additional configuration option indicates whether nested records should be generated as top-level classes or would remain nested inside the generated class for its parent schema. The default is to nest them, as a flattened structure can have one sub-schema clobber another sub-schema defined in a different top-level schema.
 

data/lib/deimos/active_record_consume/batch_consumption.rb

@@ -44,14 +44,12 @@ module Deimos
       # @param key [String] The encoded key.
       # @return [Hash] The key attributes.
       def record_key(key)
-        decoded_key = decode_key(key)
-
-        if decoded_key.nil?
+        if key.nil?
           {}
-        elsif decoded_key.is_a?(Hash)
-          @key_converter.convert(decoded_key)
+        elsif key.is_a?(Hash)
+          @key_converter.convert(key)
         else
-          { @klass.primary_key => decoded_key }
+          { @klass.primary_key => key }
         end
       end
 
@@ -102,12 +100,13 @@ module Deimos
 
         options = if key_cols.empty?
                     {} # Can't upsert with no key, just do regular insert
+                  elsif ActiveRecord::Base.connection.adapter_name.downcase =~ /mysql/
+                    {
+                      on_duplicate_key_update: :all
+                    }
                   else
                     {
                       on_duplicate_key_update: {
-                        # conflict_target must explicitly list the columns for
-                        # Postgres and SQLite. Not required for MySQL, but this
-                        # ensures consistent behaviour.
                         conflict_target: key_cols,
                         columns: :all
                       }

data/lib/deimos/consume/batch_consumption.rb

@@ -12,23 +12,23 @@ module Deimos
       # :nodoc:
       def around_consume_batch(batch, metadata)
         payloads = []
-        benchmark = Benchmark.measure do
-          if self.class.config[:key_configured]
-            metadata[:keys] = batch.map do |message|
-              decode_key(message.key)
+        _with_span do
+          benchmark = Benchmark.measure do
+            if self.class.config[:key_configured]
+              metadata[:keys] = batch.map do |message|
+                decode_key(message.key)
+              end
             end
-          end
-          metadata[:first_offset] = batch.first&.offset
+            metadata[:first_offset] = batch.first&.offset
 
-          payloads = batch.map do |message|
-            decode_message(message.payload)
-          end
-          _received_batch(payloads, metadata)
-          _with_span do
+            payloads = batch.map do |message|
+              decode_message(message.payload)
+            end
+            _received_batch(payloads, metadata)
             yield(payloads, metadata)
           end
+          _handle_batch_success(benchmark.real, payloads, metadata)
         end
-        _handle_batch_success(benchmark.real, payloads, metadata)
       rescue StandardError => e
         _handle_batch_error(e, payloads, metadata)
       end

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.14.2'
+  VERSION = '1.14.5'
 end

data/lib/generators/deimos/schema_class_generator.rb

@@ -41,7 +41,7 @@ module Deimos
         # Deimos Consumer or Producer Configuration object
         # @param schema_name [String]
         # @param namespace [String]
-        # @param key_schema_name [String]
+        # @param key_schema_name [String,nil]
         def generate_classes(schema_name, namespace, key_schema_name)
           schema_base = Deimos.schema_backend(schema: schema_name, namespace: namespace)
           schema_base.load_schema
@@ -143,10 +143,13 @@ module Deimos
       def generate
         _validate
         Rails.logger.info("Generating schemas from Deimos.config to #{Deimos.config.schema.generated_class_path}")
+        found_schemas = Set.new
         Deimos.config.producer_objects.each do |config|
           schema_name = config.schema
           namespace = config.namespace || Deimos.config.producers.schema_namespace
           key_schema_name = config.key_config[:schema]
+          found_schemas.add("#{namespace}.#{schema_name}")
+          found_schemas.add("#{namespace}.#{key_schema_name}") if key_schema_name
           generate_classes(schema_name, namespace, key_schema_name)
         end
 
@@ -154,12 +157,28 @@ module Deimos
           schema_name = config.schema
           namespace = config.namespace
           key_schema_name = config.key_config[:schema]
+          found_schemas.add("#{namespace}.#{schema_name}")
+          found_schemas.add("#{namespace}.#{key_schema_name}") if key_schema_name
           generate_classes(schema_name, namespace, key_schema_name)
         end
+
+        generate_from_schema_files(found_schemas)
+
       end
 
     private
 
+      def generate_from_schema_files(found_schemas)
+        schema_store = AvroTurf::MutableSchemaStore.new(path: Deimos.config.schema.path)
+        schema_store.load_schemas!
+        schema_store.schemas.values.each do |schema|
+          name = "#{schema.namespace}.#{schema.name}"
+          next if found_schemas.include?(name)
+
+          generate_classes(schema.name, schema.namespace, nil)
+        end
+      end
+
       # Determines if Schema Class Generation can be run.
       # @raise if Schema Backend is not of a Avro-based class
       def _validate

data/spec/generators/schema_class_generator_spec.rb

@@ -56,7 +56,6 @@ RSpec.describe Deimos::Generators::SchemaClassGenerator do
         end
       end
     end
-
   end
 
   context 'with a Consumers Schema with Complex types' do

data/spec/schemas/com/my-namespace/MySchemaId_key.avsc

@@ -6,7 +6,7 @@
   "fields": [
     {
       "name": "id",
-      "type": "integer"
+      "type": "int"
     }
   ]
 }