deimos-ruby 1.14.3 → 1.14.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0876f9454cd35ecb35310f57f9539dbb6d952aa2c5ab5b0503725de4620d7a1e'
-  data.tar.gz: 24fa6921f189d44aa36655338683992c1522ada69644c8570032df99e5d4ffc4
+  metadata.gz: a0c3d1a3e60a5e71ea4fd21de8b8f2ed9db064f8ea4f26891169f5e7690ad75c
+  data.tar.gz: afe45152f6834e529d1d9e63b5fd6e2f307d072464224f9e29d8d098d7053ca2
 SHA512:
-  metadata.gz: b5f8824b1ba370408d1cc58c6a33ee72ad15118453274a0e65bce6f421a4228b92e654cd54eac8771063ced121e93633e95b9244d046e35869c65609e34f077f
-  data.tar.gz: 0cfe69f0af3dfc463895fc1bbe16658efdb54a9afb45270cc4836235785e7137d7be36bab3a4b99503348cdfe118f63a39a18bc1dc718ec8411c4c80e57a2b98
+  metadata.gz: b1359ca3c178171f7999edb2dd43385a04a119489bfc4efa4f5811462c23203b88a1ea5661727718a6142dfc725137592166bef5df36d22fc1a4a6a7ff4a6bdc
+  data.tar.gz: 7629b9f1e75dad6ef0862958e448e7b750da434e62d0be754def299919a49f6ac3d0937588bf004c08ef331ea66aba9dcad989906781465c4a4c51c801acca51

data/CHANGELOG.md

@@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+# 1.14.6 - 2022-06-21
+
+- Add `generate_namespace_folders` to configuration; this will automatically generate subfolders to the `schemas` folder so that you can have different schemas with the same name but different namespaces generate separate classes.
+
+# 1.14.5 - 2022-06-21
+
+- Fix crash with the tracer when error happens in decoding a message during batch consuming
+- Generate schema classes for all schemas, even without a consumer/producer set
+
+# 1.14.4 - 2022-06-18
+
+- Fix import in ActiveRecordConsumer on mysql
+
 # 1.14.3 - 2022-06-17
 
 - Fix issue with ActiveRecordConsumer double-decoding keys in batch mode.

data/README.md

@@ -818,7 +818,7 @@ Add the following configurations to start using generated schema classes in your
 Additionally, you can enable or disable the usage of schema classes for a particular consumer or producer with the
 `use_schema_classes` config. See [Configuration](./docs/CONFIGURATION.md#defining-producers).
 
-Note that if you have a schema in your repo but have not configured a producer or consumer, the generator will **not** generate a schema class for it. This is because you need to supply the `key_config` for the generator to know how to handle the schema key. 
+Note that if you have a schema in your repo but have not configured a producer or consumer, the generator will generate a schema class without a key schema.
 
 One additional configuration option indicates whether nested records should be generated as top-level classes or would remain nested inside the generated class for its parent schema. The default is to nest them, as a flattened structure can have one sub-schema clobber another sub-schema defined in a different top-level schema.
 

data/docs/CONFIGURATION.md

@@ -194,6 +194,8 @@ schema.password|nil|Basic auth password.
 schema.path|nil|Local path to find your schemas.
 schema.use_schema_classes|false|Set this to true to use generated schema classes in your application.
 schema.generated_class_path|`app/lib/schema_classes`|Local path to generated schema classes.
+schema.nest_child_schemas|false|Set to true to nest subschemas within the generated class for the parent schema.
+schema.generate_namespace_folders|false|Set to true to generate folders for schemas matching the last part of the namespace.
 
 ## Database Producer Configuration
 

data/lib/deimos/active_record_consume/batch_consumption.rb

@@ -100,12 +100,13 @@ module Deimos
 
         options = if key_cols.empty?
                     {} # Can't upsert with no key, just do regular insert
+                  elsif ActiveRecord::Base.connection.adapter_name.downcase =~ /mysql/
+                    {
+                      on_duplicate_key_update: :all
+                    }
                   else
                     {
                       on_duplicate_key_update: {
-                        # conflict_target must explicitly list the columns for
-                        # Postgres and SQLite. Not required for MySQL, but this
-                        # ensures consistent behaviour.
                         conflict_target: key_cols,
                         columns: :all
                       }

data/lib/deimos/config/configuration.rb

@@ -347,6 +347,10 @@ module Deimos
       # Set to false to generate child schemas as their own files.
       # @return [Boolean]
       setting :nest_child_schemas, true
+
+      # Set to true to generate folders matching the last part of the schema namespace.
+      # @return [Boolean]
+      setting :generate_namespace_folders, false
     end
 
     # The configured metrics provider.

data/lib/deimos/consume/batch_consumption.rb

@@ -12,23 +12,23 @@ module Deimos
       # :nodoc:
       def around_consume_batch(batch, metadata)
         payloads = []
-        benchmark = Benchmark.measure do
-          if self.class.config[:key_configured]
-            metadata[:keys] = batch.map do |message|
-              decode_key(message.key)
+        _with_span do
+          benchmark = Benchmark.measure do
+            if self.class.config[:key_configured]
+              metadata[:keys] = batch.map do |message|
+                decode_key(message.key)
+              end
             end
-          end
-          metadata[:first_offset] = batch.first&.offset
+            metadata[:first_offset] = batch.first&.offset
 
-          payloads = batch.map do |message|
-            decode_message(message.payload)
-          end
-          _received_batch(payloads, metadata)
-          _with_span do
+            payloads = batch.map do |message|
+              decode_message(message.payload)
+            end
+            _received_batch(payloads, metadata)
             yield(payloads, metadata)
           end
+          _handle_batch_success(benchmark.real, payloads, metadata)
         end
-        _handle_batch_success(benchmark.real, payloads, metadata)
       rescue StandardError => e
         _handle_batch_error(e, payloads, metadata)
       end

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.14.3'
+  VERSION = '1.14.6'
 end

data/lib/generators/deimos/schema_class/templates/schema_class.rb.tt

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 # This file is autogenerated by Deimos, Do NOT modify
-module Schemas
+<%= @modules.map { |m| "module #{m}"}.join('; ') %>
   ### Primary Schema Class ###
   <%=- @main_class_definition -%>
 
-end
+<%= @modules.map { "end" }.join('; ') %>

data/lib/generators/deimos/schema_class_generator.rb

@@ -41,7 +41,7 @@ module Deimos
         # Deimos Consumer or Producer Configuration object
         # @param schema_name [String]
         # @param namespace [String]
-        # @param key_schema_name [String]
+        # @param key_schema_name [String,nil]
         def generate_classes(schema_name, namespace, key_schema_name)
           schema_base = Deimos.schema_backend(schema: schema_name, namespace: namespace)
           schema_base.load_schema
@@ -108,9 +108,17 @@ module Deimos
         # @param key_schema_base [Avro::Schema::NamedSchema, nil]
         def write_file(schema, key_schema_base)
           class_template = _generate_class_template_from_schema(schema, key_schema_base)
+          @modules = ['Schemas']
+          namespace_folder = schema.namespace.split('.').last
+          if Deimos.config.schema.generate_namespace_folders
+            @modules.push(namespace_folder.underscore.classify)
+          end
           @main_class_definition = class_template
 
           file_prefix = schema.name.underscore.singularize
+          if Deimos.config.schema.generate_namespace_folders
+            file_prefix = "#{namespace_folder}/#{file_prefix}"
+          end
           filename = "#{Deimos.config.schema.generated_class_path}/#{file_prefix}.rb"
           template(SCHEMA_CLASS_FILE, filename, force: true)
         end
@@ -143,10 +151,13 @@ module Deimos
       def generate
         _validate
         Rails.logger.info("Generating schemas from Deimos.config to #{Deimos.config.schema.generated_class_path}")
+        found_schemas = Set.new
         Deimos.config.producer_objects.each do |config|
           schema_name = config.schema
           namespace = config.namespace || Deimos.config.producers.schema_namespace
           key_schema_name = config.key_config[:schema]
+          found_schemas.add("#{namespace}.#{schema_name}")
+          found_schemas.add("#{namespace}.#{key_schema_name}") if key_schema_name
           generate_classes(schema_name, namespace, key_schema_name)
         end
 
@@ -154,12 +165,28 @@ module Deimos
           schema_name = config.schema
           namespace = config.namespace
           key_schema_name = config.key_config[:schema]
+          found_schemas.add("#{namespace}.#{schema_name}")
+          found_schemas.add("#{namespace}.#{key_schema_name}") if key_schema_name
           generate_classes(schema_name, namespace, key_schema_name)
         end
+
+        generate_from_schema_files(found_schemas)
+
       end
 
     private
 
+      def generate_from_schema_files(found_schemas)
+        schema_store = AvroTurf::MutableSchemaStore.new(path: Deimos.config.schema.path)
+        schema_store.load_schemas!
+        schema_store.schemas.values.each do |schema|
+          name = "#{schema.namespace}.#{schema.name}"
+          next if found_schemas.include?(name)
+
+          generate_classes(schema.name, schema.namespace, nil)
+        end
+      end
+
       # Determines if Schema Class Generation can be run.
       # @raise if Schema Backend is not of a Avro-based class
       def _validate

data/spec/generators/schema_class_generator_spec.rb

@@ -11,7 +11,7 @@ end
 
 RSpec.describe Deimos::Generators::SchemaClassGenerator do
   let(:schema_class_path) { 'spec/app/lib/schema_classes' }
-  let(:files) { Dir["#{schema_class_path}/*.rb"].map { |f| [f, File.read(f)]}.to_h }
+  let(:files) { Dir["#{schema_class_path}/**/*.rb"].map { |f| [f, File.read(f)]}.to_h }
 
   before(:each) do
     Deimos.config.reset!
@@ -56,7 +56,6 @@ RSpec.describe Deimos::Generators::SchemaClassGenerator do
         end
       end
     end
-
   end
 
   context 'with a Consumers Schema with Complex types' do
@@ -216,6 +215,15 @@ RSpec.describe Deimos::Generators::SchemaClassGenerator do
       end
     end
 
+    context 'with namespace folders' do
+      it 'should generate the correct classes' do
+        Deimos.with_config('schema.generate_namespace_folders' => true) do
+          described_class.start
+          expect(files).to match_snapshot('namespace_folders', snapshot_serializer: MultiFileSerializer)
+        end
+      end
+    end
+
     context 'nested true' do
       it 'should generate the correct classes' do
         Deimos.with_config('schema.nest_child_schemas' => true) do

data/spec/schemas/com/my-namespace/MySchemaId_key.avsc

@@ -6,7 +6,7 @@
   "fields": [
     {
       "name": "id",
-      "type": "integer"
+      "type": "int"
     }
   ]
 }