deimos-ruby 1.14.4 → 1.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 81c75c30750ad501a4d954681e7ea0c5ddef83c54e313a284d6abb0fde4c059a
-  data.tar.gz: 223a1cf8f77db5d0cb901a00c1a4a1eeb7eb2c9a243e8fa07a5deb32bcac849e
+  metadata.gz: 0dd1a31f2dd75e160a7d63b4588b9cfc1f489be46650508aae3797f6d032110b
+  data.tar.gz: 8e1167cd5481a96ce18a53d46a9f153f9266d909bc54a50cc727ffe8d4a86244
 SHA512:
-  metadata.gz: 95605e56b2852137a1645fa3497903d138045cdcf51902c4cad29321e78d0626f2ceb439789db46b69cb08f305f580efa804c768c9a781c0f96e677e4cf2f340
-  data.tar.gz: 4375d5c1bdde5b53598a1a11910615fda36d69e1dd92650becf710dd312425f07521fa70e0bebe4567e1d9931b5f70e71654f3dc0c74685141cb945193019672
+  metadata.gz: cfad01dea7198a9ecf8b063433abc47d81db127beedf54213a82f995713c5b28a706ca43dba9129b28e38cfc8d1e0144f078786507a32e359f822a71ef9c0927
+  data.tar.gz: 9dc3e5dadac11d3b32a386ebbd938fc128a32bdfb511674489491a266e58c13074708d30fc89f47534e481949e59eeb18cbd4858eff9d27a28946408234c55ed

data/CHANGELOG.md

@@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+# 1.15.0 - 2022-07-20
+
+- Update to `sigurd` 0.1.0 - DB producer should now exit when receiving a `SIGTERM` instead of throwing a `SignalException`
+
+# 1.14.6 - 2022-06-21
+
+- Add `generate_namespace_folders` to configuration; this will automatically generate subfolders to the `schemas` folder so that you can have different schemas with the same name but different namespaces generate separate classes.
+
+# 1.14.5 - 2022-06-21
+
+- Fix crash with the tracer when error happens in decoding a message during batch consuming
+- Generate schema classes for all schemas, even without a consumer/producer set
+
 # 1.14.4 - 2022-06-18
 
 - Fix import in ActiveRecordConsumer on mysql

data/README.md

@@ -818,7 +818,7 @@ Add the following configurations to start using generated schema classes in your
 Additionally, you can enable or disable the usage of schema classes for a particular consumer or producer with the
 `use_schema_classes` config. See [Configuration](./docs/CONFIGURATION.md#defining-producers).
 
-Note that if you have a schema in your repo but have not configured a producer or consumer, the generator will **not** generate a schema class for it. This is because you need to supply the `key_config` for the generator to know how to handle the schema key. 
+Note that if you have a schema in your repo but have not configured a producer or consumer, the generator will generate a schema class without a key schema.
 
 One additional configuration option indicates whether nested records should be generated as top-level classes or would remain nested inside the generated class for its parent schema. The default is to nest them, as a flattened structure can have one sub-schema clobber another sub-schema defined in a different top-level schema.
 

data/deimos-ruby.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency('avro_turf', '>= 0.11', '< 2')
   spec.add_runtime_dependency('phobos', '>= 1.9', '< 3.0')
   spec.add_runtime_dependency('ruby-kafka', '< 2')
-  spec.add_runtime_dependency('sigurd', '~> 0.0.1')
+  spec.add_runtime_dependency('sigurd', '>= 0.1.0', '< 1.0')
   spec.add_runtime_dependency('fig_tree', '~> 0.0.2')
 
   spec.add_development_dependency('activerecord-import')

data/docs/CONFIGURATION.md

@@ -194,6 +194,8 @@ schema.password|nil|Basic auth password.
 schema.path|nil|Local path to find your schemas.
 schema.use_schema_classes|false|Set this to true to use generated schema classes in your application.
 schema.generated_class_path|`app/lib/schema_classes`|Local path to generated schema classes.
+schema.nest_child_schemas|false|Set to true to nest subschemas within the generated class for the parent schema.
+schema.generate_namespace_folders|false|Set to true to generate folders for schemas matching the last part of the namespace.
 
 ## Database Producer Configuration
 

data/lib/deimos/config/configuration.rb

@@ -347,6 +347,10 @@ module Deimos
       # Set to false to generate child schemas as their own files.
       # @return [Boolean]
       setting :nest_child_schemas, true
+
+      # Set to true to generate folders matching the last part of the schema namespace.
+      # @return [Boolean]
+      setting :generate_namespace_folders, false
     end
 
     # The configured metrics provider.

data/lib/deimos/consume/batch_consumption.rb

@@ -12,23 +12,23 @@ module Deimos
       # :nodoc:
       def around_consume_batch(batch, metadata)
         payloads = []
-        benchmark = Benchmark.measure do
-          if self.class.config[:key_configured]
-            metadata[:keys] = batch.map do |message|
-              decode_key(message.key)
+        _with_span do
+          benchmark = Benchmark.measure do
+            if self.class.config[:key_configured]
+              metadata[:keys] = batch.map do |message|
+                decode_key(message.key)
+              end
             end
-          end
-          metadata[:first_offset] = batch.first&.offset
+            metadata[:first_offset] = batch.first&.offset
 
-          payloads = batch.map do |message|
-            decode_message(message.payload)
-          end
-          _received_batch(payloads, metadata)
-          _with_span do
+            payloads = batch.map do |message|
+              decode_message(message.payload)
+            end
+            _received_batch(payloads, metadata)
             yield(payloads, metadata)
           end
+          _handle_batch_success(benchmark.real, payloads, metadata)
         end
-        _handle_batch_success(benchmark.real, payloads, metadata)
       rescue StandardError => e
         _handle_batch_error(e, payloads, metadata)
       end

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.14.4'
+  VERSION = '1.15.0'
 end

data/lib/deimos.rb

@@ -83,6 +83,7 @@ module Deimos
     # Start the DB producers to send Kafka messages.
     # @param thread_count [Integer] the number of threads to start.
     def start_db_backend!(thread_count: 1)
+      Sigurd.exit_on_signal = true
       if self.config.producers.backend != :db
         raise('Publish backend is not set to :db, exiting')
       end

data/lib/generators/deimos/schema_class/templates/schema_class.rb.tt

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 # This file is autogenerated by Deimos, Do NOT modify
-module Schemas
+<%= @modules.map { |m| "module #{m}"}.join('; ') %>
   ### Primary Schema Class ###
   <%=- @main_class_definition -%>
 
-end
+<%= @modules.map { "end" }.join('; ') %>

data/lib/generators/deimos/schema_class_generator.rb

@@ -41,7 +41,7 @@ module Deimos
         # Deimos Consumer or Producer Configuration object
         # @param schema_name [String]
         # @param namespace [String]
-        # @param key_schema_name [String]
+        # @param key_schema_name [String,nil]
         def generate_classes(schema_name, namespace, key_schema_name)
           schema_base = Deimos.schema_backend(schema: schema_name, namespace: namespace)
           schema_base.load_schema
@@ -108,9 +108,17 @@ module Deimos
         # @param key_schema_base [Avro::Schema::NamedSchema, nil]
         def write_file(schema, key_schema_base)
           class_template = _generate_class_template_from_schema(schema, key_schema_base)
+          @modules = ['Schemas']
+          namespace_folder = schema.namespace.split('.').last
+          if Deimos.config.schema.generate_namespace_folders
+            @modules.push(namespace_folder.underscore.classify)
+          end
           @main_class_definition = class_template
 
           file_prefix = schema.name.underscore.singularize
+          if Deimos.config.schema.generate_namespace_folders
+            file_prefix = "#{namespace_folder}/#{file_prefix}"
+          end
           filename = "#{Deimos.config.schema.generated_class_path}/#{file_prefix}.rb"
           template(SCHEMA_CLASS_FILE, filename, force: true)
         end
@@ -143,10 +151,13 @@ module Deimos
       def generate
         _validate
         Rails.logger.info("Generating schemas from Deimos.config to #{Deimos.config.schema.generated_class_path}")
+        found_schemas = Set.new
         Deimos.config.producer_objects.each do |config|
           schema_name = config.schema
           namespace = config.namespace || Deimos.config.producers.schema_namespace
           key_schema_name = config.key_config[:schema]
+          found_schemas.add("#{namespace}.#{schema_name}")
+          found_schemas.add("#{namespace}.#{key_schema_name}") if key_schema_name
           generate_classes(schema_name, namespace, key_schema_name)
         end
 
@@ -154,12 +165,28 @@ module Deimos
           schema_name = config.schema
           namespace = config.namespace
           key_schema_name = config.key_config[:schema]
+          found_schemas.add("#{namespace}.#{schema_name}")
+          found_schemas.add("#{namespace}.#{key_schema_name}") if key_schema_name
           generate_classes(schema_name, namespace, key_schema_name)
         end
+
+        generate_from_schema_files(found_schemas)
+
       end
 
     private
 
+      def generate_from_schema_files(found_schemas)
+        schema_store = AvroTurf::MutableSchemaStore.new(path: Deimos.config.schema.path)
+        schema_store.load_schemas!
+        schema_store.schemas.values.each do |schema|
+          name = "#{schema.namespace}.#{schema.name}"
+          next if found_schemas.include?(name)
+
+          generate_classes(schema.name, schema.namespace, nil)
+        end
+      end
+
       # Determines if Schema Class Generation can be run.
       # @raise if Schema Backend is not of a Avro-based class
       def _validate

data/spec/generators/schema_class_generator_spec.rb

@@ -11,7 +11,7 @@ end
 
 RSpec.describe Deimos::Generators::SchemaClassGenerator do
   let(:schema_class_path) { 'spec/app/lib/schema_classes' }
-  let(:files) { Dir["#{schema_class_path}/*.rb"].map { |f| [f, File.read(f)]}.to_h }
+  let(:files) { Dir["#{schema_class_path}/**/*.rb"].map { |f| [f, File.read(f)]}.to_h }
 
   before(:each) do
     Deimos.config.reset!
@@ -56,7 +56,6 @@ RSpec.describe Deimos::Generators::SchemaClassGenerator do
         end
       end
     end
-
   end
 
   context 'with a Consumers Schema with Complex types' do
@@ -216,6 +215,15 @@ RSpec.describe Deimos::Generators::SchemaClassGenerator do
       end
     end
 
+    context 'with namespace folders' do
+      it 'should generate the correct classes' do
+        Deimos.with_config('schema.generate_namespace_folders' => true) do
+          described_class.start
+          expect(files).to match_snapshot('namespace_folders', snapshot_serializer: MultiFileSerializer)
+        end
+      end
+    end
+
     context 'nested true' do
       it 'should generate the correct classes' do
         Deimos.with_config('schema.nest_child_schemas' => true) do

data/spec/schemas/com/my-namespace/MySchemaId_key.avsc

@@ -6,7 +6,7 @@
   "fields": [
     {
       "name": "id",
-      "type": "integer"
+      "type": "int"
     }
   ]
 }