deimos-ruby 2.4.0.pre.beta8 → 2.4.0.pre.beta9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c1b2dfa779652664bad07fb938a1f4472bca69b7c61c21b4cb6fe028c86061a9
-  data.tar.gz: 0bf9e789a83563e10b3bd885de8736c5b4487268e4b9cc1225ac8f8f3ee2a866
+  metadata.gz: 63128b3ded679a1adf4a45cf2ba2cd18e5734f493d707a60d720df1d2112b249
+  data.tar.gz: 829675d8502e3229f79ea5a57f417f181e825c15f0d87f60ff4d1779580b458e
 SHA512:
-  metadata.gz: b089bcb150fabaabf23b3d4431f8de937080961b298c3160c7b02db5e41468f524b9d0f8b634f3b69ae99dadf12e2c67b06d420c0821f24ff42eb4c7f6222b2e
-  data.tar.gz: 41bf155c6813975fdda7818d9199a1334572e70601e7a16076430325f0e1bbe4a9983c0169eaac0e91ede2081947c762a1e6a618f379ad1ade21c78711cfcd49
+  metadata.gz: b6fe15786e0c5ec7bc173354ace9763ae911b922b4e3f6312e09e5cdf56071076a72d0cb561658826a98ac1214bf58c97ae3124d690408a42a6530d6fe33d7eb
+  data.tar.gz: fa6ff30280e567c252c7d763d25c2726669ce2809efcd52bf76ac8f5d8d93538bd93f5d98ba5858e97d57217c794eccb2f8983338db9cd8259218ed1f87715e4

data/README.md

@@ -281,9 +281,15 @@ MyProducer.publish({
                    })
 ```
 
+### Protobuf and Key Schemas
+
 > [!IMPORTANT]
 > Protobuf should *not* be used as a key schema, since the binary encoding is [unstable](https://protobuf.dev/programming-guides/encoding/#implications) and may break partitioning. Deimos will automatically convert keys to sorted JSON, and will use JSON Schema in the schema registry.
 
+To enable integration with [buf](https://buf.build/), Deimos provides a Rake task that will auto-generate `.proto` files. This task can be run in CI before running `buf push`. This way, downstream systems that want to use the generated key schemas can do so using their own tech stack.
+
+    bundle exec rake deimos:generate_key_protos
+
 ## Instrumentation
 
 Deimos will send events through the [Karafka instrumentation monitor](https://karafka.io/docs/Monitoring-and-Logging/#subscribing-to-the-instrumentation-events). 

data/lib/deimos/ext/producer_route.rb

@@ -10,9 +10,10 @@ module Deimos
       end
 
       def producer_class
-        self.producer_classes.first
+        self.producer_classes&.first
       end
     end
+
     module Topic
       (FIELDS + [:producer_class]).each do |field|
         define_method(field) do |*args|

data/lib/deimos/schema_backends/proto_schema_registry.rb

@@ -56,6 +56,29 @@ module Deimos
         )
       end
 
+      # @param file [String]
+      # @param field_name [String]
+      def write_key_proto(file, field_name)
+        return if field_name.nil?
+
+        proto = proto_schema
+        package = proto.file_descriptor.to_proto.package
+        writer = SchemaRegistry::Output::ProtoText::Writer.new
+        info = SchemaRegistry::Output::ProtoText::ParseInfo.new(writer, package)
+        writer.write_line(%(syntax = "proto3";))
+        writer.write_line("package #{package};")
+        writer.writenl
+
+        field = proto.to_proto.field.find { |f| f.name == field_name.to_s }
+        writer.write_line("message #{proto.to_proto.name}Key {")
+        writer.indent
+        SchemaRegistry::Output::ProtoText.write_field(info, field)
+        writer.dedent
+        writer.write_line("}")
+        path = "#{file}/#{package.gsub('.', '/')}/#{proto.to_proto.name.underscore}_key.proto"
+        File.write(path, writer.string)
+      end
+
     end
   end
 end

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '2.4.0-beta8'
+  VERSION = '2.4.0-beta9'
 end

data/lib/deimos.rb

@@ -72,9 +72,18 @@ module Deimos
       klass
     end
 
+    # @param topic_name [String]
+    # @return [SchemaBackends::Base]
+    def schema_backend_for(topic_name)
+      config = Deimos.karafka_config_for(topic: topic_name)
+      self.schema_backend(schema: config.schema,
+                          namespace: config.namespace,
+                          backend: config.schema_backend || Deimos.config.schema.backend)
+    end
+
     # @param schema [String, Symbol]
     # @param namespace [String]
-    # @return [Class<Deimos::SchemaBackends::Base>]
+    # @return [Deimos::SchemaBackends::Base]
     def schema_backend(schema:, namespace:, backend: Deimos.config.schema.backend)
       if config.schema.use_schema_classes
         # Initialize an instance of the provided schema

data/lib/tasks/deimos.rake

@@ -2,6 +2,7 @@
 
 require 'generators/deimos/schema_class_generator'
 require 'optparse'
+require 'deimos/schema_backends/proto_schema_registry'
 
 namespace :deimos do
   desc 'Starts Deimos in the rails environment'
@@ -40,4 +41,17 @@ namespace :deimos do
     Deimos::Generators::SchemaClassGenerator.start
   end
 
+  desc 'Output Protobuf key schemas'
+  task generate_key_protos: :environment do
+    puts "Generating Protobuf key schemas"
+    Deimos.karafka_configs.each do |config|
+      next if config.try(:producer_class).blank? ||
+              !Deimos.schema_backend_for(config.name).is_a?(Deimos::SchemaBackends::ProtoSchemaRegistry) ||
+              config.deserializers[:key].try(:key_field).blank?
+
+      puts "Writing key proto for #{config.name}"
+      backend = Deimos.schema_backend_for(config.name)
+      backend.write_key_proto('protos', config.deserializers[:key].key_field)
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 2.4.0.pre.beta8
+  version: 2.4.0.pre.beta9
 platform: ruby
 authors:
 - Daniel Orner