deimos-ruby 1.11.0 → 1.12.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ac454bfdafdeec22f07c9b00386dfcdd687ed1320288384f53f4541eb6037589
-  data.tar.gz: a012d3b33de492bf84d17c74da545dbd0180377fde07b0e52f8002806e3cc489
+  metadata.gz: 456ac47b0f41a41e874a109a005d50650ffc36ae0ca9e924f5e06a0c1b7599ef
+  data.tar.gz: 14f3e9591b4a32ea503bd063416d13cdd3625bcd220bbe6f61675f1fb911afe6
 SHA512:
-  metadata.gz: 6bad789e44e21108e652119640e641f4efa217398d8db89b1b734373f5bdc23cbdd60605828e4f6df9e366bac948512c9ff041222db8fd858605fb5a82e7fea9
-  data.tar.gz: 218fb6e258b3a7859644ece11feff5e27738cbe85c20830f704128c5e9fa3d50e764b4d178fc58ca31b35de90839cb1411d1cf98bcfe73c601162c8cb5017d83
+  metadata.gz: a5f0f9e5d2e0e68c42102166490a929602ef0327c84d42f987200c5c7c941135b7120ca18827707d056a57e9e9ee6e923a7bba91df84f5b7873f1b24e96597f6
+  data.tar.gz: ad75c79468a87c1a68658099ce9234730926cc049897febd797b2bedbfea316b5ff5b51a8fe215afbd087cf90ca93f2c292b9733128e5470f09cf3e3120d7cce

data/CHANGELOG.md

@@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+# 1.12.1 - 2021-11-02
+
+- ### Fixes :wrench:
+- Fixed issue where Schema Class Consumer/Producer are using `Deimos::` instead of `Schema::` for instances of classes.
+
+# 1.12.0 - 2021-11-01
+
+### Features :star:
+
+- Generate Schema classes from Avro Schemas
+- Use Schema Classes in your consumer and producer 
+
 ## 1.11.0 - 2021-08-27
 
 - ### Fixes :wrench:

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    deimos-ruby (1.10.2)
+    deimos-ruby (1.12.0)
       avro_turf (~> 0.11)
       fig_tree (~> 0.0.2)
       phobos (>= 1.9, < 3.0)
@@ -88,11 +88,11 @@ GEM
     ddtrace (0.46.0)
       msgpack
     diff-lcs (1.4.4)
-    digest-crc (0.6.3)
+    digest-crc (0.6.4)
       rake (>= 12.0.0, < 14.0.0)
     dogstatsd-ruby (4.8.3)
     erubi (1.10.0)
-    excon (0.81.0)
+    excon (0.85.0)
     exponential-backoff (0.0.4)
     ffi (1.15.0)
     fig_tree (0.0.2)
@@ -153,10 +153,10 @@ GEM
       nenv (~> 0.1)
       shellany (~> 0.0)
     parallel (1.20.1)
-    parser (3.0.0.0)
+    parser (3.0.1.1)
       ast (~> 2.4.1)
     pg (1.2.3)
-    phobos (2.0.2)
+    phobos (2.1.0)
       activesupport (>= 3.0.0)
       concurrent-ruby (>= 1.0.2)
       concurrent-ruby-ext (>= 1.0.2)
@@ -227,7 +227,7 @@ GEM
     rspec-support (3.10.2)
     rspec_junit_formatter (0.4.1)
       rspec-core (>= 2, < 4, != 2.12.0)
-    rubocop (0.88.0)
+    rubocop (0.89.0)
       parallel (~> 1.10)
       parser (>= 2.7.1.1)
       rainbow (>= 2.2.2, < 4.0)
@@ -240,7 +240,7 @@ GEM
       parser (>= 2.7.1.5)
     rubocop-rspec (1.42.0)
       rubocop (>= 0.87.0)
-    ruby-kafka (1.3.0)
+    ruby-kafka (1.4.0)
       digest-crc
     ruby-progressbar (1.11.0)
     shellany (0.0.1)
@@ -284,7 +284,7 @@ DEPENDENCIES
   rspec (~> 3)
   rspec-rails (~> 4)
   rspec_junit_formatter (~> 0.3)
-  rubocop (= 0.88.0)
+  rubocop (= 0.89.0)
   rubocop-rspec (= 1.42.0)
   sqlite3 (~> 1.3)
 

data/README.md

@@ -27,6 +27,7 @@ Built on Phobos and hence Ruby-Kafka.
    * [Database Backend](#database-backend)
    * [Database Poller](#database-poller)
    * [Running Consumers](#running-consumers)
+   * [Generated Schema Classes](#generated-schema-classes)
    * [Metrics](#metrics)
    * [Testing](#testing)
         * [Test Helpers](#test-helpers)
@@ -795,6 +796,101 @@ which can be useful if you want to figure out if you're inside the task
 as opposed to running your Rails server or console. E.g. you could start your 
 DB backend only when your rake task is running.
 
+## Generated Schema Classes
+
+Deimos offers a way to generate classes from Avro schemas. These classes are documented
+with YARD to aid in IDE auto-complete, and will help to move errors closer to the code.
+
+Add the following configurations for schema class generation: 
+
+```ruby
+config.schema.generated_class_path 'path/to/generated/classes' # Defaults to 'app/lib/schema_classes'
+```
+
+Run the following command to generate schema classes in your application. It will generate classes for every configured consumer or producer by `Deimos.configure`:
+
+    bundle exec rake deimos:generate_schema_classes
+
+Add the following configurations to start using generated schema classes in your application's Consumers and Producers:
+
+    config.schema.use_schema_classes true
+
+Additionally, you can enable or disable the usage of schema classes for a particular consumer or producer with the
+`use_schema_classes` config. See [Configuration](./docs/CONFIGURATION.md#defining-producers).
+
+### Consumer
+
+The consumer interface uses the `decode_message` method to turn JSON hash into the Schemas
+generated Class and provides it to the `consume`/`consume_batch` methods for their use.
+
+Examples of consumers would look like this:
+```ruby
+class MyConsumer < Deimos::Consumer
+  def consume(payload, metadata)
+    # Same method as Phobos consumers but payload is now an instance of Deimos::SchemaClass::Record
+    # rather than a hash. metadata is still a hash that contains information like :key and :topic.
+    # You can interact with the schema class instance in the following way: 
+    do_something(payload.test_id, payload.some_int)
+    # The original behaviour was as follows:
+    do_something(payload[:test_id], payload[:some_int])
+  end
+end
+```
+
+```ruby
+class MyActiveRecordConsumer < Deimos::ActiveRecordConsumer
+  record_class Widget
+  # Any method that expects a message payload as a hash will instead
+  # receive an instance of Deimos::SchemaClass::Record.
+  def record_attributes(payload, key)
+    # You can interact with the schema class instance in the following way:
+    super.merge(:some_field => "some_value-#{payload.test_id}")
+    # The original behaviour was as follows:
+    super.merge(:some_field => "some_value-#{payload[:test_id]}")
+  end
+end
+```
+
+### Producer
+Similarly to the consumer interface, the producer interface for using Schema Classes in your app
+relies on the `publish`/`publish_list` methods to convert a _provided_ instance of a Schema Class
+into a hash that can be used freely by the Kafka client.
+
+Examples of producers would look like this:
+```ruby
+class MyProducer < Deimos::Producer
+  class << self 
+    # @param test_id [String]
+    # @param some_int [Integer]
+    def self.send_a_message(test_id, some_int)
+      # Instead of sending in a Hash object to the publish or publish_list method,      
+      # you can initialize an instance of your schema class and send that in.
+      message = Schemas::MySchema.new(
+        test_id: test_id,
+        some_int: some_int
+      )
+      self.publish(message)
+      self.publish_list([message])
+    end
+  end
+end
+```
+
+```ruby
+class MyActiveRecordProducer < Deimos::ActiveRecordProducer
+  record_class Widget
+  # @param payload [Deimos::SchemaClass::Record]
+  # @param _record [Widget]
+  def self.generate_payload(attributes, _record)
+    # This method converts your ActiveRecord into a Deimos::SchemaClass::Record. You will be able to use super
+    # as an instance of Schemas::MySchema and set values that are not on your ActiveRecord schema.
+    res = super
+    res.some_value = "some_value-#{res.test_id}"
+    res
+  end
+end
+```
+
 # Metrics
 
 Deimos includes some metrics reporting out the box. It ships with DataDog support, but you can add custom metric providers as well.
@@ -880,8 +976,7 @@ Also see [deimos.rb](lib/deimos.rb) under `Configure tracing` to see how the tra
 
 # Testing
 
-Deimos comes with a test helper class which sets the various backends
-to mock versions, and provides useful methods for testing consumers.
+Deimos comes with a test helper class which provides useful methods for testing consumers.
 
 In `spec_helper.rb`:
 ```ruby
@@ -890,7 +985,56 @@ RSpec.configure do |config|
 end
 ```
 
-In your test, you now have the following methods available:
+## Test Configuration
+
+```ruby
+# The following can be added to a rpsec file so that each unit 
+# test can have the same settings every time it is run
+around(:each) do |example|
+  Deimos::TestHelpers.unit_test!
+  example.run
+  Deimos.config.reset!
+end
+
+# Similarly you can use the Kafka test helper
+around(:each) do |example|
+  Deimos::TestHelpers.kafka_test!
+  example.run
+  Deimos.config.reset!
+end
+
+# Kakfa test helper using schema registry
+around(:each) do |example|
+  Deimos::TestHelpers.full_integration_test!
+  example.run
+  Deimos.config.reset!
+end
+```
+
+With the help of these helper methods, rspec examples can be written without having to tinker with Deimos settings.
+This also prevents Deimos setting changes from leaking in to other examples.
+
+This does not take away the ability to configure Deimos manually in individual examples. Deimos can still be configured like so:
+```ruby
+    it 'should not fail this random test' do
+      
+      Deimos.configure do |config|
+        config.consumers.fatal_error = proc { true }
+        config.consumers.reraise_errors = false
+      end
+      ...
+      expect(some_object).to be_truthy
+      ...
+      Deimos.config.reset!
+    end
+```
+If you are using one of the test helpers in an `around(:each)` block and want to override few settings for one example, 
+you can do it like in the example shown above. These settings would only apply to that specific example and the Deimos config should
+reset once the example has finished running.
+
+## Test Usage
+
+In your tests, you now have the following methods available:
 ```ruby
 # Pass a consumer class (not instance) to validate a payload against it.
 # This will fail if the payload does not match the schema the consumer
@@ -938,6 +1082,8 @@ expect(message).to eq({
 })
 ```
 
+### Test Utilities
+
 There is also a helper method that will let you test if an existing schema
 would be compatible with a new version of it. You can use this in your 
 Ruby console but it would likely not be part of your RSpec test:
@@ -947,69 +1093,6 @@ require 'deimos/test_helpers'
 # Can pass a file path, a string or a hash into this:
 Deimos::TestHelpers.schemas_compatible?(schema1, schema2)
 ```
-### Test Helpers
-
-There are helper methods available to configure Deimos for different types of testing scenarios. 
-Currently there are helpers defined for unit tests and for testing Kafka related code. You can use it as follows:
-
-```ruby
-# The following can be added to a rpsec file so that each unit 
-# test can have the same settings every time it is run
-around(:each) do |example|
-  Deimos::TestHelpers.unit_test!
-  example.run
-  Deimos.config.reset!
-end
-
-# Similarly you can use the Kafka test helper
-around(:each) do |example|
-  Deimos::TestHelpers.kafka_test!
-  example.run
-  Deimos.config.reset!
-end
-
-# Kakfa test helper using schema registry
-around(:each) do |example|
-  Deimos::TestHelpers.full_integration_test!
-  example.run
-  Deimos.config.reset!
-end
-```
-
-With the help of these helper methods, rspec examples can be written without having to tinker with Deimos settings.
-This also prevents Deimos setting changes from leaking in to other examples.
-
-This does not take away the ability to configure Deimos manually in individual examples. Deimos can still be configured like so:
-```ruby
-    it 'should not fail this random test' do
-      
-      Deimos.configure do |config|
-        config.consumers.fatal_error = proc { true }
-        config.consumers.reraise_errors = false
-      end
-      ...
-      expect(some_object).to be_truthy
-      ...
-      Deimos.config.reset!
-    end
-```
-If you are using one of the test helpers in an `around(:each)` block and want to override few settings for one example, 
-you can do it like in the example shown above. These settings would only apply to that specific example and the Deimos conifg should
-reset once the example has finished running.
-
-
-### Integration Test Helpers
-
-When running integration tests, you'll want to override the default test helper settings:
-
-```ruby
-config.before(:each, :my_integration_metadata) do
-  Deimos.configure do
-    producers.backend :kafka
-    schema.backend :avro_schema_registry
-  end
-end
-```
 
 You can use the `InlineConsumer` class to help with integration testing,
 with a full external Kafka running.

data/deimos-ruby.gemspec

@@ -39,7 +39,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency('rspec', '~> 3')
   spec.add_development_dependency('rspec_junit_formatter', '~>0.3')
   spec.add_development_dependency('rspec-rails', '~> 4')
-  spec.add_development_dependency('rubocop', '0.88.0')
+  spec.add_development_dependency('rubocop', '0.89.0')
   spec.add_development_dependency('rubocop-rspec', '1.42.0')
   spec.add_development_dependency('sqlite3', '~> 1.3')
 end

data/docs/CONFIGURATION.md

@@ -44,6 +44,7 @@ topic|nil|Topic to produce to.
 schema|nil|Name of the schema to use to encode data before producing.
 namespace|nil|Namespace of the schema to use when finding it locally.
 key_config|nil|Configuration hash for message keys. See [Kafka Message Keys](../README.md#installation)
+use_schema_classes|nil|Set to true or false to enable or disable using the producers schema classes. See [Generated Schema Classes](../README.md#generated-schema-classes)
 
 ## Defining Consumers
 
@@ -81,6 +82,7 @@ namespace|nil|Namespace of the schema to use when finding it locally.
 key_config|nil|Configuration hash for message keys. See [Kafka Message Keys](../README.md#installation)
 disabled|false|Set to true to skip starting an actual listener for this consumer on startup.
 group_id|nil|ID of the consumer group.
+use_schema_classes|nil|Set to true or false to enable or disable using the consumers schema classes. See [Generated Schema Classes](../README.md#generated-schema-classes)
 max_concurrency|1|Number of threads created for this listener. Each thread will behave as an independent consumer. They don't share any state.
 start_from_beginning|true|Once the consumer group has checkpointed its progress in the topic's partitions, the consumers will always start from the checkpointed offsets, regardless of config. As such, this setting only applies when the consumer initially starts consuming from a topic
 max_bytes_per_partition|512.kilobytes|Maximum amount of data fetched from a single partition at a time.
@@ -176,6 +178,8 @@ Config name|Default|Description
 schema.backend|`:mock`|Backend representing the schema encoder/decoder. You can see a full list [here](../lib/deimos/schema_backends).
 schema.registry_url|`http://localhost:8081`|URL of the Confluent schema registry.
 schema.path|nil|Local path to find your schemas.
+schema.use_schema_classes|false|Set this to true to use generated schema classes in your application.
+schema.generated_schema_path|`app/lib/schema_classes`|Local path to generated schema classes.
 
 ## Database Producer Configuration
 

data/lib/deimos/active_record_consume/batch_consumption.rb

@@ -15,7 +15,7 @@ module Deimos
       # If two messages in a batch have the same key, we cannot process them
       # in the same operation as they would interfere with each other. Thus
       # they are split
-      # @param payloads [Array<Hash>] Decoded payloads.
+      # @param payloads [Array<Hash|Deimos::SchemaClass::Record>] Decoded payloads
       # @param metadata [Hash] Information about batch, including keys.
       def consume_batch(payloads, metadata)
         messages = payloads.

data/lib/deimos/active_record_consume/message_consumption.rb

@@ -9,7 +9,7 @@ module Deimos
       # Default is to use the primary key column and the value of the first
       # field in the key.
       # @param klass [Class < ActiveRecord::Base]
-      # @param _payload [Hash]
+      # @param _payload [Hash|Deimos::SchemaClass::Record]
       # @param key [Object]
       # @return [ActiveRecord::Base]
       def fetch_record(klass, _payload, key)
@@ -18,13 +18,14 @@ module Deimos
 
       # Assign a key to a new record.
       # @param record [ActiveRecord::Base]
-      # @param _payload [Hash]
+      # @param _payload [Hash|Deimos::SchemaClass::Record]
       # @param key [Object]
       def assign_key(record, _payload, key)
         record[record.class.primary_key] = key
       end
 
-      # :nodoc:
+      # @param payload [Hash|Deimos::SchemaClass::Record] Decoded payloads
+      # @param metadata [Hash] Information about batch, including keys.
       def consume(payload, metadata)
         unless self.process_message?(payload)
           Deimos.config.logger.debug(

data/lib/deimos/active_record_consumer.rb

@@ -50,14 +50,14 @@ module Deimos
 
     # Override this method (with `super`) if you want to add/change the default
     # attributes set to the new/existing record.
-    # @param payload [Hash]
+    # @param payload [Hash|Deimos::SchemaClass::Record]
     # @param _key [String]
     def record_attributes(payload, _key=nil)
       @converter.convert(payload)
     end
 
     # Override this message to conditionally save records
-    # @param payload [Hash] The kafka message as a hash
+    # @param _payload [Hash|Deimos::SchemaClass::Record] The kafka message
     # @return [Boolean] if true, record is created/update.
     #   If false, record processing is skipped but message offset is still committed.
     def process_message?(_payload)

data/lib/deimos/active_record_producer.rb

@@ -58,6 +58,9 @@ module Deimos
         payload.delete_if do |k, _|
           k.to_sym != :payload_key && !fields.map(&:name).include?(k)
         end
+        return payload unless Utils::SchemaClass.use?(config.to_h)
+
+        Utils::SchemaClass.instance(payload, config[:schema])
       end
 
       # Query to use when polling the database with the DbPoller. Add

data/lib/deimos/config/configuration.rb

@@ -18,6 +18,9 @@ module Deimos
   # :nodoc:
   after_configure do
     Phobos.configure(self.config.phobos_config)
+    if self.config.schema.use_schema_classes
+      load_generated_schema_classes
+    end
     self.config.producer_objects.each do |producer|
       configure_producer_or_consumer(producer)
     end
@@ -28,6 +31,17 @@ module Deimos
     validate_db_backend if self.config.producers.backend == :db
   end
 
+  # Loads generated classes
+  def self.load_generated_schema_classes
+    if Deimos.config.schema.generated_class_path.nil?
+      raise 'Cannot use schema classes without schema.generated_class_path. Please provide a directory.'
+    end
+
+    Dir["./#{Deimos.config.schema.generated_class_path}/**/*.rb"].sort.each { |f| require f }
+  rescue LoadError
+    raise 'Cannot load schema classes. Please regenerate classes with rake deimos:generate_schema_models.'
+  end
+
   # Ensure everything is set up correctly for the DB backend.
   def self.validate_db_backend
     begin
@@ -68,6 +82,7 @@ module Deimos
       schema(kafka_config.schema) if kafka_config.schema.present?
       namespace(kafka_config.namespace) if kafka_config.namespace.present?
       key_config(**kafka_config.key_config) if kafka_config.key_config.present?
+      schema_class_config(kafka_config.use_schema_classes) if kafka_config.use_schema_classes.present?
     end
   end
 
@@ -260,6 +275,14 @@ module Deimos
       # Local path to look for schemas in.
       # @return [String]
       setting :path
+
+      # Local path for schema classes to be generated in.
+      # @return [String]
+      setting :generated_class_path, 'app/lib/schema_classes'
+
+      # Set to true to use the generated schema classes in your application
+      # @return [Boolean]
+      setting :use_schema_classes, false
     end
 
     # The configured metrics provider.
@@ -301,6 +324,9 @@ module Deimos
       # Key configuration (see docs).
       # @return [Hash]
       setting :key_config
+      # Configure the usage of generated schema classes for this producer
+      # @return [Boolean]
+      setting :use_schema_classes
     end
 
     setting_object :consumer do
@@ -323,6 +349,9 @@ module Deimos
       # listener.
       # @return [Boolean]
       setting :disabled, false
+      # Configure the usage of generated schema classes for this consumer
+      # @return [Boolean]
+      setting :use_schema_classes
 
       # These are the phobos "listener" configs. See CONFIGURATION.md for more
       # info.

data/lib/deimos/consume/batch_consumption.rb

@@ -21,11 +21,11 @@ module Deimos
           metadata[:first_offset] = batch.first&.offset
 
           payloads = batch.map do |message|
-            message.payload ? self.class.decoder.decode(message.payload) : nil
+            decode_message(message.payload)
           end
           _received_batch(payloads, metadata)
           _with_span do
-            yield payloads, metadata
+            yield(payloads, metadata)
           end
         end
         _handle_batch_success(benchmark.real, payloads, metadata)

data/lib/deimos/consume/message_consumption.rb

@@ -15,9 +15,9 @@ module Deimos
         benchmark = Benchmark.measure do
           _with_span do
             new_metadata[:key] = decode_key(metadata[:key]) if self.class.config[:key_configured]
-            decoded_payload = payload ? self.class.decoder.decode(payload) : nil
+            decoded_payload = decode_message(payload)
             _received_message(decoded_payload, new_metadata)
-            yield decoded_payload, new_metadata
+            yield(decoded_payload, new_metadata)
           end
         end
         _handle_success(benchmark.real, decoded_payload, new_metadata)

data/lib/deimos/consumer.rb

@@ -51,6 +51,16 @@ module Deimos
       end
     end
 
+    # Helper method to decode an encoded message.
+    # @param payload [Object]
+    # @return [Object] the decoded message.
+    def decode_message(payload)
+      decoded_payload = payload.nil? ? nil : self.class.decoder.decode(payload)
+      return decoded_payload unless Utils::SchemaClass.use?(self.class.config.to_h)
+
+      Utils::SchemaClass.instance(decoded_payload, self.class.config[:schema])
+    end
+
   private
 
     def _with_span

data/lib/deimos/producer.rb

@@ -57,6 +57,7 @@ module Deimos
     MAX_BATCH_SIZE = 500
 
     class << self
+
       # @return [Hash]
       def config
         @config ||= {
@@ -86,14 +87,14 @@ module Deimos
       end
 
       # Publish the payload to the topic.
-      # @param payload [Hash] with an optional payload_key hash key.
+      # @param payload [Hash|SchemaClass::Record] with an optional payload_key hash key.
       # @param topic [String] if specifying the topic
       def publish(payload, topic: self.topic)
         publish_list([payload], topic: topic)
       end
 
       # Publish a list of messages.
-      # @param payloads [Hash|Array<Hash>] with optional payload_key hash key.
+      # @param payloads [Array<Hash|SchemaClass::Record>] with optional payload_key hash key.
       # @param sync [Boolean] if given, override the default setting of
       # whether to publish synchronously.
       # @param force_send [Boolean] if true, ignore the configured backend
@@ -113,7 +114,7 @@ module Deimos
           topic: topic,
           payloads: payloads
         ) do
-          messages = Array(payloads).map { |p| Deimos::Message.new(p, self) }
+          messages = Array(payloads).map { |p| Deimos::Message.new(p.to_h, self) }
           messages.each { |m| _process_message(m, topic) }
           messages.in_groups_of(MAX_BATCH_SIZE, false) do |batch|
             self.produce_batch(backend_class, batch)