deimos-ruby 1.11.2 → 1.12.3

data/lib/deimos/schema_class/base.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Deimos
+  module SchemaClass
+    # Base Class for Schema Classes generated from Avro.
+    class Base
+
+      # :nodoc:
+      def initialize(*_args)
+      end
+
+      # Converts the object to a string that represents a JSON object
+      # @return [String] a JSON string
+      def to_json(*_args)
+        to_h.to_json
+      end
+
+      # Converts the object to a hash which can be used for debugging or comparing objects.
+      # @return [Hash] a hash representation of the payload
+      def as_json(_opts={})
+        JSON.parse(to_json)
+      end
+
+      # Converts the object attributes to a hash which can be used for Kafka
+      # @return [Hash] the payload as a hash.
+      def to_h
+        raise NotImplementedError
+      end
+
+      # :nodoc:
+      def ==(other)
+        comparison = other
+        if other.class == self.class
+          comparison = other.as_json
+        end
+
+        comparison == self.as_json
+      end
+
+      # :nodoc:
+      def to_s
+        klass = self.class
+        "#{klass}(#{self.as_json.symbolize_keys.to_s[1..-2]})"
+      end
+
+      # Initializes this class from a given value
+      # @param value [Object]
+      def self.initialize_from_value(value)
+        raise NotImplementedError
+      end
+
+    protected
+
+      # :nodoc:
+      def hash
+        as_json.hash
+      end
+    end
+  end
+end

data/lib/deimos/schema_class/enum.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+require 'json'
+
+module Deimos
+  module SchemaClass
+    # Base Class for Enum Classes generated from Avro.
+    class Enum < Base
+      # Returns all the valid symbols for this enum.
+      # @return [Array<String>]
+      def symbols
+        raise NotImplementedError
+      end
+
+      # :nodoc:
+      def self.initialize_from_value(value)
+        return nil if value.nil?
+
+        value.is_a?(self) ? value : self.new(value)
+      end
+    end
+  end
+end

data/lib/deimos/schema_class/record.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+require 'json'
+
+module Deimos
+  module SchemaClass
+    # Base Class of Record Classes generated from Avro.
+    class Record < Base
+
+      # Converts the object to a hash which can be used for debugging or comparing objects.
+      # @return [Hash] a hash representation of the payload
+      def as_json(_opts={})
+        super.except('payload_key')
+      end
+
+      # Element access method as if this Object were a hash
+      # @param key[String||Symbol]
+      # @return [Object] The value of the attribute if exists, nil otherwise
+      def [](key)
+        self.try(key.to_sym)
+      end
+
+      # :nodoc
+      def with_indifferent_access
+        self
+      end
+
+      # Returns the schema name of the inheriting class.
+      # @return [String]
+      def schema
+        raise NotImplementedError
+      end
+
+      # Returns the namespace for the schema of the inheriting class.
+      # @return [String]
+      def namespace
+        raise NotImplementedError
+      end
+
+      # Returns the full schema name of the inheriting class.
+      # @return [String]
+      def full_schema
+        "#{namespace}.#{schema}"
+      end
+
+      # Returns the schema validator from the schema backend
+      # @return [Deimos::SchemaBackends::Base]
+      def validator
+        Deimos.schema_backend(schema: schema, namespace: namespace)
+      end
+
+      # @return [Array<String>] an array of fields names in the schema.
+      def schema_fields
+        validator.schema_fields.map(&:name)
+      end
+
+      # :nodoc:
+      def self.initialize_from_value(value)
+        return nil if value.nil?
+
+        value.is_a?(self) ? value : self.new(**value.symbolize_keys)
+      end
+    end
+  end
+end

data/lib/deimos/shared_config.rb

@@ -58,6 +58,11 @@ module Deimos
         config[:key_field] = field&.to_s
         config[:key_schema] = schema
       end
+
+      # @param enabled [Boolean]
+      def schema_class_config(use_schema_classes)
+        config[:use_schema_classes] = use_schema_classes
+      end
     end
   end
 end

data/lib/deimos/test_helpers.rb

@@ -50,7 +50,6 @@ module Deimos
     included do
 
       RSpec.configure do |config|
-
         config.prepend_before(:each) do
           client = double('client').as_null_object
           allow(client).to receive(:time) do |*_args, &block|
@@ -210,9 +209,11 @@ module Deimos
                        'value' => payload)
 
       unless skip_expectation
-        expectation = expect(handler).to receive(:consume).
-          with(payload, anything, &block)
-        expectation.and_call_original if call_original
+        _handler_expectation(:consume,
+                             payload,
+                             handler,
+                             call_original,
+                             &block)
       end
       Phobos::Actions::ProcessMessage.new(
         listener: listener,
@@ -277,9 +278,11 @@ module Deimos
                      'partition' => 1,
                      'offset_lag' => 0)
       unless skip_expectation
-        expectation = expect(handler).to receive(:consume_batch).
-          with(payloads, anything, &block)
-        expectation.and_call_original if call_original
+        _handler_expectation(:consume_batch,
+                             payloads,
+                             handler,
+                             call_original,
+                             &block)
       end
       action = Phobos::Actions::ProcessBatchInline.new(
         listener: listener,
@@ -356,5 +359,38 @@ module Deimos
 
       handler.handler.constantize
     end
+
+    # Test that a given handler will execute a `method` on an `input` correctly,
+    # If a block is given, that block will be executed when `method` is called.
+    # Otherwise it will just confirm that `method` is called at all.
+    # @param method [Symbol]
+    # @param input [Object]
+    # @param handler [Deimos::Consumer]
+    # @param call_original [Boolean]
+    def _handler_expectation(method,
+                             input,
+                             handler,
+                             call_original,
+                             &block)
+      schema_class = handler.class.config[:schema]
+      expected = input.dup
+
+      config = handler.class.config
+      use_schema_classes = config[:use_schema_classes]
+      use_schema_classes = use_schema_classes.present? ? use_schema_classes : Deimos.config.schema.use_schema_classes
+
+      if use_schema_classes && schema_class.present?
+        expected = if input.is_a?(Array)
+                     input.map do |payload|
+                       Utils::SchemaClass.instance(payload, schema_class)
+                     end
+                   else
+                     Utils::SchemaClass.instance(input, schema_class)
+                   end
+      end
+
+      expectation = expect(handler).to receive(method).with(expected, anything, &block)
+      expectation.and_call_original if call_original
+    end
   end
 end

data/lib/deimos/utils/schema_class.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Deimos
+  module Utils
+    # Class used by SchemaClassGenerator and Consumer/Producer interfaces
+    module SchemaClass
+      class << self
+        # Converts a raw payload into an instance of the Schema Class
+        # @param payload [Hash]
+        # @param schema [String]
+        # @return [Deimos::SchemaClass::Record]
+        def instance(payload, schema)
+          klass = "Schemas::#{schema.underscore.camelize}".safe_constantize
+          return payload if klass.nil? || payload.nil?
+
+          klass.new(**payload.symbolize_keys)
+        end
+
+        # @param config [Hash] Producer or Consumer config
+        # @return [Boolean]
+        def use?(config)
+          use_schema_classes = config[:use_schema_classes]
+          use_schema_classes.present? ? use_schema_classes : Deimos.config.schema.use_schema_classes
+        end
+
+      end
+    end
+  end
+end

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.11.2'
+  VERSION = '1.12.3'
 end

data/lib/deimos.rb

@@ -19,6 +19,9 @@ require 'deimos/backends/kafka_async'
 require 'deimos/backends/test'
 
 require 'deimos/schema_backends/base'
+require 'deimos/utils/schema_class'
+require 'deimos/schema_class/enum'
+require 'deimos/schema_class/record'
 
 require 'deimos/monkey_patches/phobos_producer'
 require 'deimos/monkey_patches/phobos_cli'
@@ -59,6 +62,24 @@ module Deimos
       schema_backend_class.new(schema: schema, namespace: namespace)
     end
 
+    # @param schema [String]
+    # @param namespace [String]
+    # @param payload [Hash]
+    # @param subject [String]
+    # @return [String]
+    def encode(schema:, namespace:, payload:, subject: nil)
+      self.schema_backend(schema: schema, namespace: namespace).
+        encode(payload, topic: subject || "#{namespace}.#{schema}" )
+    end
+
+    # @param schema [String]
+    # @param namespace [String]
+    # @param payload [String]
+    # @return [Hash,nil]
+    def decode(schema:, namespace:, payload:)
+      self.schema_backend(schema: schema, namespace: namespace).decode(payload)
+    end
+
     # Start the DB producers to send Kafka messages.
     # @param thread_count [Integer] the number of threads to start.
     def start_db_backend!(thread_count: 1)

data/lib/generators/deimos/schema_class/templates/schema_class.rb.tt

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+# This file is autogenerated by Deimos, Do NOT modify
+module Schemas
+<% if @sub_schema_templates.present? && @sub_schema_templates.any? -%>
+  ### Secondary Schema Classes ###
+<% end -%>
+<%- @sub_schema_templates.each do |schema_template| -%>
+  <%=- schema_template %>
+
+<%- end -%>
+  ### Primary Schema Class ###
+  <%=- @main_class_definition -%>
+
+end

data/lib/generators/deimos/schema_class/templates/schema_enum.rb.tt

@@ -0,0 +1,21 @@
+  # Autogenerated Schema for Enum at <%= @current_schema.namespace %>.<%= @current_schema.name %>
+  class <%= Deimos::SchemaBackends::AvroBase.schema_classname(@current_schema) %> < Deimos::SchemaClass::Enum
+    # @return ['<%= @current_schema.symbols.join("', '") %>']
+    attr_accessor :<%= @current_schema.name.underscore %>
+
+    # :nodoc:
+    def initialize(<%= @current_schema.name.underscore %>)
+      super
+      self.<%= @current_schema.name.underscore %> = <%= @current_schema.name.underscore %>
+    end
+
+    # @override
+    def symbols
+      %w(<%= @current_schema.symbols.join(' ') %>)
+    end
+
+    # @override
+    def to_h
+      @<%= @current_schema.name.underscore %>
+    end
+  end

data/lib/generators/deimos/schema_class/templates/schema_record.rb.tt

@@ -0,0 +1,65 @@
+  # Autogenerated Schema for Record at <%= @current_schema.namespace %>.<%= @current_schema.name %>
+  class <%= Deimos::SchemaBackends::AvroBase.schema_classname(@current_schema) %> < Deimos::SchemaClass::Record
+<%- if @field_assignments.select{ |h| h[:is_schema_class] }.any? -%>
+    ### Attribute Readers ###
+  <%- @field_assignments.select{ |h| h[:is_schema_class] }.each do |method_definition| -%>
+    # @return [<%= method_definition[:deimos_type] %>]
+    attr_reader :<%= method_definition[:field].name %>
+  <%- end -%>
+
+<% end -%>
+<%- if @field_assignments.select{ |h| !h[:is_schema_class] }.any? -%>
+    ### Attribute Accessors ###
+  <%- @field_assignments.select{ |h| !h[:is_schema_class] }.each do |method_definition| -%>
+    # @param <%= method_definition[:method_argument] %> [<%= method_definition[:deimos_type] %>]
+    attr_accessor :<%= method_definition[:field].name %>
+  <%- end -%>
+
+<% end -%>
+<%- if @field_assignments.select{ |h| h[:is_schema_class] }.any? -%>
+    ### Attribute Writers ###
+  <%- @field_assignments.select{ |h| h[:is_schema_class] }.each do |method_definition| -%>
+    # @param <%= method_definition[:method_argument] %> [<%= method_definition[:deimos_type] %>]
+    def <%= method_definition[:field].name %>=(<%= method_definition[:method_argument] %>)
+    <%- if method_definition[:field_type] == :array -%>
+      @<%= method_definition[:field].name %> = values.map do |value|
+        <%= method_definition[:field_initialization] %>
+      end
+    <%- elsif method_definition[:field_type] == :map -%>
+      @<%= method_definition[:field].name %> = values.transform_values do |value|
+        <%= method_definition[:field_initialization] %>
+      end
+    <%- else -%>
+      @<%= method_definition[:field].name %> = <%= method_definition[:field_initialization] %>
+    <%- end -%>
+    end
+
+  <%- end -%>
+<% end -%>
+    # @override
+    <%= @initialization_definition %>
+      super
+<%- @fields.each do |field| -%>
+      self.<%= field.name %> = <%= field.name %>
+<% end -%>
+    end
+
+    # @override
+    def schema
+      '<%= @current_schema.name %>'
+    end
+
+    # @override
+    def namespace
+      '<%= @current_schema.namespace %>'
+    end
+
+    # @override
+    def to_h
+      {
+<%- @fields.each do |field| -%>
+        <%= field_to_h(field) %>
+<% end -%>
+      }
+    end
+  end

data/lib/generators/deimos/schema_class_generator.rb

@@ -0,0 +1,247 @@
+# frozen_string_literal: true
+
+require 'rails/generators'
+require 'deimos'
+require 'deimos/schema_backends/avro_base'
+require 'deimos/config/configuration'
+
+# Generates new schema classes.
+module Deimos
+  module Generators
+    # Generator for Schema Classes used for the IDE and consumer/producer interfaces
+    class SchemaClassGenerator < Rails::Generators::Base
+
+      SPECIAL_TYPES = %i(record enum).freeze
+      INITIALIZE_WHITESPACE = "\n#{' ' * 19}"
+      IGNORE_DEFAULTS = %w(message_id timestamp).freeze
+      SCHEMA_CLASS_FILE = 'schema_class.rb'
+      SCHEMA_RECORD_PATH = File.expand_path('schema_class/templates/schema_record.rb.tt', __dir__).freeze
+      SCHEMA_ENUM_PATH = File.expand_path('schema_class/templates/schema_enum.rb.tt', __dir__).freeze
+
+      source_root File.expand_path('schema_class/templates', __dir__)
+
+      no_commands do
+        # Retrieve the fields from this Avro Schema
+        # @param schema [Avro::Schema::NamedSchema]
+        # @return [Array<SchemaField>]
+        def fields(schema)
+          schema.fields.map do |field|
+            Deimos::SchemaField.new(field.name, field.type, [], field.default)
+          end
+        end
+
+        # Converts Deimos::SchemaField's to String form for generated YARD docs
+        # @param schema_field [Deimos::SchemaField]
+        # @return [String] A string representation of the Type of this SchemaField
+        def deimos_field_type(schema_field)
+          _field_type(schema_field.type)
+        end
+
+        # Generate a Schema Model Class and all of its Nested Records from a
+        # Deimos Consumer or Producer Configuration object
+        # @param schema_name [String]
+        # @param namespace [String]
+        # @param key_schema_name [String]
+        def generate_classes(schema_name, namespace, key_schema_name)
+          schema_base = Deimos.schema_backend(schema: schema_name, namespace: namespace)
+          schema_base.load_schema
+          if key_schema_name.present?
+            key_schema_base = Deimos.schema_backend(schema: key_schema_name, namespace: namespace)
+            key_schema_base.load_schema
+            generate_class_from_schema_base(key_schema_base)
+          end
+          generate_class_from_schema_base(schema_base, key_schema_base: key_schema_base)
+        end
+
+        # @param schema_base [Deimos::SchemaBackends::Base]
+        # @param key_schema_base[Avro::Schema::NamedSchema]
+        def generate_class_from_schema_base(schema_base, key_schema_base: nil)
+          schemas = schema_base.schema_store.schemas.values
+          sub_schemas = schemas.reject { |s| s.name == schema_base.schema }
+          @sub_schema_templates = sub_schemas.map do |schema|
+            _generate_class_template_from_schema(schema)
+          end
+
+          main_schema = schemas.find { |s| s.name == schema_base.schema }
+          class_template = _generate_class_template_from_schema(main_schema, key_schema_base)
+          @main_class_definition = class_template
+
+          file_prefix = main_schema.name.underscore
+          namespace_path = main_schema.namespace.tr('.', '/')
+          filename = "#{Deimos.config.schema.generated_class_path}/#{namespace_path}/#{file_prefix}.rb"
+          template(SCHEMA_CLASS_FILE, filename, force: true)
+        end
+
+        # Format a given field into its appropriate to_h representation.
+        # @param field[Deimos::SchemaField]
+        # @return [String]
+        def field_to_h(field)
+          res = "'#{field.name}' => @#{field.name}"
+          field_base_type = _schema_base_class(field.type).type_sym
+
+          if %i(record enum).include?(field_base_type)
+            res += case field.type.type_sym
+                   when :array
+                     '.map { |v| v&.to_h }'
+                   when :map
+                     '.transform_values { |v| v&.to_h }'
+                   else
+                     '&.to_h'
+                   end
+          end
+
+          res + (field.name == @fields.last.name ? '' : ',')
+        end
+
+      end
+
+      desc 'Generate a class based on configured consumer and producers.'
+      # :nodoc:
+      def generate
+        _validate
+        Rails.logger.info("Generating schemas from Deimos.config to #{Deimos.config.schema.generated_class_path}")
+        Deimos.config.producer_objects.each do |config|
+          schema_name = config.schema
+          namespace = config.namespace || Deimos.config.producers.schema_namespace
+          key_schema_name = config.key_config[:schema]
+          generate_classes(schema_name, namespace, key_schema_name)
+        end
+
+        Deimos.config.consumer_objects.each do |config|
+          schema_name = config.schema
+          namespace = config.namespace
+          key_schema_name = config.key_config[:schema]
+          generate_classes(schema_name, namespace, key_schema_name)
+        end
+      end
+
+    private
+
+      # Determines if Schema Class Generation can be run.
+      # @raise if Schema Backend is not of a Avro-based class
+      def _validate
+        backend = Deimos.config.schema.backend.to_s
+        raise 'Schema Class Generation requires an Avro-based Schema Backend' if backend !~ /^avro/
+      end
+
+      # @param schema[Avro::Schema::NamedSchema]
+      # @param key_schema_base[Avro::Schema::NamedSchema]
+      # @return [String]
+      def _generate_class_template_from_schema(schema, key_schema_base=nil)
+        _set_instance_variables(schema, key_schema_base)
+
+        temp = schema.is_a?(Avro::Schema::RecordSchema) ? _record_class_template : _enum_class_template
+        res = ERB.new(temp, nil, '-')
+        res.result(binding)
+      end
+
+      # @param schema[Avro::Schema::NamedSchema]
+      # @param key_schema_base[Avro::Schema::NamedSchema]
+      def _set_instance_variables(schema, key_schema_base=nil)
+        schema_is_record = schema.is_a?(Avro::Schema::RecordSchema)
+        @current_schema = schema
+        return unless schema_is_record
+
+        @fields = fields(schema)
+        if key_schema_base.present?
+          key_schema_base.load_schema
+          key_schema = key_schema_base.schema_store.schemas.values.first
+          @fields << Deimos::SchemaField.new('payload_key', key_schema, [], nil)
+        end
+        @initialization_definition = _initialization_definition
+        @field_assignments = _field_assignments
+      end
+
+      # Defines the initialization method for Schema Records with one keyword argument per line
+      # @return [String] A string which defines the method signature for the initialize method
+      def _initialization_definition
+        arguments = @fields.map do |schema_field|
+          arg = "#{schema_field.name}:"
+          arg += _field_default(schema_field)
+          arg.strip
+        end
+
+        result = "def initialize(#{arguments.first}"
+        arguments[1..-1].each_with_index do |arg, _i|
+          result += ",#{INITIALIZE_WHITESPACE}#{arg}"
+        end
+        "#{result})"
+      end
+
+      # @param [SchemaField]
+      # @return [String]
+      def _field_default(field)
+        default = field.default
+        return ' nil' if default == :no_default || default.nil? || IGNORE_DEFAULTS.include?(field.name)
+
+        case field.type.type_sym
+        when :string
+          " '#{default}'"
+        when :record
+          schema_name = Deimos::SchemaBackends::AvroBase.schema_classname(field.type)
+          class_instance = Utils::SchemaClass.instance(field.default, schema_name)
+          " #{class_instance.to_h}"
+        else
+          " #{default}"
+        end
+      end
+
+      # Overrides default attr accessor methods
+      # @return [Array<String>]
+      def _field_assignments
+        result = []
+        @fields.each do |field|
+          field_type = field.type.type_sym # Record, Union, Enum, Array or Map
+          schema_base_type = _schema_base_class(field.type)
+          field_base_type = _field_type(schema_base_type)
+          method_argument = %i(array map).include?(field_type) ? 'values' : 'value'
+          is_schema_class = %i(record enum).include?(schema_base_type.type_sym)
+
+          field_initialization = method_argument
+
+          if is_schema_class
+            field_initialization = "#{field_base_type}.initialize_from_value(value)"
+          end
+
+          result << {
+            field: field,
+            field_type: field_type,
+            is_schema_class: is_schema_class,
+            method_argument: method_argument,
+            deimos_type: deimos_field_type(field),
+            field_initialization: field_initialization
+          }
+        end
+
+        result
+      end
+
+      # Converts Avro::Schema::NamedSchema's to String form for generated YARD docs.
+      # Recursively handles the typing for Arrays, Maps and Unions.
+      # @param avro_schema [Avro::Schema::NamedSchema]
+      # @return [String] A string representation of the Type of this SchemaField
+      def _field_type(avro_schema)
+        Deimos::SchemaBackends::AvroBase.field_type(avro_schema)
+      end
+
+      # Returns the base class for this schema. Decodes Arrays, Maps and Unions
+      # @param avro_schema [Avro::Schema::NamedSchema]
+      # @return [Avro::Schema::NamedSchema]
+      def _schema_base_class(avro_schema)
+        Deimos::SchemaBackends::AvroBase.schema_base_class(avro_schema)
+      end
+
+      # An ERB template for schema record classes
+      # @return [String]
+      def _record_class_template
+        File.read(SCHEMA_RECORD_PATH).strip
+      end
+
+      # An ERB template for schema enum classes
+      # @return [String]
+      def _enum_class_template
+        File.read(SCHEMA_ENUM_PATH).strip
+      end
+    end
+  end
+end

data/lib/tasks/deimos.rake

@@ -2,6 +2,8 @@
 
 require 'phobos'
 require 'phobos/cli'
+require 'generators/deimos/schema_class_generator'
+require 'optparse'
 
 namespace :deimos do
   desc 'Starts Deimos in the rails environment'
@@ -31,4 +33,10 @@ namespace :deimos do
     Deimos::Utils::DbPoller.start!
   end
 
+  desc 'Run Schema Model Generator'
+  task generate_schema_classes: :environment do
+    Rails.logger.info("Running deimos:generate_schema_classes")
+    Deimos::Generators::SchemaClassGenerator.start
+  end
+
 end