deimos-ruby 2.6.0.pre.beta1 → 2.6.0

data/lib/deimos/producer.rb

@@ -75,7 +75,7 @@ module Deimos
       end
 
       # Publish the payload to the topic.
-      # @param payload [Hash, AvroGen::SchemaClass::Record] with an optional payload_key hash key.
+      # @param payload [Hash, SchemaClass::Record] with an optional payload_key hash key.
       # @param topic [String] if specifying the topic
       # @param headers [Hash] if specifying headers
       # @return [void]
@@ -99,7 +99,7 @@ module Deimos
       end
 
       # Publish a list of messages.
-      # @param payloads [Array<Hash, AvroGen::SchemaClass::Record>] with optional payload_key hash key.
+      # @param payloads [Array<Hash, SchemaClass::Record>] with optional payload_key hash key.
       # @param sync [Boolean] if given, override the default setting of
       # whether to publish synchronously.
       # @param force_send [Boolean] if true, ignore the configured backend
@@ -112,7 +112,7 @@ module Deimos
 
         messages = Array(payloads).map do |p|
           payload = p
-          payload = payload.to_h if p.is_a?(AvroGen::SchemaClass::Record)
+          payload = payload.to_h if p.is_a?(SchemaClass::Record)
           m = {
             payload: payload,
             headers: headers,
@@ -121,10 +121,10 @@ module Deimos
           }
           if payload.is_a?(Hash) && payload.key?(:key) && payload.key?(:message)
             m[:key] = payload[:key]
-            m[:key] = m[:key].to_h if m[:key].nil? || m[:key].is_a?(AvroGen::SchemaClass::Record)
+            m[:key] = m[:key].to_h if m[:key].nil? || m[:key].is_a?(SchemaClass::Record)
             m[:payload] = payload[:message]
             m[:payload] = m[:payload].to_h if m[:payload].nil? ||
-                                              m[:payload].is_a?(AvroGen::SchemaClass::Record)
+                                              m[:payload].is_a?(SchemaClass::Record)
           end
           m
         end

data/lib/deimos/schema_backends/avro_base.rb

@@ -2,7 +2,6 @@
 
 require_relative 'base'
 require 'schema_registry_client'
-require 'avro_gen/avro_parser'
 require_relative 'avro_schema_coercer'
 
 module Deimos
@@ -107,25 +106,55 @@ module Deimos
 
       # @param schema [Avro::Schema::NamedSchema] A named schema
       # @return [String]
-      # @deprecated Use AvroGen::AvroParser.schema_classname instead.
       def self.schema_classname(schema)
-        AvroGen::AvroParser.schema_classname(schema)
+        schema.name.underscore.camelize.singularize
       end
 
       # Converts Avro::Schema::NamedSchema's to String form for generated YARD docs.
+      # Recursively handles the typing for Arrays, Maps and Unions.
       # @param avro_schema [Avro::Schema::NamedSchema]
       # @return [String] A string representation of the Type of this SchemaField
-      # @deprecated Use AvroGen::AvroParser.field_type instead.
       def self.field_type(avro_schema)
-        AvroGen::AvroParser.field_type(avro_schema)
+        case avro_schema.type_sym
+        when :string, :boolean
+          avro_schema.type_sym.to_s.titleize
+        when :int, :long
+          'Integer'
+        when :float, :double
+          'Float'
+        when :record, :enum
+          schema_classname(avro_schema)
+        when :array
+          arr_t = field_type(Deimos::SchemaField.new('n/a', avro_schema.items).type)
+          "Array<#{arr_t}>"
+        when :map
+          map_t = field_type(Deimos::SchemaField.new('n/a', avro_schema.values).type)
+          "Hash<String, #{map_t}>"
+        when :union
+          types = avro_schema.schemas.map do |t|
+            field_type(Deimos::SchemaField.new('n/a', t).type)
+          end
+          types.join(', ')
+        when :null
+          'nil'
+        end
       end
 
       # Returns the base type of this schema. Decodes Arrays, Maps and Unions
       # @param schema [Avro::Schema::NamedSchema]
       # @return [Avro::Schema::NamedSchema]
-      # @deprecated Use AvroGen::AvroParser.schema_base_class instead.
       def self.schema_base_class(schema)
-        AvroGen::AvroParser.schema_base_class(schema)
+        case schema.type_sym
+        when :array
+          schema_base_class(schema.items)
+        when :map
+          schema_base_class(schema.values)
+        when :union
+          schema.schemas.map(&method(:schema_base_class)).
+            reject { |s| s.type_sym == :null }.first
+        else
+          schema
+        end
       end
 
       def generate_key_schema(field_name)

data/lib/deimos/schema_class/base.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Deimos
+  module SchemaClass
+    # Base Class for Schema Classes generated from Avro.
+    class Base
+
+      # @param _args [Array<Object>]
+      def initialize(*_args)
+      end
+
+      # Converts the object to a hash which can be used for debugging or comparing objects.
+      # @param _opts [Hash]
+      # @return [Hash] a hash representation of the payload
+      def as_json(_opts={})
+        raise MissingImplementationError
+      end
+
+      # @param key [String,Symbol]
+      # @param val [Object]
+      # @return [void]
+      def []=(key, val)
+        self.send("#{key}=", val)
+      end
+
+      # @param other [SchemaClass::Base]
+      # @return [Boolean]
+      def ==(other)
+        comparison = other
+        if other.class == self.class
+          comparison = other.as_json
+        end
+
+        comparison == self.as_json
+      end
+
+      alias_method :eql?, :==
+
+      # @return [String]
+      def inspect
+        klass = self.class
+        "#{klass}(#{self.as_json})"
+      end
+
+      # Initializes this class from a given value
+      # @param value [Object]
+      # @return [SchemaClass::Base]
+      def self.initialize_from_value(_value)
+        raise MissingImplementationError
+      end
+
+    protected
+
+      # @return [Integer]
+      def hash
+        as_json.hash
+      end
+    end
+  end
+end

data/lib/deimos/schema_class/enum.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+require 'json'
+
+module Deimos
+  module SchemaClass
+    # Base Class for Enum Classes generated from Avro.
+    class Enum < Base
+
+      # @return [String]
+      attr_accessor :value
+
+      # @param other [Deimos::SchemaClass::Enum]
+      # @return [Boolean]
+      def ==(other)
+        other.is_a?(self.class) ? other.value == @value : other == @value
+      end
+
+      # @return [String]
+      def to_s
+        @value.to_s
+      end
+
+      # @param value [String]
+      def initialize(value)
+        @value = value
+      end
+
+      # Returns all the valid symbols for this enum.
+      # @return [Array<String>]
+      def symbols
+        raise MissingImplementationError
+      end
+
+      # @return [String]
+      def as_json(_opts={})
+        @value
+      end
+
+      # @return [SchemaClass::Enum]
+      def self.initialize_from_value(value, from_message: false)
+        return nil if value.nil?
+
+        value.is_a?(self) ? value : self.new(value)
+      end
+    end
+  end
+end

data/lib/deimos/schema_class/record.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+require 'json'
+
+module Deimos
+  module SchemaClass
+    # Base Class of Record Classes generated from Avro.
+    class Record < Base
+
+      attr_accessor :tombstone_key, :_from_message
+
+      # Converts the object attributes to a hash which can be used for Kafka
+      # @return [Hash] the payload as a hash.
+      def to_h
+        if self.tombstone_key
+          { payload_key: self.tombstone_key&.as_json }
+        else
+          self.as_json
+        end
+      end
+
+      # Merge a hash or an identical schema object with this one and return a new object.
+      # @param other_hash [Hash,SchemaClass::Base]
+      # @return [SchemaClass::Base]
+      def merge(other_hash)
+        obj = self.class.new(**self.to_h.symbolize_keys)
+        other_hash.to_h.each do |k, v|
+          obj.send("#{k}=", v)
+        end
+        obj
+      end
+
+      # Element access method as if this Object were a hash
+      # @param key[String,Symbol]
+      # @return [Object] The value of the attribute if exists, nil otherwise
+      def [](key)
+        self.try(key.to_sym)
+      end
+
+      # @return [SchemaClass::Record]
+      def with_indifferent_access
+        self
+      end
+
+      # Returns the schema name of the inheriting class.
+      # @return [String]
+      def schema
+        raise MissingImplementationError
+      end
+
+      # Returns the namespace for the schema of the inheriting class.
+      # @return [String]
+      def namespace
+        raise MissingImplementationError
+      end
+
+      # Returns the full schema name of the inheriting class.
+      # @return [String]
+      def full_schema
+        "#{namespace}.#{schema}"
+      end
+
+      # Returns the schema validator from the schema backend
+      # @return [Deimos::SchemaBackends::Base]
+      def validator
+        Deimos.schema_backend(schema: schema, namespace: namespace)
+      end
+
+      # @return [Array<String>] an array of fields names in the schema.
+      def schema_fields
+        validator.schema_fields.map(&:name)
+      end
+
+      # Used internally within Deimos so that we don't crash on unknown fields that come from
+      # a backwards compatible schema.
+      # @param kwargs [Hash] the attributes to set on the new object.
+      # @return [SchemaClass::Record]
+      def self.new_from_message(**kwargs)
+        record = self.new
+        attrs = kwargs.select { |k, _v| record.respond_to?("#{k}=") }
+        self.new(_from_message: true, **attrs)
+      end
+
+      # @return [SchemaClass::Record]
+      # @param from_message [Boolean] whether it's being initialized from a real Avro message.
+      def self.initialize_from_value(value, from_message: false)
+        return nil if value.nil?
+
+        return value if value.is_a?(self)
+
+        if from_message
+          self.new_from_message(**value&.symbolize_keys || {})
+        else
+          self.new(**value&.symbolize_keys || {})
+        end
+      end
+    end
+  end
+end

data/lib/deimos/transcoder.rb

@@ -58,9 +58,9 @@ module Deimos
 
       schema_key = decoded_key.is_a?(Hash) ? decoded_key : { self.key_field => decoded_key }
 
-      AvroGen::SchemaClass.instance(schema_key,
-                                    "#{@schema}_key",
-                                    @namespace)
+      Utils::SchemaClass.instance(schema_key,
+                                  "#{@schema}_key",
+                                  @namespace)
     end
 
     # @param payload [String]
@@ -71,9 +71,9 @@ module Deimos
       decoded_payload = self.backend.decode(payload)
       return decoded_payload if !@use_schema_classes || !decoded_payload.is_a?(Hash)
 
-      AvroGen::SchemaClass.instance(decoded_payload,
-                                    @schema,
-                                    @namespace)
+      Utils::SchemaClass.instance(decoded_payload,
+                                  @schema,
+                                  @namespace)
     end
 
     # @param payload [Object]

data/lib/deimos/utils/deadlock_retry.rb

@@ -21,6 +21,15 @@ module Deimos
           'deadlock detected'
         ].freeze
 
+        # Whether the given exception is a deadlock or lock wait timeout, i.e. transient
+        # contention on the database rather than a problem with the data being written.
+        # @param error [Exception]
+        # @return [Boolean]
+        def deadlock?(error)
+          error.is_a?(ActiveRecord::StatementInvalid) &&
+            DEADLOCK_MESSAGES.any? { |m| error.message.include?(m) }
+        end
+
         # Retry the given block when encountering a deadlock. For any other
         # exceptions, they are reraised. This is used to handle cases where
         # the database may be busy but the transaction would succeed if
@@ -28,10 +37,15 @@ module Deimos
         # be wrapped in a transaction.
         # Sleeps for a random number of seconds to prevent multiple transactions
         # from retrying at the same time.
-        # @param tags [Array] Tags to attach when logging and reporting metrics.
+        # @param tags [Array, String, nil] Tags to attach when logging and reporting metrics.
+        #   A scalar (e.g. a topic String) is coerced to a single-element array.
         # @yield Yields to the block that may deadlock.
         # @return [void]
         def wrap(tags=[])
+          # Normalize to an Array so downstream logging/metrics (and dogstatsd's
+          # `tags.to_a`) never receive a bare String, which would raise
+          # NoMethodError inside this rescue and defeat the retry mechanism.
+          tags = Array(tags)
           count = RETRY_COUNT
 
           begin
@@ -40,7 +54,7 @@ module Deimos
             end
           rescue ActiveRecord::StatementInvalid => e
             # Reraise if not a known deadlock
-            raise if DEADLOCK_MESSAGES.none? { |m| e.message.include?(m) }
+            raise unless deadlock?(e)
 
             # Reraise if all retries exhausted
             raise if count <= 0

data/lib/deimos/utils/schema_class.rb

@@ -1,27 +1,60 @@
 # frozen_string_literal: true
 
-require 'avro_gen'
-
 module Deimos
   module Utils
-    # Backwards-compatible shim. The schema class helpers moved to
-    # AvroGen::SchemaClass. This delegates with a deprecation warning.
+    # Class used by SchemaClassGenerator and Consumer/Producer interfaces
     module SchemaClass
       class << self
-        def method_missing(name, ...)
-          if AvroGen::SchemaClass.respond_to?(name)
-            Deimos::Logging.deprecate(
-              "Deimos::Utils::SchemaClass.#{name} is deprecated; use AvroGen::SchemaClass.#{name} instead."
-            )
-            AvroGen::SchemaClass.send(name, ...)
-          else
-            super
+
+        # @param namespace [String]
+        # @return [Array<String>]
+        def modules_for(namespace)
+          modules = ['Schemas']
+          namespace_override = nil
+          module_namespace = namespace
+
+          if Deimos.config.schema.use_full_namespace
+            if Deimos.config.schema.schema_namespace_map.present?
+              namespace_keys = Deimos.config.schema.schema_namespace_map.keys.sort_by { |k| -k.length }
+              namespace_override = namespace_keys.find { |k| module_namespace.include?(k) }
+            end
+
+            if namespace_override.present?
+              # override default module
+              modules = Array(Deimos.config.schema.schema_namespace_map[namespace_override])
+              module_namespace = module_namespace.gsub(/#{namespace_override}\.?/, '')
+            end
+
+            namespace_folders = module_namespace.split('.').map { |f| f.underscore.camelize }
+            modules.concat(namespace_folders) if namespace_folders.any?
           end
+
+          modules
         end
 
-        def respond_to_missing?(name, include_private=false)
-          AvroGen::SchemaClass.respond_to?(name, include_private) || super
+        # Converts a raw payload into an instance of the Schema Class
+        # @param payload [Hash, Deimos::SchemaClass::Base]
+        # @param schema [String]
+        # @param namespace [String]
+        # @return [Deimos::SchemaClass::Record]
+        def instance(payload, schema, namespace='')
+          return payload if payload.is_a?(Deimos::SchemaClass::Base)
+
+          klass = klass(schema, namespace)
+          return payload if klass.nil? || payload.nil?
+
+          klass.new_from_message(**payload.symbolize_keys)
         end
+
+        # Determine and return the SchemaClass with the provided schema and namespace
+        # @param schema [String]
+        # @param namespace [String]
+        # @return [Deimos::SchemaClass]
+        def klass(schema, namespace)
+          constants = modules_for(namespace) + [schema.underscore.camelize.singularize]
+          constants.join('::').safe_constantize
+        end
+
       end
     end
   end

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '2.6.0-beta1'
+  VERSION = '2.6.0'
 end

data/lib/deimos.rb

@@ -17,9 +17,9 @@ require 'deimos/backends/kafka_async'
 require 'deimos/backends/test'
 
 require 'deimos/schema_backends/base'
-require 'avro_gen'
-require 'deimos/schema_class'
 require 'deimos/utils/schema_class'
+require 'deimos/schema_class/enum'
+require 'deimos/schema_class/record'
 
 require 'deimos/ext/schema_route'
 require 'deimos/ext/consumer_route'
@@ -49,6 +49,7 @@ module Deimos
     deimos.encode_message
     deimos.batch_consumption.invalid_records
     deimos.batch_consumption.valid_records
+    deimos.batch_consumption.initial_failure
     deimos.outbox.produce
   ).freeze
 
@@ -102,7 +103,7 @@ module Deimos
         # Initialize an instance of the provided schema
         # in the event the schema class is an override, the inherited
         # schema and namespace will be applied
-        schema_class = AvroGen::SchemaClass.klass(schema, namespace)
+        schema_class = Utils::SchemaClass.klass(schema, namespace)
         if schema_class.nil?
           schema_backend_class(backend: backend).
             new(

data/lib/generators/deimos/schema_class/templates/schema_class.rb.tt

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+# This file is autogenerated by Deimos, Do NOT modify
+<%= @modules.map { |m| "module #{m}"}.join('; ') %>
+  ### Primary Schema Class ###
+  <%=- @main_class_definition -%>
+
+<%= @modules.map { "end" }.join('; ') %>

data/lib/generators/deimos/schema_class/templates/schema_enum.rb.tt

@@ -0,0 +1,13 @@
+  # Autogenerated Schema for Enum at <%= @current_schema.namespace %>.<%= @current_schema.name %>
+  class <%= Deimos::SchemaBackends::AvroBase.schema_classname(@current_schema) %> < Deimos::SchemaClass::Enum
+    # @override
+    def symbols
+      %w(<%= @current_schema.symbols.join(' ') %>)
+    end
+
+    <% @current_schema.symbols.each do |symbol| %>
+    def <%= symbol.underscore %>?
+      @value == '<%= symbol %>'
+    end
+    <% end %>
+  end

data/lib/generators/deimos/schema_class/templates/schema_record.rb.tt

@@ -0,0 +1,102 @@
+  # Autogenerated Schema for Record at <%= @current_schema.namespace %>.<%= @current_schema.name %>
+  class <%= Deimos::SchemaBackends::AvroBase.schema_classname(@current_schema) %> < Deimos::SchemaClass::Record
+<% if @sub_schema_templates.present? -%>
+
+    ### Secondary Schema Classes ###
+  <%- @sub_schema_templates.each do |schema_template| -%>
+  <%=- schema_template.gsub(/^/, "  ") %>
+
+  <%- end -%>
+<% end -%>
+
+  <%- if @field_assignments.select{ |h| h[:is_schema_class] }.any? -%>
+    ### Attribute Readers ###
+  <%- @field_assignments.select{ |h| h[:is_schema_class] }.each do |method_definition| -%>
+    # @return [<%= method_definition[:deimos_type] %>]
+    attr_reader :<%= method_definition[:field].name %>
+  <%- end -%>
+
+<% end -%>
+<%- if @field_assignments.select{ |h| !h[:is_schema_class] }.any? -%>
+    ### Attribute Accessors ###
+  <%- @field_assignments.select{ |h| !h[:is_schema_class] }.each do |method_definition| -%>
+    # @return [<%= method_definition[:deimos_type] %>]
+    attr_accessor :<%= method_definition[:field].name %>
+  <%- end -%>
+
+<% end -%>
+<%- if @field_assignments.select{ |h| h[:is_schema_class] }.any? -%>
+    ### Attribute Writers ###
+  <%- @field_assignments.select{ |h| h[:is_schema_class] }.each do |method_definition| -%>
+    # @return [<%= method_definition[:deimos_type] %>]
+    def <%= method_definition[:field].name %>=(<%= method_definition[:method_argument] %>)
+    <%- if method_definition[:field_type] == :array -%>
+      @<%= method_definition[:field].name %> = values&.map do |value|
+        <%= method_definition[:field_initialization] %>
+      end
+    <%- elsif method_definition[:field_type] == :map -%>
+      @<%= method_definition[:field].name %> = values&.transform_values do |value|
+        <%= method_definition[:field_initialization] %>
+      end
+    <%- else -%>
+      @<%= method_definition[:field].name %> = <%= method_definition[:field_initialization] %>
+    <%- end -%>
+    end
+
+  <%- end -%>
+<% end -%>
+<%- if @field_assignments.select{ |h| h[:is_complex_union] }.any? -%>
+    <%- @field_assignments.select{ |h| h[:is_complex_union] }.each do |method_definition| -%>
+    # Helper method to determine which schema type to use for <%= method_definition[:field].name %>
+    # @param value [Hash, nil]
+    # @param from_message [Boolean] whether this was initialized from a real Avro message
+    # @return [Object, nil]
+    def initialize_<%= method_definition[:field].name %>_type(value, from_message: false)
+      return nil if value.nil?
+
+      klass = [<%= method_definition[:field].type.schemas.reject { |s| s.type_sym == :null }.select { |s| s.type_sym == :record }.map { |s| Deimos::SchemaBackends::AvroBase.schema_classname(s) }.join(', ') %>].find do |candidate|
+        fields = candidate.new.as_json.keys
+        (value.keys - fields).empty?
+      end
+
+      klass.initialize_from_value(value, from_message: self._from_message)
+    end
+
+    <%- end -%>
+<% end -%>
+    # @override
+    <%= @initialization_definition %>
+      @_from_message = _from_message
+      super
+<%- @fields.each do |field| -%>
+      self.<%= field.name %> = <%= field.name %>
+<% end -%>
+    end
+
+    # @override
+    def schema
+      '<%= @current_schema.name %>'
+    end
+
+    # @override
+    def namespace
+      '<%= @current_schema.namespace %>'
+    end
+<%- if @tombstone_assignment %>
+    def self.tombstone(key)
+      record = self.allocate
+<%- if @tombstone_assignment.present? -%>
+      <%= @tombstone_assignment %>
+<%- end -%>
+      record
+    end
+<%- end %>
+    # @override
+    def as_json(_opts={})
+      {
+<%- @fields.each do |field| -%>
+        <%= field_as_json(field) %>
+<% end -%>
+      }
+    end
+  end