deimos-temp-fork 0.0.1

data/lib/deimos/railtie.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+# Add rake task to Rails.
+class Deimos::Railtie < Rails::Railtie
+  rake_tasks do
+    load 'tasks/deimos.rake'
+  end
+end

data/lib/deimos/schema_backends/avro_base.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+require 'avro'
+require 'avro_turf'
+require 'avro_turf/mutable_schema_store'
+require_relative 'avro_schema_coercer'
+
+module Deimos
+  module SchemaBackends
+    # Encode / decode using Avro, either locally or via schema registry.
+    class AvroBase < Base
+      attr_accessor :schema_store
+
+      # @override
+      def initialize(schema:, namespace:)
+        super(schema: schema, namespace: namespace)
+        @schema_store = AvroTurf::MutableSchemaStore.new(path: Deimos.config.schema.path)
+      end
+
+      # @override
+      def encode_key(key_id, key, topic: nil)
+        @key_schema ||= _generate_key_schema(key_id)
+        field_name = _field_name_from_schema(@key_schema)
+        payload = { field_name => key }
+        encode(payload, schema: @key_schema['name'], topic: topic)
+      end
+
+      # @override
+      def decode_key(payload, key_id)
+        @key_schema ||= _generate_key_schema(key_id)
+        field_name = _field_name_from_schema(@key_schema)
+        decode(payload, schema: @key_schema['name'])[field_name]
+      end
+
+      # :nodoc:
+      def sql_type(field)
+        type = field.type.type
+        return type if %w(array map record).include?(type)
+
+        if type == :union
+          non_null = field.type.schemas.reject { |f| f.type == :null }
+          if non_null.size > 1
+            warn("WARNING: #{field.name} has more than one non-null type. Picking the first for the SQL type.")
+          end
+          return non_null.first.type
+        end
+        return type.to_sym if %w(float boolean).include?(type)
+        return :integer if type == 'int'
+        return :bigint if type == 'long'
+
+        if type == 'double'
+          warn('Avro `double` type turns into SQL `float` type. Please ensure you have the correct `limit` set.')
+          return :float
+        end
+
+        :string
+      end
+
+      # @override
+      def coerce_field(field, value)
+        AvroSchemaCoercer.new(avro_schema).coerce_type(field.type, value)
+      end
+
+      # @override
+      def schema_fields
+        avro_schema.fields.map do |field|
+          enum_values = field.type.type == 'enum' ? field.type.symbols : []
+          SchemaField.new(field.name, field.type, enum_values)
+        end
+      end
+
+      # @override
+      def validate(payload, schema:)
+        Avro::SchemaValidator.validate!(avro_schema(schema), payload,
+                                        recursive: true,
+                                        fail_on_extra_fields: true)
+      end
+
+      # @override
+      def self.mock_backend
+        :avro_validation
+      end
+
+      # @override
+      def self.content_type
+        'avro/binary'
+      end
+
+    private
+
+      # @param schema [String]
+      # @return [Avro::Schema]
+      def avro_schema(schema=nil)
+        schema ||= @schema
+        @schema_store.find(schema, @namespace)
+      end
+
+      # Generate a key schema from the given value schema and key ID. This
+      # is used when encoding or decoding keys from an existing value schema.
+      # @param key_id [Symbol]
+      # @return [Hash]
+      def _generate_key_schema(key_id)
+        key_field = avro_schema.fields.find { |f| f.name == key_id.to_s }
+        name = _key_schema_name(@schema)
+        key_schema = {
+          'type' => 'record',
+          'name' => name,
+          'namespace' => @namespace,
+          'doc' => "Key for #{@namespace}.#{@schema} - autogenerated by Deimos",
+          'fields' => [
+            {
+              'name' => key_id,
+              'type' => key_field.type.type_sym.to_s
+            }
+          ]
+        }
+        @schema_store.add_schema(key_schema)
+        key_schema
+      end
+
+      # @param value_schema [Hash]
+      # @return [String]
+      def _field_name_from_schema(value_schema)
+        raise "Schema #{@schema} not found!" if value_schema.nil?
+        if value_schema['fields'].nil? || value_schema['fields'].empty?
+          raise "Schema #{@schema} has no fields!"
+        end
+
+        value_schema['fields'][0]['name']
+      end
+
+      # @param schema [String]
+      # @return [String]
+      def _key_schema_name(schema)
+        "#{schema}_key"
+      end
+    end
+  end
+end

data/lib/deimos/schema_backends/avro_local.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require_relative 'avro_base'
+
+module Deimos
+  module SchemaBackends
+    # Encode / decode using local Avro encoding.
+    class AvroLocal < AvroBase
+      # @override
+      def decode_payload(payload, schema:)
+        avro_turf.decode(payload, schema_name: schema, namespace: @namespace)
+      end
+
+      # @override
+      def encode_payload(payload, schema: nil, topic: nil)
+        avro_turf.encode(payload, schema_name: schema, namespace: @namespace)
+      end
+
+    private
+
+      # @return [AvroTurf]
+      def avro_turf
+        @avro_turf ||= AvroTurf.new(
+          schemas_path: Deimos.config.schema.path,
+          schema_store: @schema_store
+        )
+      end
+    end
+  end
+end

data/lib/deimos/schema_backends/avro_schema_coercer.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+require 'active_support/time'
+
+module Deimos
+  # Class to coerce values in a payload to match a schema.
+  class AvroSchemaCoercer
+    # @param schema [Avro::Schema]
+    def initialize(schema)
+      @schema = schema
+    end
+
+    # Coerce sub-records in a payload to match the schema.
+    # @param type [Avro::Schema::UnionSchema]
+    # @param val [Object]
+    # @return [Object]
+    def coerce_union(type, val)
+      union_types = type.schemas.map { |s| s.type.to_sym }
+      return nil if val.nil? && union_types.include?(:null)
+
+      schema_type = type.schemas.find { |s| s.type.to_sym != :null }
+      coerce_type(schema_type, val)
+    end
+
+    # Coerce sub-records in a payload to match the schema.
+    # @param type [Avro::Schema::RecordSchema]
+    # @param val [Object]
+    # @return [Object]
+    def coerce_record(type, val)
+      record = val.map do |name, value|
+        field = type.fields.find { |f| f.name == name }
+        coerce_type(field.type, value)
+      end
+      val.keys.zip(record).to_h
+    end
+
+    # Coerce values in a payload to match the schema.
+    # @param type [Avro::Schema]
+    # @param val [Object]
+    # @return [Object]
+    def coerce_type(type, val)
+      int_classes = [Time, ActiveSupport::TimeWithZone]
+      field_type = type.type.to_sym
+
+      case field_type
+      when :int, :long
+        if %w(timestamp-millis timestamp-micros).include?(type.logical_type)
+          val
+        elsif val.is_a?(Integer) ||
+              _is_integer_string?(val) ||
+              int_classes.any? { |klass| val.is_a?(klass) }
+          val.to_i
+        else
+          val # this will fail
+        end
+      when :float, :double
+        if val.is_a?(Numeric) || _is_float_string?(val)
+          val.to_f
+        else
+          val # this will fail
+        end
+      when :string
+        if val.respond_to?(:to_str)
+          val.to_s
+        elsif _is_to_s_defined?(val)
+          val.to_s
+        else
+          val # this will fail
+        end
+      when :boolean
+        if val.nil? || val == false
+          false
+        else
+          true
+        end
+      when :union
+        coerce_union(type, val)
+      when :record
+        coerce_record(type, val)
+      else
+        val
+      end
+    end
+
+  private
+
+    # @param val [String]
+    # @return [Boolean]
+    def _is_integer_string?(val)
+      return false unless val.is_a?(String)
+
+      begin
+        true if Integer(val)
+      rescue StandardError
+        false
+      end
+    end
+
+    # @param val [String]
+    # @return [Boolean]
+    def _is_float_string?(val)
+      return false unless val.is_a?(String)
+
+      begin
+        true if Float(val)
+      rescue StandardError
+        false
+      end
+    end
+
+    # @param val [Object]
+    # @return [Boolean]
+    def _is_to_s_defined?(val)
+      return false if val.nil?
+
+      Object.instance_method(:to_s).bind(val).call != val.to_s
+    end
+  end
+end

data/lib/deimos/schema_backends/avro_schema_registry.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require_relative 'avro_base'
+require_relative 'avro_validation'
+require 'avro_turf/messaging'
+
+module Deimos
+  module SchemaBackends
+    # Encode / decode using the Avro schema registry.
+    class AvroSchemaRegistry < AvroBase
+      # @override
+      def decode_payload(payload, schema:)
+        avro_turf_messaging.decode(payload, schema_name: schema)
+      end
+
+      # @override
+      def encode_payload(payload, schema: nil, topic: nil)
+        avro_turf_messaging.encode(payload, schema_name: schema, subject: topic || schema)
+      end
+
+    private
+
+      # @return [AvroTurf::Messaging]
+      def avro_turf_messaging
+        @avro_turf_messaging ||= AvroTurf::Messaging.new(
+          schema_store: @schema_store,
+          registry_url: Deimos.config.schema.registry_url,
+          schemas_path: Deimos.config.schema.path,
+          namespace: @namespace
+        )
+      end
+    end
+  end
+end

data/lib/deimos/schema_backends/avro_validation.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require_relative 'avro_base'
+
+module Deimos
+  module SchemaBackends
+    # Leave Ruby hashes as is but validate them against the schema.
+    # Useful for unit tests.
+    class AvroValidation < AvroBase
+      # @override
+      def decode_payload(payload, schema: nil)
+        payload.with_indifferent_access
+      end
+
+      # @override
+      def encode_payload(payload, schema: nil, topic: nil)
+        payload.with_indifferent_access
+      end
+    end
+  end
+end

data/lib/deimos/schema_backends/base.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+module Deimos
+  # Represents a field in the schema.
+  class SchemaField
+    attr_accessor :name, :type, :enum_values
+
+    # @param name [String]
+    # @param type [Object]
+    # @param enum_values [Array<String>]
+    def initialize(name, type, enum_values=[])
+      @name = name
+      @type = type
+      @enum_values = enum_values
+    end
+  end
+
+  module SchemaBackends
+    # Base class for encoding / decoding.
+    class Base
+      attr_accessor :schema, :namespace, :key_schema
+
+      # @param schema [String|Symbol]
+      # @param namespace [String]
+      def initialize(schema:, namespace: nil)
+        @schema = schema
+        @namespace = namespace
+      end
+
+      # Encode a payload with a schema. Public method.
+      # @param payload [Hash]
+      # @param schema [Symbol|String]
+      # @param topic [String]
+      # @return [String]
+      def encode(payload, schema: nil, topic: nil)
+        validate(payload, schema: schema || @schema)
+        encode_payload(payload, schema: schema || @schema, topic: topic)
+      end
+
+      # Decode a payload with a schema. Public method.
+      # @param payload [String]
+      # @param schema [Symbol|String]
+      # @return [Hash,nil]
+      def decode(payload, schema: nil)
+        return nil if payload.nil?
+        decode_payload(payload, schema: schema || @schema)
+      end
+
+      # Given a hash, coerce its types to our schema. To be defined by subclass.
+      # @param payload [Hash]
+      # @return [Hash]
+      def coerce(payload)
+        result = {}
+        self.schema_fields.each do |field|
+          name = field.name
+          next unless payload.key?(name)
+
+          val = payload[name]
+          result[name] = coerce_field(field, val)
+        end
+        result.with_indifferent_access
+      end
+
+      # Indicate a class which should act as a mocked version of this backend.
+      # This class should perform all validations but not actually do any
+      # encoding.
+      # Note that the "mock" version (e.g. avro_validation) should return
+      # its own symbol when this is called, since it may be called multiple
+      # times depending on the order of RSpec helpers.
+      # @return [Symbol]
+      def self.mock_backend
+        :mock
+      end
+
+      # The content type to use when encoding / decoding requests over HTTP via ActionController.
+      # @return [String]
+      def self.content_type
+        raise NotImplementedError
+      end
+
+      # Encode a payload. To be defined by subclass.
+      # @param payload [Hash]
+      # @param schema [Symbol|String]
+      # @param topic [String]
+      # @return [String]
+      def encode_payload(_payload, schema:, topic: nil)
+        raise NotImplementedError
+      end
+
+      # Decode a payload. To be defined by subclass.
+      # @param payload [String]
+      # @param schema [String|Symbol]
+      # @return [Hash]
+      def decode_payload(_payload, schema:)
+        raise NotImplementedError
+      end
+
+      # Validate that a payload matches the schema. To be defined by subclass.
+      # @param payload [Hash]
+      # @param schema [String|Symbol]
+      def validate(_payload, schema:)
+        raise NotImplementedError
+      end
+
+      # List of field names belonging to the schema. To be defined by subclass.
+      # @return [Array<SchemaField>]
+      def schema_fields
+        raise NotImplementedError
+      end
+
+      # Given a value and a field definition (as defined by whatever the
+      # underlying schema library is), coerce the given value to
+      # the given field type.
+      # @param field [SchemaField]
+      # @param value [Object]
+      # @return [Object]
+      def coerce_field(_field, _value)
+        raise NotImplementedError
+      end
+
+      # Given a field definition, return the SQL type that might be used in
+      # ActiveRecord table creation - e.g. for Avro, a `long` type would
+      # return `:bigint`. There are also special values that need to be returned:
+      # `:array`, `:map` and `:record`, for types representing those structures.
+      # `:enum` is also recognized.
+      # @param field [SchemaField]
+      # @return [Symbol]
+      def sql_type(field)
+        raise NotImplementedError
+      end
+
+      # Encode a message key. To be defined by subclass.
+      # @param key [String|Hash] the value to use as the key.
+      # @param key_id [Symbol|String] the field name of the key.
+      # @param topic [String]
+      # @return [String]
+      def encode_key(_key, _key_id, topic: nil)
+        raise NotImplementedError
+      end
+
+      # Decode a message key. To be defined by subclass.
+      # @param payload [Hash] the message itself.
+      # @param key_id [Symbol|String] the field in the message to decode.
+      # @return [String]
+      def decode_key(_payload, _key_id)
+        raise NotImplementedError
+      end
+    end
+  end
+end