deimos-ruby 1.8.1.pre.beta1 → 1.8.1.pre.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 335328a0ca5fb9147405485a66944ebe0d05af552ec09ec9605d76a7e7da7e9a
-  data.tar.gz: 438a9a91bda52be144d63867a4ae42dd1901cc7472f61232044aa934efebb464
+  metadata.gz: e6f44746ef34713f973b24f4cf80954915545edec5e0f006e4d1a6c751478afa
+  data.tar.gz: 17adb23fdc3e735cf8eab51f4c1df2bbcf1140c9a5696c0f8e17dc38bd728ba9
 SHA512:
-  metadata.gz: 3e01c5c9db93cb6e0f56dafa2f1b301c30b9cc11c397ca35f77b80cfe62bd9600eacb8774516ee1c81337193687944a8c7bec3cba400df6ea8b1c33af7e82c27
-  data.tar.gz: 9de8cea35db671c43123d096653a6423ccfcaad8a2b8e9478f787e879bd7fd50a626604b8626f79b5f8654c84130fded58385d6f75fe44e59e1bfb5a284e1096
+  metadata.gz: d920dc6b7b2020364a7095ab61a986a3bb16c5fac5367227ed8cbded422399e78887fc91ecce81d9857c66bce5739f6423335e835ca87612a1a2306c50cd93b0
+  data.tar.gz: 78fd60871b0852b03e91e2223afbe5ac419dc929d6de6340799f8376f89efffd2775e4d93025ab8089cac5a7579d00bc33c64c3f9ec50cc3fb603f6381e3d82a

data/CHANGELOG.md

@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+## 1.8.1-beta2 - 2020-07-28
+
+### Features :star:
+- Add `SchemaControllerMixin` to encode and decode schema-encoded
+  payloads in Rails controllers.
+
 ## 1.8.1-beta1 - 2020-07-22
 
 ### Fixes :wrench:

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    deimos-ruby (1.8.1.pre.beta1)
+    deimos-ruby (1.8.1.pre.beta2)
       avro_turf (~> 0.11)
       phobos (~> 1.9)
       ruby-kafka (~> 0.7)
@@ -87,7 +87,7 @@ GEM
       rake (~> 13.0)
     dogstatsd-ruby (4.8.1)
     erubi (1.9.0)
-    excon (0.75.0)
+    excon (0.76.0)
     exponential-backoff (0.0.4)
     ffi (1.13.1)
     formatador (0.2.5)
@@ -205,6 +205,14 @@ GEM
     rspec-mocks (3.9.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
+    rspec-rails (4.0.1)
+      actionpack (>= 4.2)
+      activesupport (>= 4.2)
+      railties (>= 4.2)
+      rspec-core (~> 3.9)
+      rspec-expectations (~> 3.9)
+      rspec-mocks (~> 3.9)
+      rspec-support (~> 3.9)
     rspec-support (3.9.3)
     rspec_junit_formatter (0.4.1)
       rspec-core (>= 2, < 4, != 2.12.0)
@@ -250,7 +258,6 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  activerecord (~> 6)
   activerecord-import
   avro (~> 1.9)
   database_cleaner (~> 1.7)
@@ -265,6 +272,7 @@ DEPENDENCIES
   rails (~> 6)
   rake (~> 13)
   rspec (~> 3)
+  rspec-rails (~> 4)
   rspec_junit_formatter (~> 0.3)
   rubocop (~> 0.72)
   rubocop-rspec (~> 1.27)

data/README.md

@@ -22,6 +22,7 @@ Built on Phobos and hence Ruby-Kafka.
         * [Kafka Message Keys](#kafka-message-keys)
    * [Consumers](#consumers)
    * [Rails Integration](#rails-integration)
+        * [Controller Mixin](#controller-mixin)
    * [Database Backend](#database-backend)
    * [Database Poller](#database-poller)
    * [Running Consumers](#running-consumers)
@@ -447,6 +448,58 @@ class Widget < ActiveRecord::Base
 end
 ```
 
+### Controller Mixin
+
+Deimos comes with a mixin for `ActionController` which automatically encodes and decodes schema
+payloads. There are some advantages to encoding your data in e.g. Avro rather than straight JSON,
+particularly if your service is talking to another backend service rather than the front-end
+browser:
+
+* It enforces a contract between services. Solutions like [OpenAPI](https://swagger.io/specification/) 
+  do this as well, but in order for the client to know the contract, usually some kind of code 
+  generation has to happen. Using schemas ensures both sides know the contract without having to change code.
+  In addition, OpenAPI is now a huge and confusing format, and using simpler schema formats
+  can be beneficial.
+* Using Avro or Protobuf ensures both forwards and backwards compatibility,
+  which reduces the need for versioning since both sides can simply ignore fields they aren't aware
+  of.
+* Encoding and decoding using Avro or Protobuf is generally faster than straight JSON, and
+  results in smaller payloads and therefore less network traffic.
+
+To use the mixin, add the following to your `WhateverController`:
+
+```ruby
+class WhateverController < ApplicationController
+  include Deimos::Utils::SchemaControllerMixin
+
+  request_namespace 'my.namespace.requests'
+  response_namespace 'my.namespace.responses'
+  
+  # Add a "schemas" line for all routes that should encode/decode schemas.
+  # Default is to match the schema name to the route name.
+  schemas :index
+  # will look for: my.namespace.requests.Index.avsc
+  #                my.namespace.responses.Index.avsc 
+  
+  # If all routes use the default, you can add them all at once
+  schemas :index, :show, :update
+
+  # Different schemas can be specified as well
+  schemas :index, :show, request: 'IndexRequest', response: 'IndexResponse'
+
+  # To access the encoded data, use the `payload` helper method, and to render it back,
+  # use the `render_schema` method.
+  
+  def index
+    response = { 'response_id' => payload['request_id'] + 'hi mom' }
+    render_schema(response)
+  end
+end
+```
+
+To make use of this feature, your requests and responses need to have the correct content type.
+For Avro content, this is the `avro/binary` content type.
+
 # Database Backend
 
 Deimos provides a way to allow Kafka messages to be created inside a

data/deimos-ruby.gemspec

@@ -23,7 +23,6 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency('ruby-kafka', '~> 0.7')
   spec.add_runtime_dependency('sigurd', '0.0.1')
 
-  spec.add_development_dependency('activerecord', '~> 6')
   spec.add_development_dependency('activerecord-import')
   spec.add_development_dependency('avro', '~> 1.9')
   spec.add_development_dependency('database_cleaner', '~> 1.7')
@@ -38,6 +37,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency('rake', '~> 13')
   spec.add_development_dependency('rspec', '~> 3')
   spec.add_development_dependency('rspec_junit_formatter', '~>0.3')
+  spec.add_development_dependency('rspec-rails', '~> 4')
   spec.add_development_dependency('rubocop', '~> 0.72')
   spec.add_development_dependency('rubocop-rspec', '~> 1.27')
   spec.add_development_dependency('sqlite3', '~> 1.3')

data/lib/deimos.rb

@@ -23,6 +23,7 @@ require 'deimos/monkey_patches/phobos_producer'
 require 'deimos/monkey_patches/phobos_cli'
 
 require 'deimos/railtie' if defined?(Rails)
+require 'deimos/utils/schema_controller_mixin' if defined?(ActionController)
 
 if defined?(ActiveRecord)
   require 'deimos/kafka_source'

data/lib/deimos/schema_backends/avro_base.rb

@@ -82,6 +82,11 @@ module Deimos
         :avro_validation
       end
 
+      # @override
+      def self.content_type
+        'avro/binary'
+      end
+
     private
 
       # @param schema [String]

data/lib/deimos/schema_backends/base.rb

@@ -71,6 +71,12 @@ module Deimos
         :mock
       end
 
+      # The content type to use when encoding / decoding requests over HTTP via ActionController.
+      # @return [String]
+      def self.content_type
+        raise NotImplementedError
+      end
+
       # Encode a payload. To be defined by subclass.
       # @param payload [Hash]
       # @param schema [Symbol|String]

data/lib/deimos/utils/schema_controller_mixin.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module Deimos
+  module Utils
+    # Mixin to automatically decode schema-encoded payloads when given the correct content type,
+    # and provide the `render_schema` method to encode the payload for responses.
+    module SchemaControllerMixin
+      extend ActiveSupport::Concern
+
+      included do
+        Mime::Type.register('avro/binary', :avro)
+
+        attr_accessor :payload
+
+        if respond_to?(:before_filter)
+          before_filter(:decode_schema, if: :schema_format?)
+        else
+          before_action(:decode_schema, if: :schema_format?)
+        end
+      end
+
+      # :nodoc:
+      module ClassMethods
+        # @return [Hash<String, Hash<Symbol, String>>]
+        def schema_mapping
+          @schema_mapping ||= {}
+        end
+
+        # Indicate which schemas should be assigned to actions.
+        # @param actions [Symbol]
+        # @param request [String]
+        # @param response [String]
+        def schemas(*actions, request: nil, response: nil)
+          actions.each do |action|
+            request ||= action.to_s.titleize
+            response ||= action.to_s.titleize
+            schema_mapping[action.to_s] = { request: request, response: response }
+          end
+        end
+
+        # @return [Hash<Symbol, String>]
+        def namespaces
+          @namespaces ||= {}
+        end
+
+        # Set the namespace for both requests and responses.
+        # @param name [String]
+        def namespace(name)
+          request_namespace(name)
+          response_namespace(name)
+        end
+
+        # Set the namespace for requests.
+        # @param name [String]
+        def request_namespace(name)
+          namespaces[:request] = name
+        end
+
+        # Set the namespace for repsonses.
+        # @param name [String]
+        def response_namespace(name)
+          namespaces[:response] = name
+        end
+      end
+
+      # @return [Boolean]
+      def schema_format?
+        request.content_type == Deimos.schema_backend_class.content_type
+      end
+
+      # Get the namespace from either an existing instance variable, or tease it out of the schema.
+      # @param type [Symbol] :request or :response
+      # @return [Array<String, String>] the namespace and schema.
+      def parse_namespace(type)
+        namespace = self.class.namespaces[type]
+        schema = self.class.schema_mapping[params['action']][type]
+        if schema.nil?
+          raise "No #{type} schema defined for #{params[:controller]}##{params[:action]}!"
+        end
+
+        if namespace.nil?
+          last_period = schema.rindex('.')
+          namespace, schema = schema.split(last_period)
+        end
+        if namespace.nil? || schema.nil?
+          raise "No request namespace defined for #{params[:controller]}##{params[:action]}!"
+        end
+
+        [namespace, schema]
+      end
+
+      # Decode the payload with the parameters.
+      def decode_schema
+        namespace, schema = parse_namespace(:request)
+        decoder = Deimos.schema_backend(schema: schema, namespace: namespace)
+        @payload = decoder.decode(request.body.read).with_indifferent_access
+        request.body.rewind if request.body.respond_to?(:rewind)
+      end
+
+      # Render a hash into a payload as specified by the configured schema and namespace.
+      # @param payload [Hash]
+      def render_schema(payload, schema: nil, namespace: nil)
+        namespace, schema = parse_namespace(:response) if !schema && !namespace
+        encoder = Deimos.schema_backend(schema: schema, namespace: namespace)
+        encoded = encoder.encode(payload)
+        response.headers['Content-Type'] = encoder.class.content_type
+        send_data(encoded)
+      end
+    end
+  end
+end

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.8.1-beta1'
+  VERSION = '1.8.1-beta2'
 end

data/spec/schemas/com/my-namespace/request/Index.avsc

@@ -0,0 +1,11 @@
+{
+  "namespace": "com.my-namespace.request",
+  "name": "Index",
+  "type": "record",
+  "fields": [
+    {
+      "name": "request_id",
+      "type": "string"
+    }
+  ]
+}

data/spec/schemas/com/my-namespace/request/UpdateRequest.avsc

@@ -0,0 +1,11 @@
+{
+  "namespace": "com.my-namespace.request",
+  "name": "UpdateRequest",
+  "type": "record",
+  "fields": [
+    {
+      "name": "update_request_id",
+      "type": "string"
+    }
+  ]
+}

data/spec/schemas/com/my-namespace/response/Index.avsc

@@ -0,0 +1,11 @@
+{
+  "namespace": "com.my-namespace.response",
+  "name": "Index",
+  "type": "record",
+  "fields": [
+    {
+      "name": "response_id",
+      "type": "string"
+    }
+  ]
+}

data/spec/schemas/com/my-namespace/response/UpdateResponse.avsc

@@ -0,0 +1,11 @@
+{
+  "namespace": "com.my-namespace.response",
+  "name": "UpdateResponse",
+  "type": "record",
+  "fields": [
+    {
+      "name": "update_response_id",
+      "type": "string"
+    }
+  ]
+}

data/spec/spec_helper.rb

@@ -2,6 +2,7 @@
 
 $LOAD_PATH.unshift(File.expand_path('../lib', __dir__))
 require 'active_record'
+require 'action_controller/railtie'
 require 'database_cleaner'
 require 'deimos'
 require 'deimos/metrics/mock'
@@ -11,6 +12,11 @@ require 'active_support/testing/time_helpers'
 require 'activerecord-import'
 require 'handlers/my_batch_consumer'
 require 'handlers/my_consumer'
+require 'rspec/rails'
+
+class DeimosApp < Rails::Application
+end
+DeimosApp.initialize!
 
 # Helpers for Executor/DbProducer
 module TestRunners

data/spec/utils/schema_controller_mixin_spec.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require 'deimos/utils/schema_controller_mixin'
+require 'deimos/schema_backends/avro_local'
+
+RSpec.describe Deimos::Utils::SchemaControllerMixin, type: :controller do
+
+  before(:each) do
+    Deimos.configure do
+      schema.backend(:avro_local)
+    end
+  end
+
+  controller(ActionController::Base) do
+    include Deimos::Utils::SchemaControllerMixin # rubocop:disable RSpec/DescribedClass
+
+    request_namespace 'com.my-namespace.request'
+    response_namespace 'com.my-namespace.response'
+    schemas :index, :show
+    schemas :update, request: 'UpdateRequest', response: 'UpdateResponse'
+
+    # :nodoc:
+    def index
+      render_schema({ 'response_id' => payload[:request_id] + ' mom' })
+    end
+
+    # :nodoc:
+    def show
+      render_schema({ 'response_id' => payload[:request_id] + ' dad' })
+    end
+
+    # :nodoc:
+    def update
+      render_schema({ 'update_response_id' => payload[:update_request_id] + ' sis' })
+    end
+  end
+
+  it 'should render the correct response for index' do
+    request_backend = Deimos.schema_backend(schema: 'Index',
+                                            namespace: 'com.my-namespace.request')
+    response_backend = Deimos.schema_backend(schema: 'Index',
+                                             namespace: 'com.my-namespace.response')
+    request.content_type = 'avro/binary'
+    get :index, body: request_backend.encode({ 'request_id' => 'hi' })
+    expect(response_backend.decode(response.body)).to eq({ 'response_id' => 'hi mom' })
+  end
+
+  it 'should render the correct response for show' do
+    request_backend = Deimos.schema_backend(schema: 'Index',
+                                            namespace: 'com.my-namespace.request')
+    response_backend = Deimos.schema_backend(schema: 'Index',
+                                             namespace: 'com.my-namespace.response')
+    request.content_type = 'avro/binary'
+    get :show, params: { id: 1 }, body: request_backend.encode({ 'request_id' => 'hi' })
+    expect(response_backend.decode(response.body)).to eq({ 'response_id' => 'hi dad' })
+  end
+
+  it 'should render the correct response for update' do
+    request_backend = Deimos.schema_backend(schema: 'UpdateRequest',
+                                            namespace: 'com.my-namespace.request')
+    response_backend = Deimos.schema_backend(schema: 'UpdateResponse',
+                                             namespace: 'com.my-namespace.response')
+    request.content_type = 'avro/binary'
+    post :update, params: { id: 1 }, body: request_backend.encode({ 'update_request_id' => 'hi' })
+    expect(response_backend.decode(response.body)).to eq({ 'update_response_id' => 'hi sis' })
+  end
+
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.8.1.pre.beta1
+  version: 1.8.1.pre.beta2
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-07-22 00:00:00.000000000 Z
+date: 2020-07-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf
@@ -66,20 +66,6 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.0.1
-- !ruby/object:Gem::Dependency
-  name: activerecord
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '6'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '6'
 - !ruby/object:Gem::Dependency
   name: activerecord-import
   requirement: !ruby/object:Gem::Requirement
@@ -276,6 +262,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: rspec-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
@@ -399,6 +399,7 @@ files:
 - lib/deimos/utils/deadlock_retry.rb
 - lib/deimos/utils/inline_consumer.rb
 - lib/deimos/utils/lag_reporter.rb
+- lib/deimos/utils/schema_controller_mixin.rb
 - lib/deimos/version.rb
 - lib/generators/deimos/active_record/templates/migration.rb.tt
 - lib/generators/deimos/active_record/templates/model.rb.tt
@@ -450,12 +451,17 @@ files:
 - spec/schemas/com/my-namespace/Wibble.avsc
 - spec/schemas/com/my-namespace/Widget.avsc
 - spec/schemas/com/my-namespace/WidgetTheSecond.avsc
+- spec/schemas/com/my-namespace/request/Index.avsc
+- spec/schemas/com/my-namespace/request/UpdateRequest.avsc
+- spec/schemas/com/my-namespace/response/Index.avsc
+- spec/schemas/com/my-namespace/response/UpdateResponse.avsc
 - spec/spec_helper.rb
 - spec/utils/db_poller_spec.rb
 - spec/utils/db_producer_spec.rb
 - spec/utils/deadlock_retry_spec.rb
 - spec/utils/lag_reporter_spec.rb
 - spec/utils/platform_schema_validation_spec.rb
+- spec/utils/schema_controller_mixin_spec.rb
 - support/deimos-solo.png
 - support/deimos-with-name-next.png
 - support/deimos-with-name.png
@@ -524,9 +530,14 @@ test_files:
 - spec/schemas/com/my-namespace/Wibble.avsc
 - spec/schemas/com/my-namespace/Widget.avsc
 - spec/schemas/com/my-namespace/WidgetTheSecond.avsc
+- spec/schemas/com/my-namespace/request/Index.avsc
+- spec/schemas/com/my-namespace/request/UpdateRequest.avsc
+- spec/schemas/com/my-namespace/response/Index.avsc
+- spec/schemas/com/my-namespace/response/UpdateResponse.avsc
 - spec/spec_helper.rb
 - spec/utils/db_poller_spec.rb
 - spec/utils/db_producer_spec.rb
 - spec/utils/deadlock_retry_spec.rb
 - spec/utils/lag_reporter_spec.rb
 - spec/utils/platform_schema_validation_spec.rb
+- spec/utils/schema_controller_mixin_spec.rb