deimos-ruby 1.8.0.pre.beta1 → 1.8.1.pre.beta4

data/spec/schemas/com/my-namespace/MyNestedSchema.avsc

@@ -0,0 +1,62 @@
+{
+  "namespace": "com.my-namespace",
+  "name": "MyNestedSchema",
+  "type": "record",
+  "doc": "Test schema",
+  "fields": [
+    {
+      "name": "test_id",
+      "type": "string",
+      "doc": "test string"
+    },
+    {
+      "name": "test_float",
+      "type": "float",
+      "doc": "test float"
+    },
+    {
+      "name": "test_array",
+      "type": {
+        "type": "array",
+        "items": "string"
+      }
+    },
+    {
+      "name": "some_nested_record",
+      "doc": "some nested record",
+      "type": {
+        "name": "MyNestedRecord",
+        "type": "record",
+        "fields": [
+          {
+            "name": "some_int",
+            "type": "int",
+            "doc": "some int"
+          },
+          {
+            "name": "some_float",
+            "type": "float",
+            "doc": "some float"
+          },
+          {
+            "name": "some_string",
+            "type": "string",
+            "doc": "some string"
+          },
+          {
+            "name": "some_optional_int",
+            "type": [ "null", "int" ],
+            "doc": "some optional int",
+            "default": null
+          }
+        ]
+      }
+    },
+    {
+      "name": "some_optional_record",
+      "doc": "some optional record",
+      "type": [ "null", "MyNestedRecord" ],
+      "default": null
+    }
+  ]
+}

data/spec/schemas/com/my-namespace/request/Index.avsc

@@ -0,0 +1,11 @@
+{
+  "namespace": "com.my-namespace.request",
+  "name": "Index",
+  "type": "record",
+  "fields": [
+    {
+      "name": "request_id",
+      "type": "string"
+    }
+  ]
+}

data/spec/schemas/com/my-namespace/request/UpdateRequest.avsc

@@ -0,0 +1,11 @@
+{
+  "namespace": "com.my-namespace.request",
+  "name": "UpdateRequest",
+  "type": "record",
+  "fields": [
+    {
+      "name": "update_request_id",
+      "type": "string"
+    }
+  ]
+}

data/spec/schemas/com/my-namespace/response/Index.avsc

@@ -0,0 +1,11 @@
+{
+  "namespace": "com.my-namespace.response",
+  "name": "Index",
+  "type": "record",
+  "fields": [
+    {
+      "name": "response_id",
+      "type": "string"
+    }
+  ]
+}

data/spec/schemas/com/my-namespace/response/UpdateResponse.avsc

@@ -0,0 +1,11 @@
+{
+  "namespace": "com.my-namespace.response",
+  "name": "UpdateResponse",
+  "type": "record",
+  "fields": [
+    {
+      "name": "update_response_id",
+      "type": "string"
+    }
+  ]
+}

data/spec/spec_helper.rb

@@ -2,6 +2,7 @@
 
 $LOAD_PATH.unshift(File.expand_path('../lib', __dir__))
 require 'active_record'
+require 'action_controller/railtie'
 require 'database_cleaner'
 require 'deimos'
 require 'deimos/metrics/mock'
@@ -11,6 +12,11 @@ require 'active_support/testing/time_helpers'
 require 'activerecord-import'
 require 'handlers/my_batch_consumer'
 require 'handlers/my_consumer'
+require 'rspec/rails'
+
+class DeimosApp < Rails::Application
+end
+DeimosApp.initialize!
 
 # Helpers for Executor/DbProducer
 module TestRunners
@@ -32,6 +38,7 @@ module TestRunners
   # Test runner
   class TestRunner
     attr_accessor :id, :started, :stopped, :should_error
+
     # :nodoc:
     def initialize(id=nil)
       @id = id

data/spec/utils/db_producer_spec.rb

@@ -16,11 +16,13 @@ each_db_config(Deimos::Utils::DbProducer) do
 
   before(:each) do
     stub_const('Deimos::Utils::DbProducer::BATCH_SIZE', 2)
+    stub_const('Deimos::Utils::DbProducer::DELETE_BATCH_SIZE', 1)
   end
 
   specify '#process_next_messages' do
     expect(producer).to receive(:retrieve_topics).and_return(%w(topic1 topic2))
     expect(producer).to receive(:process_topic).twice
+    expect(Deimos::KafkaTopicInfo).to receive(:ping_empty_topics).with(%w(topic1 topic2))
     expect(producer).to receive(:sleep).with(0.5)
     producer.process_next_messages
   end
@@ -40,6 +42,9 @@ each_db_config(Deimos::Utils::DbProducer) do
       Deimos::KafkaMessage.create!(topic: 'topic1',
                                    message: 'blah',
                                    key: "key#{i}")
+      Deimos::KafkaMessage.create!(topic: 'topic2',
+                                   message: 'blah',
+                                   key: "key#{i}")
     end
     stub_const('Deimos::Utils::DbProducer::BATCH_SIZE', 5)
     producer.current_topic = 'topic1'
@@ -280,6 +285,12 @@ each_db_config(Deimos::Utils::DbProducer) do
           message: "mess#{i}",
           partition_key: "key#{i}"
         )
+        Deimos::KafkaMessage.create!(
+          id: i,
+          topic: 'my-topic2',
+          message: "mess#{i}",
+          partition_key: "key#{i}"
+        )
       end
 
       expect(Deimos::KafkaTopicInfo).to receive(:lock).
@@ -288,9 +299,60 @@ each_db_config(Deimos::Utils::DbProducer) do
       expect(producer).to receive(:retrieve_messages).and_return(messages)
       expect(Deimos::KafkaTopicInfo).to receive(:register_error)
 
+      expect(Deimos::KafkaMessage.count).to eq(8)
+      producer.process_topic('my-topic')
       expect(Deimos::KafkaMessage.count).to eq(4)
+    end
+
+    it 'should retry deletes and not re-publish' do
+      messages = (1..4).map do |i|
+        Deimos::KafkaMessage.create!(
+          id: i,
+          topic: 'my-topic',
+          message: "mess#{i}",
+          partition_key: "key#{i}"
+        )
+      end
+      (5..8).each do |i|
+        Deimos::KafkaMessage.create!(
+          id: i,
+          topic: 'my-topic2',
+          message: "mess#{i}",
+          partition_key: "key#{i}"
+        )
+      end
+
+      raise_error = true
+      expect(Deimos::KafkaMessage).to receive(:where).exactly(5).times.and_wrap_original do |m, *args|
+        if raise_error
+          raise_error = false
+          raise 'Lock wait timeout'
+        end
+        m.call(*args)
+      end
+
+      expect(Deimos::KafkaTopicInfo).to receive(:lock).
+        with('my-topic', 'abc').and_return(true)
+      expect(producer).to receive(:retrieve_messages).ordered.and_return(messages)
+      expect(producer).to receive(:retrieve_messages).ordered.and_return([])
+      expect(phobos_producer).to receive(:publish_list).once.with(messages.map(&:phobos_message))
+
+      expect(Deimos::KafkaMessage.count).to eq(8)
       producer.process_topic('my-topic')
-      expect(Deimos::KafkaMessage.count).to eq(0)
+      expect(Deimos::KafkaMessage.count).to eq(4)
+    end
+
+    it 'should re-raise misc errors on delete' do
+      messages = (1..3).map do |i|
+        Deimos::KafkaMessage.create!(
+          id: i,
+          topic: 'my-topic',
+          message: "mess#{i}",
+          partition_key: "key#{i}"
+        )
+      end
+      expect(Deimos::KafkaMessage).to receive(:where).once.and_raise('OH NOES')
+      expect { producer.delete_messages(messages) }.to raise_exception('OH NOES')
     end
 
   end
@@ -309,21 +371,34 @@ each_db_config(Deimos::Utils::DbProducer) do
           Deimos::KafkaMessage.create!(topic: "topic#{i}", message: nil,
                                        created_at: (1 + i).minute.ago)
         end
+        Deimos::KafkaTopicInfo.create!(topic: 'topic1',
+                                       last_processed_at: 6.minutes.ago)
+        Deimos::KafkaTopicInfo.create!(topic: 'topic2',
+                                       last_processed_at: 3.minutes.ago)
+        Deimos::KafkaTopicInfo.create!(topic: 'topic3',
+                                       last_processed_at: 5.minutes.ago)
         allow(Deimos.config.metrics).to receive(:gauge)
         producer.send_pending_metrics
-        expect(Deimos.config.metrics).to have_received(:gauge).twice
+        expect(Deimos.config.metrics).to have_received(:gauge).exactly(6).times
+        # topic1 has the earliest message 4 minutes ago and last processed 6
+        # minutes ago, so the most amount of time we've seen nothing is 4 minutes
         expect(Deimos.config.metrics).to have_received(:gauge).
           with('pending_db_messages_max_wait', 4.minutes.to_i, tags: ['topic:topic1'])
+        # topic2 has earliest message 5 minutes ago and last processed 3 minutes
+        # ago, so we should give it 3 minutes
+        expect(Deimos.config.metrics).to have_received(:gauge).
+          with('pending_db_messages_max_wait', 3.minutes.to_i, tags: ['topic:topic2'])
+        # topic3 has no messages, so should get 0
         expect(Deimos.config.metrics).to have_received(:gauge).
-          with('pending_db_messages_max_wait', 5.minutes.to_i, tags: ['topic:topic2'])
+          with('pending_db_messages_max_wait', 0, tags: ['topic:topic3'])
+        expect(Deimos.config.metrics).to have_received(:gauge).
+          with('pending_db_messages_count', 3, tags: ['topic:topic1'])
+        expect(Deimos.config.metrics).to have_received(:gauge).
+          with('pending_db_messages_count', 3, tags: ['topic:topic2'])
+        expect(Deimos.config.metrics).to have_received(:gauge).
+          with('pending_db_messages_count', 0, tags: ['topic:topic3'])
       end
     end
-
-    it 'should send 0 if no messages' do
-      expect(Deimos.config.metrics).to receive(:gauge).
-        with('pending_db_messages_max_wait', 0)
-      producer.send_pending_metrics
-    end
   end
 
   example 'Full integration test' do
@@ -403,5 +478,4 @@ each_db_config(Deimos::Utils::DbProducer) do
                                                                 }
                                                               ])
   end
-
 end

data/spec/utils/schema_controller_mixin_spec.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require 'deimos/utils/schema_controller_mixin'
+require 'deimos/schema_backends/avro_local'
+
+RSpec.describe Deimos::Utils::SchemaControllerMixin, type: :controller do
+
+  before(:each) do
+    Deimos.configure do
+      schema.backend(:avro_local)
+    end
+  end
+
+  controller(ActionController::Base) do
+    include Deimos::Utils::SchemaControllerMixin # rubocop:disable RSpec/DescribedClass
+
+    request_namespace 'com.my-namespace.request'
+    response_namespace 'com.my-namespace.response'
+    schemas :index, :show
+    schemas :update, request: 'UpdateRequest', response: 'UpdateResponse'
+
+    # :nodoc:
+    def index
+      render_schema({ 'response_id' => payload[:request_id] + ' mom' })
+    end
+
+    # :nodoc:
+    def show
+      render_schema({ 'response_id' => payload[:request_id] + ' dad' })
+    end
+
+    # :nodoc:
+    def update
+      render_schema({ 'update_response_id' => payload[:update_request_id] + ' sis' })
+    end
+  end
+
+  it 'should render the correct response for index' do
+    request_backend = Deimos.schema_backend(schema: 'Index',
+                                            namespace: 'com.my-namespace.request')
+    response_backend = Deimos.schema_backend(schema: 'Index',
+                                             namespace: 'com.my-namespace.response')
+    request.content_type = 'avro/binary'
+    get :index, body: request_backend.encode({ 'request_id' => 'hi' })
+    expect(response_backend.decode(response.body)).to eq({ 'response_id' => 'hi mom' })
+  end
+
+  it 'should render the correct response for show' do
+    request_backend = Deimos.schema_backend(schema: 'Index',
+                                            namespace: 'com.my-namespace.request')
+    response_backend = Deimos.schema_backend(schema: 'Index',
+                                             namespace: 'com.my-namespace.response')
+    request.content_type = 'avro/binary'
+    get :show, params: { id: 1 }, body: request_backend.encode({ 'request_id' => 'hi' })
+    expect(response_backend.decode(response.body)).to eq({ 'response_id' => 'hi dad' })
+  end
+
+  it 'should render the correct response for update' do
+    request_backend = Deimos.schema_backend(schema: 'UpdateRequest',
+                                            namespace: 'com.my-namespace.request')
+    response_backend = Deimos.schema_backend(schema: 'UpdateResponse',
+                                             namespace: 'com.my-namespace.response')
+    request.content_type = 'avro/binary'
+    post :update, params: { id: 1 }, body: request_backend.encode({ 'update_request_id' => 'hi' })
+    expect(response_backend.decode(response.body)).to eq({ 'update_response_id' => 'hi sis' })
+  end
+
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.8.0.pre.beta1
+  version: 1.8.1.pre.beta4
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-07-06 00:00:00.000000000 Z
+date: 2020-08-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf
@@ -66,20 +66,6 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.0.1
-- !ruby/object:Gem::Dependency
-  name: activerecord
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.2'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.2'
 - !ruby/object:Gem::Dependency
   name: activerecord-import
   requirement: !ruby/object:Gem::Requirement
@@ -226,20 +212,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.2'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 5.2.4.2
+        version: '6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.2'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 5.2.4.2
+        version: '6'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -282,6 +262,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: rspec-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
@@ -405,7 +399,11 @@ files:
 - lib/deimos/utils/deadlock_retry.rb
 - lib/deimos/utils/inline_consumer.rb
 - lib/deimos/utils/lag_reporter.rb
+- lib/deimos/utils/schema_controller_mixin.rb
 - lib/deimos/version.rb
+- lib/generators/deimos/active_record/templates/migration.rb.tt
+- lib/generators/deimos/active_record/templates/model.rb.tt
+- lib/generators/deimos/active_record_generator.rb
 - lib/generators/deimos/db_backend/templates/migration
 - lib/generators/deimos/db_backend/templates/rails3_migration
 - lib/generators/deimos/db_backend_generator.rb
@@ -427,8 +425,10 @@ files:
 - spec/config/configuration_spec.rb
 - spec/consumer_spec.rb
 - spec/deimos_spec.rb
+- spec/generators/active_record_generator_spec.rb
 - spec/handlers/my_batch_consumer.rb
 - spec/handlers/my_consumer.rb
+- spec/kafka_listener_spec.rb
 - spec/kafka_source_spec.rb
 - spec/kafka_topic_info_spec.rb
 - spec/message_spec.rb
@@ -441,6 +441,8 @@ files:
 - spec/schema_backends/avro_schema_registry_spec.rb
 - spec/schema_backends/avro_validation_spec.rb
 - spec/schema_backends/base_spec.rb
+- spec/schemas/com/my-namespace/Generated.avsc
+- spec/schemas/com/my-namespace/MyNestedSchema.avsc
 - spec/schemas/com/my-namespace/MySchema-key.avsc
 - spec/schemas/com/my-namespace/MySchema.avsc
 - spec/schemas/com/my-namespace/MySchemaCompound-key.avsc
@@ -451,12 +453,17 @@ files:
 - spec/schemas/com/my-namespace/Wibble.avsc
 - spec/schemas/com/my-namespace/Widget.avsc
 - spec/schemas/com/my-namespace/WidgetTheSecond.avsc
+- spec/schemas/com/my-namespace/request/Index.avsc
+- spec/schemas/com/my-namespace/request/UpdateRequest.avsc
+- spec/schemas/com/my-namespace/response/Index.avsc
+- spec/schemas/com/my-namespace/response/UpdateResponse.avsc
 - spec/spec_helper.rb
 - spec/utils/db_poller_spec.rb
 - spec/utils/db_producer_spec.rb
 - spec/utils/deadlock_retry_spec.rb
 - spec/utils/lag_reporter_spec.rb
 - spec/utils/platform_schema_validation_spec.rb
+- spec/utils/schema_controller_mixin_spec.rb
 - support/deimos-solo.png
 - support/deimos-with-name-next.png
 - support/deimos-with-name.png
@@ -499,8 +506,10 @@ test_files:
 - spec/config/configuration_spec.rb
 - spec/consumer_spec.rb
 - spec/deimos_spec.rb
+- spec/generators/active_record_generator_spec.rb
 - spec/handlers/my_batch_consumer.rb
 - spec/handlers/my_consumer.rb
+- spec/kafka_listener_spec.rb
 - spec/kafka_source_spec.rb
 - spec/kafka_topic_info_spec.rb
 - spec/message_spec.rb
@@ -513,6 +522,8 @@ test_files:
 - spec/schema_backends/avro_schema_registry_spec.rb
 - spec/schema_backends/avro_validation_spec.rb
 - spec/schema_backends/base_spec.rb
+- spec/schemas/com/my-namespace/Generated.avsc
+- spec/schemas/com/my-namespace/MyNestedSchema.avsc
 - spec/schemas/com/my-namespace/MySchema-key.avsc
 - spec/schemas/com/my-namespace/MySchema.avsc
 - spec/schemas/com/my-namespace/MySchemaCompound-key.avsc
@@ -523,9 +534,14 @@ test_files:
 - spec/schemas/com/my-namespace/Wibble.avsc
 - spec/schemas/com/my-namespace/Widget.avsc
 - spec/schemas/com/my-namespace/WidgetTheSecond.avsc
+- spec/schemas/com/my-namespace/request/Index.avsc
+- spec/schemas/com/my-namespace/request/UpdateRequest.avsc
+- spec/schemas/com/my-namespace/response/Index.avsc
+- spec/schemas/com/my-namespace/response/UpdateResponse.avsc
 - spec/spec_helper.rb
 - spec/utils/db_poller_spec.rb
 - spec/utils/db_producer_spec.rb
 - spec/utils/deadlock_retry_spec.rb
 - spec/utils/lag_reporter_spec.rb
 - spec/utils/platform_schema_validation_spec.rb
+- spec/utils/schema_controller_mixin_spec.rb