deimos-ruby 1.10.2 → 1.12.0

data/spec/active_record_consumer_spec.rb

@@ -5,7 +5,6 @@ require 'date'
 # Wrapped in a module to prevent class leakage
 module ActiveRecordConsumerTest
   describe Deimos::ActiveRecordConsumer, 'Message Consumer' do
-
     before(:all) do
       ActiveRecord::Base.connection.create_table(:widgets, force: true) do |t|
         t.string(:test_id)
@@ -61,94 +60,104 @@ module ActiveRecordConsumerTest
       Time.zone = 'Eastern Time (US & Canada)'
     end
 
-    it 'should receive events correctly' do
-      travel 1.day do
-        expect(Widget.count).to eq(0)
-        test_consume_message(MyConsumer, {
-                               test_id: 'abc',
-                               some_int: 3,
-                               updated_at: 1.day.ago.to_i,
-                               some_datetime_int: Time.zone.now.to_i,
-                               timestamp: 2.minutes.ago.to_s
-                             }, { call_original: true, key: 5 })
-
-        expect(Widget.count).to eq(1)
-        widget = Widget.last
-        expect(widget.id).to eq(5)
-        expect(widget.test_id).to eq('abc')
-        expect(widget.some_int).to eq(3)
-        expect(widget.some_datetime_int).to eq(Time.zone.now)
-        expect(widget.some_bool).to eq(false)
-        expect(widget.updated_at).to eq(Time.zone.now)
-
-        # test unscoped
-        widget.update_attribute(:some_bool, true)
-
-        # test update
-        test_consume_message(MyConsumer, {
-                               test_id: 'abcd',
-                               some_int: 3,
-                               some_datetime_int: Time.zone.now.to_i,
-                               timestamp: 2.minutes.ago.to_s
-                             }, { call_original: true, key: 5 })
-        expect(Widget.unscoped.count).to eq(1)
-        widget = Widget.unscoped.last
-        expect(widget.id).to eq(5)
-        expect(widget.test_id).to eq('abcd')
-        expect(widget.some_int).to eq(3)
-
-        # test delete
-        test_consume_message(MyConsumer, nil, call_original: true, key: 5)
-        expect(Widget.count).to eq(0)
-
+    describe 'consume' do
+      SCHEMA_CLASS_SETTINGS.each do |setting, use_schema_classes|
+        context "with Schema Class consumption #{setting}" do
+          before(:each) do
+            Deimos.configure { |config| config.schema.use_schema_classes = use_schema_classes }
+          end
+
+          it 'should receive events correctly' do
+            travel 1.day do
+              expect(Widget.count).to eq(0)
+              test_consume_message(MyConsumer, {
+                                     test_id: 'abc',
+                                     some_int: 3,
+                                     updated_at: 1.day.ago.to_i,
+                                     some_datetime_int: Time.zone.now.to_i,
+                                     timestamp: 2.minutes.ago.to_s
+                                   }, { call_original: true, key: 5 })
+
+              expect(Widget.count).to eq(1)
+              widget = Widget.last
+              expect(widget.id).to eq(5)
+              expect(widget.test_id).to eq('abc')
+              expect(widget.some_int).to eq(3)
+              expect(widget.some_datetime_int).to eq(Time.zone.now)
+              expect(widget.some_bool).to eq(false)
+              expect(widget.updated_at).to eq(Time.zone.now)
+
+              # test unscoped
+              widget.update_attribute(:some_bool, true)
+
+              # test update
+              test_consume_message(MyConsumer, {
+                                     test_id: 'abcd',
+                                     some_int: 3,
+                                     some_datetime_int: Time.zone.now.to_i,
+                                     timestamp: 2.minutes.ago.to_s
+                                   }, { call_original: true, key: 5 })
+              expect(Widget.unscoped.count).to eq(1)
+              widget = Widget.unscoped.last
+              expect(widget.id).to eq(5)
+              expect(widget.test_id).to eq('abcd')
+              expect(widget.some_int).to eq(3)
+
+              # test delete
+              test_consume_message(MyConsumer, nil, call_original: true, key: 5)
+              expect(Widget.count).to eq(0)
+
+            end
+
+          end
+
+          it 'should update only updated_at' do
+            travel_to Time.local(2020, 5, 5, 5, 5, 5)
+            widget1 = Widget.create!(test_id: 'id1', some_int: 3)
+            expect(widget1.updated_at.in_time_zone).to eq(Time.local(2020, 5, 5, 5, 5, 5))
+
+            travel 1.day
+            test_consume_message(MyCustomFetchConsumer, {
+                                   test_id: 'id1',
+                                   some_int: 3
+                                 }, { call_original: true })
+            expect(widget1.reload.updated_at.in_time_zone).
+              to eq(Time.local(2020, 5, 6, 5, 5, 5))
+            travel_back
+          end
+
+          it 'should find widgets by custom logic' do
+            widget1 = Widget.create!(test_id: 'id1')
+            expect(widget1.some_int).to be_nil
+            test_consume_message(MyCustomFetchConsumer, {
+                                   test_id: 'id1',
+                                   some_int: 3
+                                 }, { call_original: true })
+            expect(widget1.reload.some_int).to eq(3)
+            expect(Widget.count).to eq(1)
+            test_consume_message(MyCustomFetchConsumer, {
+                                   test_id: 'id2',
+                                   some_int: 4
+                                 }, { call_original: true })
+            expect(Widget.count).to eq(2)
+            expect(Widget.find_by_test_id('id1').some_int).to eq(3)
+            expect(Widget.find_by_test_id('id2').some_int).to eq(4)
+          end
+
+          it 'should not create record of process_message returns false' do
+            allow_any_instance_of(MyConsumer).to receive(:process_message?).and_return(false)
+            expect(Widget.count).to eq(0)
+            test_consume_message(MyConsumer, {
+                                   test_id: 'abc',
+                                   some_int: 3,
+                                   updated_at: 1.day.ago.to_i,
+                                   some_datetime_int: Time.zone.now.to_i,
+                                   timestamp: 2.minutes.ago.to_s
+                                 }, { call_original: true, key: 5 })
+            expect(Widget.count).to eq(0)
+          end
+        end
       end
-
-    end
-
-    it 'should update only updated_at' do
-      travel_to Time.local(2020, 5, 5, 5, 5, 5)
-      widget1 = Widget.create!(test_id: 'id1', some_int: 3)
-      expect(widget1.updated_at.in_time_zone).to eq(Time.local(2020, 5, 5, 5, 5, 5))
-
-      travel 1.day
-      test_consume_message(MyCustomFetchConsumer, {
-                             test_id: 'id1',
-                             some_int: 3
-                           }, { call_original: true })
-      expect(widget1.reload.updated_at.in_time_zone).
-        to eq(Time.local(2020, 5, 6, 5, 5, 5))
-      travel_back
-    end
-
-    it 'should find widgets by custom logic' do
-      widget1 = Widget.create!(test_id: 'id1')
-      expect(widget1.some_int).to be_nil
-      test_consume_message(MyCustomFetchConsumer, {
-                             test_id: 'id1',
-                             some_int: 3
-                           }, { call_original: true })
-      expect(widget1.reload.some_int).to eq(3)
-      expect(Widget.count).to eq(1)
-      test_consume_message(MyCustomFetchConsumer, {
-                             test_id: 'id2',
-                             some_int: 4
-                           }, { call_original: true })
-      expect(Widget.count).to eq(2)
-      expect(Widget.find_by_test_id('id1').some_int).to eq(3)
-      expect(Widget.find_by_test_id('id2').some_int).to eq(4)
-    end
-
-    it 'should not create record of process_message returns false' do
-      MyConsumer.any_instance.stub(:process_message?).and_return(false)
-      expect(Widget.count).to eq(0)
-      test_consume_message(MyConsumer, {
-                             test_id: 'abc',
-                             some_int: 3,
-                             updated_at: 1.day.ago.to_i,
-                             some_datetime_int: Time.zone.now.to_i,
-                             timestamp: 2.minutes.ago.to_s
-                           }, { call_original: true, key: 5 })
-      expect(Widget.count).to eq(0)
     end
   end
 end

data/spec/active_record_producer_spec.rb

@@ -47,35 +47,46 @@ describe Deimos::ActiveRecordProducer do
     stub_const('MyProducerWithUniqueID', producer_class)
   end
 
-  it 'should send events correctly' do
-    MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 3))
-    expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
-  end
+  describe 'produce' do
+    SCHEMA_CLASS_SETTINGS.each do |setting, use_schema_classes|
+      context "with Schema Class consumption #{setting}" do
+        before(:each) do
+          Deimos.configure { |config| config.schema.use_schema_classes = use_schema_classes }
+        end
 
-  it 'should coerce values' do
-    MyProducer.send_event(Widget.new(test_id: 'abc', some_int: '3'))
-    MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 4.5))
-    expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
-    expect('my-topic').to have_sent(test_id: 'abc', some_int: 4)
-    expect {
-      MyProducer.send_event(Widget.new(test_id: 'abc', some_int: nil))
-    }.to raise_error(Avro::SchemaValidator::ValidationError)
-
-    MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: nil))
-    MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: true))
-    expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: false)
-    expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: true)
-  end
+        it 'should send events correctly' do
+          MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 3))
+          expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
+        end
+
+        it 'should coerce values' do
+          MyProducer.send_event(Widget.new(test_id: 'abc', some_int: '3'))
+          MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 4.5))
+          expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
+          expect('my-topic').to have_sent(test_id: 'abc', some_int: 4)
+          expect {
+            MyProducer.send_event(Widget.new(test_id: 'abc', some_int: nil))
+          }.to raise_error(Avro::SchemaValidator::ValidationError)
+
+          MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: nil))
+          MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: true))
+          expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: false)
+          expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: true)
+        end
 
-  it 'should be able to call the record' do
-    widget = Widget.create!(test_id: 'abc2', some_int: 3)
-    MyProducerWithID.send_event(id: widget.id, test_id: 'abc2', some_int: 3)
-    expect('my-topic-with-id').to have_sent(
-      test_id: 'abc2',
-      some_int: 3,
-      message_id: 'generated_id',
-      timestamp: anything
-    )
+        it 'should be able to call the record' do
+          widget = Widget.create!(test_id: 'abc2', some_int: 3)
+          MyProducerWithID.send_event(id: widget.id, test_id: 'abc2', some_int: 3)
+          expect('my-topic-with-id').to have_sent(
+            test_id: 'abc2',
+            some_int: 3,
+            message_id: 'generated_id',
+            timestamp: anything
+          )
+        end
+
+      end
+    end
   end
 
   specify '#watched_attributes' do

data/spec/batch_consumer_spec.rb

@@ -3,7 +3,6 @@
 # :nodoc:
 module ConsumerTest
   describe Deimos::Consumer, 'Batch Consumer' do
-
     prepend_before(:each) do
       # :nodoc:
       consumer_class = Class.new(described_class) do
@@ -30,40 +29,50 @@ module ConsumerTest
       batch.concat([{ 'invalid' => 'key' }])
     end
 
-    it 'should provide backwards compatibility for BatchConsumer class' do
-      consumer_class = Class.new(Deimos::BatchConsumer) do
-        schema 'MySchema'
-        namespace 'com.my-namespace'
-        key_config field: 'test_id'
+    describe 'consume_batch' do
+      SCHEMA_CLASS_SETTINGS.each do |setting, use_schema_classes|
+        context "with Schema Class consumption #{setting}" do
+          before(:each) do
+            Deimos.configure { |config| config.schema.use_schema_classes = use_schema_classes }
+          end
 
-        # :nodoc:
-        def consume_batch(_payloads, _metadata)
-          raise 'This should not be called unless call_original is set'
-        end
-      end
-      stub_const('ConsumerTest::MyOldBatchConsumer', consumer_class)
+          it 'should provide backwards compatibility for BatchConsumer class' do
+            consumer_class = Class.new(Deimos::BatchConsumer) do
+              schema 'MySchema'
+              namespace 'com.my-namespace'
+              key_config field: 'test_id'
+
+              # :nodoc:
+              def consume_batch(_payloads, _metadata)
+                raise 'This should not be called unless call_original is set'
+              end
+            end
+            stub_const('ConsumerTest::MyOldBatchConsumer', consumer_class)
+
+            test_consume_batch(MyOldBatchConsumer, batch) do |received, _metadata|
+              expect(received).to eq(batch)
+            end
+          end
 
-      test_consume_batch(MyOldBatchConsumer, batch) do |received, _metadata|
-        expect(received).to eq(batch)
-      end
-    end
+          it 'should consume a batch of messages' do
+            test_consume_batch(MyBatchConsumer, batch) do |received, _metadata|
+              expect(received).to eq(batch)
+            end
+          end
 
-    it 'should consume a batch of messages' do
-      test_consume_batch(MyBatchConsumer, batch) do |received, _metadata|
-        expect(received).to eq(batch)
-      end
-    end
+          it 'should consume a message on a topic' do
+            test_consume_batch('my_batch_consume_topic', batch) do |received, _metadata|
+              expect(received).to eq(batch)
+            end
+          end
 
-    it 'should consume a message on a topic' do
-      test_consume_batch('my_batch_consume_topic', batch) do |received, _metadata|
-        expect(received).to eq(batch)
+          it 'should fail on an invalid message in the batch' do
+            test_consume_batch_invalid_message(MyBatchConsumer, batch.concat(invalid_payloads))
+          end
+        end
       end
     end
 
-    it 'should fail on an invalid message in the batch' do
-      test_consume_batch_invalid_message(MyBatchConsumer, batch.concat(invalid_payloads))
-    end
-
     describe 'when reraising errors is disabled' do
       before(:each) do
         Deimos.configure { |config| config.consumers.reraise_errors = false }

data/spec/config/configuration_spec.rb

@@ -21,6 +21,7 @@ describe Deimos, 'configuration' do
       kafka_logger logger
       reraise_consumer_errors true
       schema_registry_url 'http://schema.registry'
+      schema.use_schema_classes false
       seed_broker 'whatever'
       schema_path 'some_path'
       producer_schema_namespace 'namespace'
@@ -37,6 +38,7 @@ describe Deimos, 'configuration' do
     expect(described_class.config.kafka.logger).to eq(logger)
     expect(described_class.config.consumers.reraise_errors).to eq(true)
     expect(described_class.config.schema.registry_url).to eq('http://schema.registry')
+    expect(described_class.config.schema.use_schema_classes).to eq(false)
     expect(described_class.config.kafka.seed_brokers).to eq('whatever')
     expect(described_class.config.producers.schema_namespace).to eq('namespace')
     expect(described_class.config.producers.topic_prefix).to eq('prefix')
@@ -86,7 +88,8 @@ describe Deimos, 'configuration' do
           offset_commit_threshold: 0,
           offset_retention_time: nil,
           heartbeat_interval: 10,
-          handler: 'ConsumerTest::MyConsumer'
+          handler: 'ConsumerTest::MyConsumer',
+          use_schema_classes: nil
         }, {
           topic: 'my_batch_consume_topic',
           group_id: 'my_batch_group_id',
@@ -102,7 +105,8 @@ describe Deimos, 'configuration' do
           offset_commit_threshold: 0,
           offset_retention_time: nil,
           heartbeat_interval: 10,
-          handler: 'ConsumerTest::MyBatchConsumer'
+          handler: 'ConsumerTest::MyBatchConsumer',
+          use_schema_classes: nil
         }
       ],
       producer: {
@@ -178,6 +182,7 @@ describe Deimos, 'configuration' do
         offset_commit_threshold 13
         offset_retention_time 13
         heartbeat_interval 13
+        use_schema_classes false
       end
       consumer do
         disabled true
@@ -185,6 +190,7 @@ describe Deimos, 'configuration' do
         schema 'blah2'
         topic 'blah2'
         group_id 'myconsumerid2'
+        use_schema_classes false
       end
     end
 
@@ -227,7 +233,8 @@ describe Deimos, 'configuration' do
             offset_commit_threshold: 13,
             offset_retention_time: 13,
             heartbeat_interval: 13,
-            handler: 'MyConfigConsumer'
+            handler: 'MyConfigConsumer',
+            use_schema_classes: false
           }
         ],
         producer: {

data/spec/consumer_spec.rb

@@ -3,7 +3,6 @@
 # :nodoc:
 module ConsumerTest
   describe Deimos::Consumer, 'Message Consumer' do
-
     prepend_before(:each) do
       # :nodoc:
       consumer_class = Class.new(described_class) do
@@ -24,98 +23,110 @@ module ConsumerTest
       stub_const('ConsumerTest::MyConsumer', consumer_class)
     end
 
-    it 'should consume a message' do
-      test_consume_message(MyConsumer,
-                           'test_id' => 'foo',
-                           'some_int' => 123) do |payload, _metadata|
-                             expect(payload['test_id']).to eq('foo')
-                           end
-    end
+    describe 'consume' do
+      SCHEMA_CLASS_SETTINGS.each do |setting, use_schema_classes|
+        context "with Schema Class consumption #{setting}" do
+          before(:each) do
+            Deimos.configure { |config| config.schema.use_schema_classes = use_schema_classes }
+          end
 
-    it 'should consume a nil message' do
-      test_consume_message(MyConsumer, nil) do |payload, _metadata|
-        expect(payload).to be_nil
-      end
-    end
+          it 'should consume a message' do
+            test_consume_message(MyConsumer,
+                                 'test_id' => 'foo',
+                                 'some_int' => 123) do |payload, _metadata|
+                                   expect(payload['test_id']).to eq('foo')
+                                   expect(payload['some_int']).to eq(123)
+                                 end
+          end
 
-    it 'should consume a message idempotently' do
-      # testing for a crash and re-consuming the same message/metadata
-      key = { 'test_id' => 'foo' }
-      test_metadata = { key: key }
-      allow_any_instance_of(MyConsumer).to(receive(:decode_key)) do |_instance, k|
-        k['test_id']
-      end
-      MyConsumer.new.around_consume({ 'test_id' => 'foo',
-                                      'some_int' => 123 }, test_metadata) do |_payload, metadata|
-                                        expect(metadata[:key]).to eq('foo')
-                                      end
-      MyConsumer.new.around_consume({ 'test_id' => 'foo',
-                                      'some_int' => 123 }, test_metadata) do |_payload, metadata|
-                                        expect(metadata[:key]).to eq('foo')
-                                      end
-    end
+          it 'should consume a nil message' do
+            test_consume_message(MyConsumer, nil) do |payload, _metadata|
+              expect(payload).to be_nil
+            end
+          end
 
-    it 'should consume a message on a topic' do
-      test_consume_message('my_consume_topic',
-                           'test_id' => 'foo',
-                           'some_int' => 123) do |payload, _metadata|
-                             expect(payload['test_id']).to eq('foo')
-                           end
-    end
+          it 'should consume a message idempotently' do
+            # testing for a crash and re-consuming the same message/metadata
+            key = { 'test_id' => 'foo' }
+            test_metadata = { key: key }
+            allow_any_instance_of(MyConsumer).to(receive(:decode_key)) do |_instance, k|
+              k['test_id']
+            end
+            MyConsumer.new.around_consume({ 'test_id' => 'foo',
+                                            'some_int' => 123 }, test_metadata) do |_payload, metadata|
+                                              expect(metadata[:key]).to eq('foo')
+                                            end
+            MyConsumer.new.around_consume({ 'test_id' => 'foo',
+                                            'some_int' => 123 }, test_metadata) do |_payload, metadata|
+                                              expect(metadata[:key]).to eq('foo')
+                                            end
+          end
 
-    it 'should fail on invalid message' do
-      test_consume_invalid_message(MyConsumer, 'invalid' => 'key')
-    end
+          it 'should consume a message on a topic' do
+            test_consume_message('my_consume_topic',
+                                 'test_id' => 'foo',
+                                 'some_int' => 123) do |payload, _metadata|
+                                   expect(payload['test_id']).to eq('foo')
+                                   expect(payload['some_int']).to eq(123)
+                                 end
+          end
 
-    it 'should fail if reraise is false but fatal_error is true' do
-      Deimos.configure { |config| config.consumers.reraise_errors = false }
-      test_consume_invalid_message(MyConsumer, 'fatal')
-    end
+          it 'should fail on invalid message' do
+            test_consume_invalid_message(MyConsumer, 'invalid' => 'key')
+          end
 
-    it 'should fail if fatal_error is true globally' do
-      Deimos.configure do |config|
-        config.consumers.fatal_error = proc { true }
-        config.consumers.reraise_errors = false
-      end
-      test_consume_invalid_message(MyConsumer, 'invalid' => 'key')
-    end
+          it 'should fail if reraise is false but fatal_error is true' do
+            Deimos.configure { |config| config.consumers.reraise_errors = false }
+            test_consume_invalid_message(MyConsumer, 'fatal')
+          end
 
-    it 'should fail on message with extra fields' do
-      test_consume_invalid_message(MyConsumer,
-                                   'test_id' => 'foo',
-                                   'some_int' => 123,
-                                   'extra_field' => 'field name')
-    end
+          it 'should fail if fatal_error is true globally' do
+            Deimos.configure do |config|
+              config.consumers.fatal_error = proc { true }
+              config.consumers.reraise_errors = false
+            end
+            test_consume_invalid_message(MyConsumer, 'invalid' => 'key')
+          end
 
-    it 'should not fail when before_consume fails without reraising errors' do
-      Deimos.configure { |config| config.consumers.reraise_errors = false }
-      expect {
-        test_consume_message(
-          MyConsumer,
-          { 'test_id' => 'foo',
-            'some_int' => 123 },
-          { skip_expectation: true }
-        ) { raise 'OH NOES' }
-      }.not_to raise_error
-    end
+          it 'should fail on message with extra fields' do
+            test_consume_invalid_message(MyConsumer,
+                                         'test_id' => 'foo',
+                                         'some_int' => 123,
+                                         'extra_field' => 'field name')
+          end
 
-    it 'should not fail when consume fails without reraising errors' do
-      Deimos.configure { |config| config.consumers.reraise_errors = false }
-      expect {
-        test_consume_message(
-          MyConsumer,
-          { 'invalid' => 'key' },
-          { skip_expectation: true }
-        )
-      }.not_to raise_error
-    end
+          it 'should not fail when before_consume fails without reraising errors' do
+            Deimos.configure { |config| config.consumers.reraise_errors = false }
+            expect {
+              test_consume_message(
+                MyConsumer,
+                { 'test_id' => 'foo',
+                  'some_int' => 123 },
+                { skip_expectation: true }
+              ) { raise 'OH NOES' }
+            }.not_to raise_error
+          end
+
+          it 'should not fail when consume fails without reraising errors' do
+            Deimos.configure { |config| config.consumers.reraise_errors = false }
+            expect {
+              test_consume_message(
+                MyConsumer,
+                { 'invalid' => 'key' },
+                { skip_expectation: true }
+              )
+            }.not_to raise_error
+          end
 
-    it 'should call original' do
-      expect {
-        test_consume_message(MyConsumer,
-                             { 'test_id' => 'foo', 'some_int' => 123 },
-                             { call_original: true })
-      }.to raise_error('This should not be called unless call_original is set')
+          it 'should call original' do
+            expect {
+              test_consume_message(MyConsumer,
+                                   { 'test_id' => 'foo', 'some_int' => 123 },
+                                   { call_original: true })
+            }.to raise_error('This should not be called unless call_original is set')
+          end
+        end
+      end
     end
 
     describe 'decode_key' do

data/spec/generators/active_record_generator_spec.rb

@@ -28,6 +28,7 @@ RSpec.describe Deimos::Generators::ActiveRecordGenerator do
             t.bigint :a_long
             t.float :a_float
             t.float :a_double
+            t.string :an_optional_int
             t.string :an_enum
             t.json :an_array
             t.json :a_map