deimos-ruby 1.6.3 → 1.8.1.pre.beta1

data/spec/active_record_consume/batch_slicer_spec.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+RSpec.describe Deimos::ActiveRecordConsume::BatchSlicer do
+  describe '#slice' do
+    let(:batch) do
+      [
+        Deimos::Message.new({ v: 1 }, nil, key: 'C'),
+        Deimos::Message.new({ v: 123 }, nil, key: 'A'),
+        Deimos::Message.new({ v: 999 }, nil, key: 'B'),
+        Deimos::Message.new({ v: 456 }, nil, key: 'A'),
+        Deimos::Message.new({ v: 2 }, nil, key: 'C'),
+        Deimos::Message.new({ v: 3 }, nil, key: 'C')
+      ]
+    end
+
+    it 'should slice a batch by key' do
+      slices = described_class.slice(batch)
+
+      expect(slices).
+        to match([
+                   match_array([
+                                 Deimos::Message.new({ v: 1 }, nil, key: 'C'),
+                                 Deimos::Message.new({ v: 123 }, nil, key: 'A'),
+                                 Deimos::Message.new({ v: 999 }, nil, key: 'B')
+                               ]),
+                   match_array([
+                                 Deimos::Message.new({ v: 456 }, nil, key: 'A'),
+                                 Deimos::Message.new({ v: 2 }, nil, key: 'C')
+                               ]),
+                   match_array([
+                                 Deimos::Message.new({ v: 3 }, nil, key: 'C')
+                               ])
+                 ])
+    end
+
+    it 'should handle empty batches' do
+      slices = described_class.slice([])
+
+      expect(slices).to be_empty
+    end
+  end
+end

data/spec/active_record_consume/schema_model_converter_spec.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require 'deimos/active_record_consume/schema_model_converter'
+require 'deimos/schema_backends/avro_local'
+
+# Wrapped in a module to prevent class leakage
+module SchemaModelConverterTest
+  describe Deimos::ActiveRecordConsume::SchemaModelConverter do
+    # Create ActiveRecord table and model
+    before(:all) do
+      ActiveRecord::Base.connection.create_table(:wibbles, force: true) do |t|
+        t.integer(:wibble_id)
+        t.string(:name)
+        t.integer(:bar)
+        t.datetime(:birthday_int)
+        t.datetime(:birthday_long)
+        t.datetime(:birthday_optional)
+        t.timestamps
+      end
+
+      # :nodoc:
+      class Wibble < ActiveRecord::Base
+      end
+      Wibble.reset_column_information
+    end
+
+    after(:all) do
+      ActiveRecord::Base.connection.drop_table(:wibbles)
+    end
+
+    let(:schema) { Deimos::SchemaBackends::AvroLocal.new(schema: 'Wibble', namespace: 'com.my-namespace') }
+    let(:inst) { described_class.new(schema, Wibble) }
+
+    describe '#convert' do
+      it 'should extract attributes from the payload' do
+        payload = { 'id' => 123, 'wibble_id' => 456, 'name' => 'wibble' }
+
+        expect(inst.convert(payload)).to include('id' => 123, 'wibble_id' => 456, 'name' => 'wibble')
+      end
+
+      it 'should ignore payload fields that don\'t exist' do
+        payload = { 'foo' => 'abc' }
+
+        expect(inst.convert(payload)).not_to include('foo')
+      end
+
+      it 'should ignore fields in the schema but not in the model' do
+        payload = { 'floop' => 'def' }
+
+        expect(inst.convert(payload)).not_to include('floop')
+      end
+
+      it 'should ignore model fields not in the schema' do
+        payload = { 'bar' => 'xyz' }
+
+        expect(inst.convert(payload)).not_to include('bar')
+      end
+
+      it 'should handle nils' do
+        payload = { 'name' => nil }
+
+        expect(inst.convert(payload)['name']).to be_nil
+      end
+
+      describe 'timestamps' do
+        it 'should ignore AR timestamp fields' do
+          payload = { 'updated_at' => '1234567890', 'created_at' => '2345678901' }
+
+          expect(inst.convert(payload)).not_to include('updated_at', 'created_at')
+        end
+
+        it 'should parse timestamps' do
+          first = Time.zone.local(2019, 1, 1, 11, 12, 13)
+          second = Time.zone.local(2019, 2, 2, 12, 13, 14)
+
+          payload = { 'birthday_int' => first.to_i, 'birthday_long' => second.to_i }
+
+          expect(inst.convert(payload)).to include('birthday_int' => first, 'birthday_long' => second)
+        end
+
+        it 'should parse integer-string timestamps' do
+          date = Time.zone.local(2019, 1, 1, 11, 12, 13)
+
+          payload = { 'birthday_int' => '1546359133' }
+
+          expect(inst.convert(payload)).to include('birthday_int' => date)
+        end
+
+        it 'should ignore other strings for timestamps' do
+          payload = { 'birthday_int' => 'some-other-val' }
+
+          expect(inst.convert(payload)).to include('birthday_int' => 'some-other-val')
+        end
+      end
+
+      it 'should coerce nullable unions' do
+        date = Time.zone.local(2019, 1, 1, 11, 12, 13)
+
+        payload = { 'birthday_optional' => date.to_i }
+
+        expect(inst.convert(payload)).to include('birthday_optional' => date)
+      end
+    end
+  end
+end

data/spec/active_record_consumer_spec.rb

@@ -2,9 +2,9 @@
 
 require 'date'
 
-# :nodoc:
-module ActiveRecordProducerTest
-  describe Deimos::ActiveRecordConsumer do
+# Wrapped in a module to prevent class leakage
+module ActiveRecordConsumerTest
+  describe Deimos::ActiveRecordConsumer, 'Message Consumer' do
 
     before(:all) do
       ActiveRecord::Base.connection.create_table(:widgets, force: true) do |t|
@@ -137,13 +137,5 @@ module ActiveRecordProducerTest
       expect(Widget.find_by_test_id('id1').some_int).to eq(3)
       expect(Widget.find_by_test_id('id2').some_int).to eq(4)
     end
-
-    it 'should coerce int values to datetimes' do
-      column = Widget.columns.find { |c| c.name == 'some_datetime_int' }
-      expect(MyConsumer.new.send(:_coerce_field, column, 1_579_046_400)).to eq('2020-01-14 19:00:00 -0500')
-      expect(MyConsumer.new.send(:_coerce_field, column, '1579046400')).to eq('2020-01-14 19:00:00 -0500')
-      expect(MyConsumer.new.send(:_coerce_field, column, 'some-other-val')).to eq('some-other-val')
-    end
-
   end
 end

data/spec/active_record_producer_spec.rb

@@ -1,107 +1,85 @@
 # frozen_string_literal: true
 
-# :nodoc:
-module ActiveRecordProducerTest
-  describe Deimos::ActiveRecordProducer do
+describe Deimos::ActiveRecordProducer do
 
-    before(:all) do
-      ActiveRecord::Base.connection.create_table(:widgets) do |t|
-        t.string(:test_id)
-        t.integer(:some_int)
-        t.boolean(:some_bool)
-        t.timestamps
-      end
+  include_context 'with widgets'
 
-      # :nodoc:
-      class Widget < ActiveRecord::Base
-        # @return [String]
-        def generated_id
-          'generated_id'
-        end
-      end
-    end
+  prepend_before(:each) do
 
-    after(:all) do
-      ActiveRecord::Base.connection.drop_table(:widgets)
+    producer_class = Class.new(Deimos::ActiveRecordProducer) do
+      schema 'MySchema'
+      namespace 'com.my-namespace'
+      topic 'my-topic'
+      key_config none: true
     end
+    stub_const('MyProducer', producer_class)
 
-    prepend_before(:each) do
-
-      producer_class = Class.new(Deimos::ActiveRecordProducer) do
-        schema 'MySchema'
-        namespace 'com.my-namespace'
-        topic 'my-topic'
-        key_config none: true
-      end
-      stub_const('MyProducer', producer_class)
-
-      producer_class = Class.new(Deimos::ActiveRecordProducer) do
-        schema 'MySchemaWithBooleans'
-        namespace 'com.my-namespace'
-        topic 'my-topic-with-boolean'
-        key_config none: true
-      end
-      stub_const('MyBooleanProducer', producer_class)
-
-      producer_class = Class.new(Deimos::ActiveRecordProducer) do
-        schema 'MySchemaWithId'
-        namespace 'com.my-namespace'
-        topic 'my-topic-with-id'
-        key_config none: true
-        record_class Widget
+    producer_class = Class.new(Deimos::ActiveRecordProducer) do
+      schema 'MySchemaWithBooleans'
+      namespace 'com.my-namespace'
+      topic 'my-topic-with-boolean'
+      key_config none: true
+    end
+    stub_const('MyBooleanProducer', producer_class)
 
-        # :nodoc:
-        def self.generate_payload(attrs, widget)
-          super.merge(message_id: widget.generated_id)
-        end
+    producer_class = Class.new(Deimos::ActiveRecordProducer) do
+      schema 'MySchemaWithId'
+      namespace 'com.my-namespace'
+      topic 'my-topic-with-id'
+      key_config none: true
+      record_class Widget
 
+      # :nodoc:
+      def self.generate_payload(attrs, widget)
+        super.merge(message_id: widget.generated_id)
       end
-      stub_const('MyProducerWithID', producer_class)
 
-      producer_class = Class.new(Deimos::ActiveRecordProducer) do
-        schema 'MySchemaWithUniqueId'
-        namespace 'com.my-namespace'
-        topic 'my-topic-with-unique-id'
-        key_config field: :id
-        record_class Widget
-      end
-      stub_const('MyProducerWithUniqueID', producer_class)
     end
-
-    it 'should send events correctly' do
-      MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 3))
-      expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
+    stub_const('MyProducerWithID', producer_class)
+
+    producer_class = Class.new(Deimos::ActiveRecordProducer) do
+      schema 'MySchemaWithUniqueId'
+      namespace 'com.my-namespace'
+      topic 'my-topic-with-unique-id'
+      key_config field: :id
+      record_class Widget
     end
+    stub_const('MyProducerWithUniqueID', producer_class)
+  end
 
-    it 'should coerce values' do
-      MyProducer.send_event(Widget.new(test_id: 'abc', some_int: '3'))
-      MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 4.5))
-      expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
-      expect('my-topic').to have_sent(test_id: 'abc', some_int: 4)
-      expect {
-        MyProducer.send_event(Widget.new(test_id: 'abc', some_int: nil))
-      }.to raise_error(Avro::SchemaValidator::ValidationError)
-
-      MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: nil))
-      MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: true))
-      expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: false)
-      expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: true)
-    end
+  it 'should send events correctly' do
+    MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 3))
+    expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
+  end
 
-    it 'should be able to call the record' do
-      widget = Widget.create!(test_id: 'abc2', some_int: 3)
-      MyProducerWithID.send_event(id: widget.id, test_id: 'abc2', some_int: 3)
-      expect('my-topic-with-id').to have_sent(
-        test_id: 'abc2',
-        some_int: 3,
-        message_id: 'generated_id',
-        timestamp: anything
-      )
-    end
+  it 'should coerce values' do
+    MyProducer.send_event(Widget.new(test_id: 'abc', some_int: '3'))
+    MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 4.5))
+    expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
+    expect('my-topic').to have_sent(test_id: 'abc', some_int: 4)
+    expect {
+      MyProducer.send_event(Widget.new(test_id: 'abc', some_int: nil))
+    }.to raise_error(Avro::SchemaValidator::ValidationError)
+
+    MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: nil))
+    MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: true))
+    expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: false)
+    expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: true)
+  end
 
-    specify '#watched_attributes' do
-      expect(MyProducer.watched_attributes).to eq(%w(test_id some_int))
-    end
+  it 'should be able to call the record' do
+    widget = Widget.create!(test_id: 'abc2', some_int: 3)
+    MyProducerWithID.send_event(id: widget.id, test_id: 'abc2', some_int: 3)
+    expect('my-topic-with-id').to have_sent(
+      test_id: 'abc2',
+      some_int: 3,
+      message_id: 'generated_id',
+      timestamp: anything
+    )
+  end
 
+  specify '#watched_attributes' do
+    expect(MyProducer.watched_attributes).to eq(%w(test_id some_int))
   end
+
 end

data/spec/batch_consumer_spec.rb

@@ -1,14 +1,12 @@
 # frozen_string_literal: true
 
-require 'deimos/batch_consumer'
-
 # :nodoc:
 module ConsumerTest
-  describe Deimos::BatchConsumer do
+  describe Deimos::Consumer, 'Batch Consumer' do
 
     prepend_before(:each) do
       # :nodoc:
-      consumer_class = Class.new(Deimos::BatchConsumer) do
+      consumer_class = Class.new(described_class) do
         schema 'MySchema'
         namespace 'com.my-namespace'
         key_config field: 'test_id'
@@ -32,6 +30,24 @@ module ConsumerTest
       batch.concat([{ 'invalid' => 'key' }])
     end
 
+    it 'should provide backwards compatibility for BatchConsumer class' do
+      consumer_class = Class.new(Deimos::BatchConsumer) do
+        schema 'MySchema'
+        namespace 'com.my-namespace'
+        key_config field: 'test_id'
+
+        # :nodoc:
+        def consume_batch(_payloads, _metadata)
+          raise 'This should not be called unless call_original is set'
+        end
+      end
+      stub_const('ConsumerTest::MyOldBatchConsumer', consumer_class)
+
+      test_consume_batch(MyOldBatchConsumer, batch) do |received, _metadata|
+        expect(received).to eq(batch)
+      end
+    end
+
     it 'should consume a batch of messages' do
       test_consume_batch(MyBatchConsumer, batch) do |received, _metadata|
         expect(received).to eq(batch)
@@ -95,11 +111,12 @@ module ConsumerTest
         test_consume_batch('my_batch_consume_topic', batch, keys: keys) do |_received, metadata|
           # Mock decode_key extracts the value of the first field as the key
           expect(metadata[:keys]).to eq(%w(foo bar))
+          expect(metadata[:first_offset]).to eq(1)
         end
       end
 
       it 'should decode plain keys for all messages in the batch' do
-        consumer_class = Class.new(Deimos::BatchConsumer) do
+        consumer_class = Class.new(described_class) do
           schema 'MySchema'
           namespace 'com.my-namespace'
           key_config plain: true
@@ -115,7 +132,7 @@ module ConsumerTest
     describe 'timestamps' do
       before(:each) do
         # :nodoc:
-        consumer_class = Class.new(Deimos::BatchConsumer) do
+        consumer_class = Class.new(described_class) do
           schema 'MySchemaWithDateTimes'
           namespace 'com.my-namespace'
           key_config plain: true
@@ -197,7 +214,7 @@ module ConsumerTest
     describe 'logging' do
       before(:each) do
         # :nodoc:
-        consumer_class = Class.new(Deimos::BatchConsumer) do
+        consumer_class = Class.new(described_class) do
           schema 'MySchemaWithUniqueId'
           namespace 'com.my-namespace'
           key_config plain: true

data/spec/config/configuration_spec.rb

@@ -1,6 +1,10 @@
 # frozen_string_literal: true
 
+# Mock consumer
 class MyConfigConsumer < Deimos::Consumer
+  # :no-doc:
+  def consume
+  end
 end
 describe Deimos, 'configuration' do
   it 'should configure with deprecated fields' do

data/spec/consumer_spec.rb

@@ -2,11 +2,11 @@
 
 # :nodoc:
 module ConsumerTest
-  describe Deimos::Consumer do
+  describe Deimos::Consumer, 'Message Consumer' do
 
     prepend_before(:each) do
       # :nodoc:
-      consumer_class = Class.new(Deimos::Consumer) do
+      consumer_class = Class.new(described_class) do
         schema 'MySchema'
         namespace 'com.my-namespace'
         key_config field: 'test_id'
@@ -90,7 +90,7 @@ module ConsumerTest
             'some_int' => 123 },
           { skip_expectation: true }
         ) { raise 'OH NOES' }
-      } .not_to raise_error
+      }.not_to raise_error
     end
 
     it 'should not fail when consume fails without reraising errors' do
@@ -101,7 +101,7 @@ module ConsumerTest
           { 'invalid' => 'key' },
           { skip_expectation: true }
         )
-      } .not_to raise_error
+      }.not_to raise_error
     end
 
     it 'should call original' do
@@ -121,7 +121,7 @@ module ConsumerTest
       end
 
       it 'should use the key schema if set' do
-        consumer_class = Class.new(Deimos::Consumer) do
+        consumer_class = Class.new(described_class) do
           schema 'MySchema'
           namespace 'com.my-namespace'
           key_config schema: 'MySchema_key'
@@ -132,7 +132,7 @@ module ConsumerTest
       end
 
       it 'should not decode if plain is set' do
-        consumer_class = Class.new(Deimos::Consumer) do
+        consumer_class = Class.new(described_class) do
           schema 'MySchema'
           namespace 'com.my-namespace'
           key_config plain: true
@@ -142,7 +142,7 @@ module ConsumerTest
       end
 
       it 'should error with nothing set' do
-        consumer_class = Class.new(Deimos::Consumer) do
+        consumer_class = Class.new(described_class) do
           schema 'MySchema'
           namespace 'com.my-namespace'
         end
@@ -156,7 +156,7 @@ module ConsumerTest
     describe 'timestamps' do
       before(:each) do
         # :nodoc:
-        consumer_class = Class.new(Deimos::Consumer) do
+        consumer_class = Class.new(described_class) do
           schema 'MySchemaWithDateTimes'
           namespace 'com.my-namespace'
           key_config plain: true