deimos-ruby 1.7.0.pre.beta1 → 1.8.1.pre.beta3

data/spec/active_record_consume/batch_slicer_spec.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+RSpec.describe Deimos::ActiveRecordConsume::BatchSlicer do
+  describe '#slice' do
+    let(:batch) do
+      [
+        Deimos::Message.new({ v: 1 }, nil, key: 'C'),
+        Deimos::Message.new({ v: 123 }, nil, key: 'A'),
+        Deimos::Message.new({ v: 999 }, nil, key: 'B'),
+        Deimos::Message.new({ v: 456 }, nil, key: 'A'),
+        Deimos::Message.new({ v: 2 }, nil, key: 'C'),
+        Deimos::Message.new({ v: 3 }, nil, key: 'C')
+      ]
+    end
+
+    it 'should slice a batch by key' do
+      slices = described_class.slice(batch)
+
+      expect(slices).
+        to match([
+                   match_array([
+                                 Deimos::Message.new({ v: 1 }, nil, key: 'C'),
+                                 Deimos::Message.new({ v: 123 }, nil, key: 'A'),
+                                 Deimos::Message.new({ v: 999 }, nil, key: 'B')
+                               ]),
+                   match_array([
+                                 Deimos::Message.new({ v: 456 }, nil, key: 'A'),
+                                 Deimos::Message.new({ v: 2 }, nil, key: 'C')
+                               ]),
+                   match_array([
+                                 Deimos::Message.new({ v: 3 }, nil, key: 'C')
+                               ])
+                 ])
+    end
+
+    it 'should handle empty batches' do
+      slices = described_class.slice([])
+
+      expect(slices).to be_empty
+    end
+  end
+end

data/spec/active_record_consume/schema_model_converter_spec.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require 'deimos/active_record_consume/schema_model_converter'
+require 'deimos/schema_backends/avro_local'
+
+# Wrapped in a module to prevent class leakage
+module SchemaModelConverterTest
+  describe Deimos::ActiveRecordConsume::SchemaModelConverter do
+    # Create ActiveRecord table and model
+    before(:all) do
+      ActiveRecord::Base.connection.create_table(:wibbles, force: true) do |t|
+        t.integer(:wibble_id)
+        t.string(:name)
+        t.integer(:bar)
+        t.datetime(:birthday_int)
+        t.datetime(:birthday_long)
+        t.datetime(:birthday_optional)
+        t.timestamps
+      end
+
+      # :nodoc:
+      class Wibble < ActiveRecord::Base
+      end
+      Wibble.reset_column_information
+    end
+
+    after(:all) do
+      ActiveRecord::Base.connection.drop_table(:wibbles)
+    end
+
+    let(:schema) { Deimos::SchemaBackends::AvroLocal.new(schema: 'Wibble', namespace: 'com.my-namespace') }
+    let(:inst) { described_class.new(schema, Wibble) }
+
+    describe '#convert' do
+      it 'should extract attributes from the payload' do
+        payload = { 'id' => 123, 'wibble_id' => 456, 'name' => 'wibble' }
+
+        expect(inst.convert(payload)).to include('id' => 123, 'wibble_id' => 456, 'name' => 'wibble')
+      end
+
+      it 'should ignore payload fields that don\'t exist' do
+        payload = { 'foo' => 'abc' }
+
+        expect(inst.convert(payload)).not_to include('foo')
+      end
+
+      it 'should ignore fields in the schema but not in the model' do
+        payload = { 'floop' => 'def' }
+
+        expect(inst.convert(payload)).not_to include('floop')
+      end
+
+      it 'should ignore model fields not in the schema' do
+        payload = { 'bar' => 'xyz' }
+
+        expect(inst.convert(payload)).not_to include('bar')
+      end
+
+      it 'should handle nils' do
+        payload = { 'name' => nil }
+
+        expect(inst.convert(payload)['name']).to be_nil
+      end
+
+      describe 'timestamps' do
+        it 'should ignore AR timestamp fields' do
+          payload = { 'updated_at' => '1234567890', 'created_at' => '2345678901' }
+
+          expect(inst.convert(payload)).not_to include('updated_at', 'created_at')
+        end
+
+        it 'should parse timestamps' do
+          first = Time.zone.local(2019, 1, 1, 11, 12, 13)
+          second = Time.zone.local(2019, 2, 2, 12, 13, 14)
+
+          payload = { 'birthday_int' => first.to_i, 'birthday_long' => second.to_i }
+
+          expect(inst.convert(payload)).to include('birthday_int' => first, 'birthday_long' => second)
+        end
+
+        it 'should parse integer-string timestamps' do
+          date = Time.zone.local(2019, 1, 1, 11, 12, 13)
+
+          payload = { 'birthday_int' => '1546359133' }
+
+          expect(inst.convert(payload)).to include('birthday_int' => date)
+        end
+
+        it 'should ignore other strings for timestamps' do
+          payload = { 'birthday_int' => 'some-other-val' }
+
+          expect(inst.convert(payload)).to include('birthday_int' => 'some-other-val')
+        end
+      end
+
+      it 'should coerce nullable unions' do
+        date = Time.zone.local(2019, 1, 1, 11, 12, 13)
+
+        payload = { 'birthday_optional' => date.to_i }
+
+        expect(inst.convert(payload)).to include('birthday_optional' => date)
+      end
+    end
+  end
+end

data/spec/active_record_consumer_spec.rb

@@ -2,9 +2,9 @@
 
 require 'date'
 
-# :nodoc:
-module ActiveRecordProducerTest
-  describe Deimos::ActiveRecordConsumer do
+# Wrapped in a module to prevent class leakage
+module ActiveRecordConsumerTest
+  describe Deimos::ActiveRecordConsumer, 'Message Consumer' do
 
     before(:all) do
       ActiveRecord::Base.connection.create_table(:widgets, force: true) do |t|
@@ -137,13 +137,5 @@ module ActiveRecordProducerTest
       expect(Widget.find_by_test_id('id1').some_int).to eq(3)
       expect(Widget.find_by_test_id('id2').some_int).to eq(4)
     end
-
-    it 'should coerce int values to datetimes' do
-      column = Widget.columns.find { |c| c.name == 'some_datetime_int' }
-      expect(MyConsumer.new.send(:_coerce_field, column, 1_579_046_400)).to eq('2020-01-14 19:00:00 -0500')
-      expect(MyConsumer.new.send(:_coerce_field, column, '1579046400')).to eq('2020-01-14 19:00:00 -0500')
-      expect(MyConsumer.new.send(:_coerce_field, column, 'some-other-val')).to eq('some-other-val')
-    end
-
   end
 end

data/spec/batch_consumer_spec.rb

@@ -1,14 +1,12 @@
 # frozen_string_literal: true
 
-require 'deimos/batch_consumer'
-
 # :nodoc:
 module ConsumerTest
-  describe Deimos::BatchConsumer do
+  describe Deimos::Consumer, 'Batch Consumer' do
 
     prepend_before(:each) do
       # :nodoc:
-      consumer_class = Class.new(Deimos::BatchConsumer) do
+      consumer_class = Class.new(described_class) do
         schema 'MySchema'
         namespace 'com.my-namespace'
         key_config field: 'test_id'
@@ -32,6 +30,24 @@ module ConsumerTest
       batch.concat([{ 'invalid' => 'key' }])
     end
 
+    it 'should provide backwards compatibility for BatchConsumer class' do
+      consumer_class = Class.new(Deimos::BatchConsumer) do
+        schema 'MySchema'
+        namespace 'com.my-namespace'
+        key_config field: 'test_id'
+
+        # :nodoc:
+        def consume_batch(_payloads, _metadata)
+          raise 'This should not be called unless call_original is set'
+        end
+      end
+      stub_const('ConsumerTest::MyOldBatchConsumer', consumer_class)
+
+      test_consume_batch(MyOldBatchConsumer, batch) do |received, _metadata|
+        expect(received).to eq(batch)
+      end
+    end
+
     it 'should consume a batch of messages' do
       test_consume_batch(MyBatchConsumer, batch) do |received, _metadata|
         expect(received).to eq(batch)
@@ -95,11 +111,12 @@ module ConsumerTest
         test_consume_batch('my_batch_consume_topic', batch, keys: keys) do |_received, metadata|
           # Mock decode_key extracts the value of the first field as the key
           expect(metadata[:keys]).to eq(%w(foo bar))
+          expect(metadata[:first_offset]).to eq(1)
         end
       end
 
       it 'should decode plain keys for all messages in the batch' do
-        consumer_class = Class.new(Deimos::BatchConsumer) do
+        consumer_class = Class.new(described_class) do
           schema 'MySchema'
           namespace 'com.my-namespace'
           key_config plain: true
@@ -115,7 +132,7 @@ module ConsumerTest
     describe 'timestamps' do
       before(:each) do
         # :nodoc:
-        consumer_class = Class.new(Deimos::BatchConsumer) do
+        consumer_class = Class.new(described_class) do
           schema 'MySchemaWithDateTimes'
           namespace 'com.my-namespace'
           key_config plain: true
@@ -197,7 +214,7 @@ module ConsumerTest
     describe 'logging' do
       before(:each) do
         # :nodoc:
-        consumer_class = Class.new(Deimos::BatchConsumer) do
+        consumer_class = Class.new(described_class) do
           schema 'MySchemaWithUniqueId'
           namespace 'com.my-namespace'
           key_config plain: true

data/spec/config/configuration_spec.rb

@@ -1,6 +1,10 @@
 # frozen_string_literal: true
 
+# Mock consumer
 class MyConfigConsumer < Deimos::Consumer
+  # :no-doc:
+  def consume
+  end
 end
 describe Deimos, 'configuration' do
   it 'should configure with deprecated fields' do

data/spec/consumer_spec.rb

@@ -2,11 +2,11 @@
 
 # :nodoc:
 module ConsumerTest
-  describe Deimos::Consumer do
+  describe Deimos::Consumer, 'Message Consumer' do
 
     prepend_before(:each) do
       # :nodoc:
-      consumer_class = Class.new(Deimos::Consumer) do
+      consumer_class = Class.new(described_class) do
         schema 'MySchema'
         namespace 'com.my-namespace'
         key_config field: 'test_id'
@@ -121,7 +121,7 @@ module ConsumerTest
       end
 
       it 'should use the key schema if set' do
-        consumer_class = Class.new(Deimos::Consumer) do
+        consumer_class = Class.new(described_class) do
           schema 'MySchema'
           namespace 'com.my-namespace'
           key_config schema: 'MySchema_key'
@@ -132,7 +132,7 @@ module ConsumerTest
       end
 
       it 'should not decode if plain is set' do
-        consumer_class = Class.new(Deimos::Consumer) do
+        consumer_class = Class.new(described_class) do
           schema 'MySchema'
           namespace 'com.my-namespace'
           key_config plain: true
@@ -142,7 +142,7 @@ module ConsumerTest
       end
 
       it 'should error with nothing set' do
-        consumer_class = Class.new(Deimos::Consumer) do
+        consumer_class = Class.new(described_class) do
           schema 'MySchema'
           namespace 'com.my-namespace'
         end
@@ -156,7 +156,7 @@ module ConsumerTest
     describe 'timestamps' do
       before(:each) do
         # :nodoc:
-        consumer_class = Class.new(Deimos::Consumer) do
+        consumer_class = Class.new(described_class) do
           schema 'MySchemaWithDateTimes'
           namespace 'com.my-namespace'
           key_config plain: true

data/spec/deimos_spec.rb

@@ -68,11 +68,11 @@ describe Deimos do
 
   describe '#start_db_backend!' do
     it 'should start if backend is db and thread_count is > 0' do
-      signal_handler = instance_double(Deimos::Utils::SignalHandler)
+      signal_handler = instance_double(Sigurd::SignalHandler)
       allow(signal_handler).to receive(:run!)
-      expect(Deimos::Utils::Executor).to receive(:new).
+      expect(Sigurd::Executor).to receive(:new).
         with(anything, sleep_seconds: 5, logger: anything).and_call_original
-      expect(Deimos::Utils::SignalHandler).to receive(:new) do |executor|
+      expect(Sigurd::SignalHandler).to receive(:new) do |executor|
         expect(executor.runners.size).to eq(2)
         signal_handler
       end
@@ -83,7 +83,7 @@ describe Deimos do
     end
 
     it 'should not start if backend is not db' do
-      expect(Deimos::Utils::SignalHandler).not_to receive(:new)
+      expect(Sigurd::SignalHandler).not_to receive(:new)
       described_class.configure do |config|
         config.producers.backend = :kafka
       end
@@ -92,7 +92,7 @@ describe Deimos do
     end
 
     it 'should not start if thread_count is nil' do
-      expect(Deimos::Utils::SignalHandler).not_to receive(:new)
+      expect(Sigurd::SignalHandler).not_to receive(:new)
       described_class.configure do |config|
         config.producers.backend = :db
       end
@@ -101,61 +101,69 @@ describe Deimos do
     end
 
     it 'should not start if thread_count is 0' do
-      expect(Deimos::Utils::SignalHandler).not_to receive(:new)
+      expect(Sigurd::SignalHandler).not_to receive(:new)
       described_class.configure do |config|
         config.producers.backend = :db
       end
       expect { described_class.start_db_backend!(thread_count: 0) }.
         to raise_error('Thread count is not given or set to zero, exiting')
     end
+  end
 
-    describe 'delivery configuration' do
-      before(:each) do
-        allow(YAML).to receive(:load).and_return(phobos_configuration)
-      end
-
-      it 'should not raise an error with properly configured handlers' do
-        path = config_path # for scope issues in the block below
-        # Add explicit consumers
-        phobos_configuration['listeners'] << { 'handler' => 'ConsumerTest::MyConsumer',
-                                               'delivery' => 'message' }
-        phobos_configuration['listeners'] << { 'handler' => 'ConsumerTest::MyConsumer',
-                                               'delivery' => 'batch' }
-
-        expect {
-          described_class.configure { |c| c.phobos_config_file = path }
-        }.not_to raise_error
-      end
-
-      it 'should raise an error if BatchConsumers do not have inline_batch delivery' do
-        path = config_path # for scope issues in the block below
-        phobos_configuration['listeners'] = [{ 'handler' => 'ConsumerTest::MyBatchConsumer',
-                                               'delivery' => 'message' }]
-
-        expect {
-          described_class.configure { |c| c.phobos_config_file = path }
-        }.to raise_error('BatchConsumer ConsumerTest::MyBatchConsumer must have delivery set to `inline_batch`')
-      end
-
-      it 'should raise an error if Consumers do not have message or batch delivery' do
-        path = config_path # for scope issues in the block below
-        phobos_configuration['listeners'] = [{ 'handler' => 'ConsumerTest::MyConsumer',
-                                               'delivery' => 'inline_batch' }]
+  describe 'delivery configuration' do
+    before(:each) do
+      allow(YAML).to receive(:load).and_return(phobos_configuration)
+    end
 
-        expect {
-          described_class.configure { |c| c.phobos_config_file = path }
-        }.to raise_error('Non-batch Consumer ConsumerTest::MyConsumer must have delivery set to `message` or `batch`')
-      end
+    it 'should not raise an error with properly configured handlers' do
+      expect {
+        described_class.configure do
+          consumer do
+            class_name 'ConsumerTest::MyConsumer'
+            delivery :message
+          end
+          consumer do
+            class_name 'ConsumerTest::MyConsumer'
+            delivery :batch
+          end
+          consumer do
+            class_name 'ConsumerTest::MyBatchConsumer'
+            delivery :inline_batch
+          end
+        end
+      }.not_to raise_error
+    end
 
-      it 'should treat nil as `batch`' do
-        path = config_path # for scope issues in the block below
-        phobos_configuration['listeners'] = [{ 'handler' => 'ConsumerTest::MyConsumer' }]
+    it 'should raise an error if inline_batch listeners do not implement consume_batch' do
+      expect {
+        described_class.configure do
+          consumer do
+            class_name 'ConsumerTest::MyConsumer'
+            delivery :inline_batch
+          end
+        end
+      }.to raise_error('BatchConsumer ConsumerTest::MyConsumer does not implement `consume_batch`')
+    end
 
-        expect {
-          described_class.configure { |c| c.phobos_config_file = path }
-        }.not_to raise_error
-      end
+    it 'should raise an error if Consumers do not have message or batch delivery' do
+      expect {
+        described_class.configure do
+          consumer do
+            class_name 'ConsumerTest::MyBatchConsumer'
+            delivery :message
+          end
+        end
+      }.to raise_error('Non-batch Consumer ConsumerTest::MyBatchConsumer does not implement `consume`')
+    end
 
+    it 'should treat nil as `batch`' do
+      expect {
+        described_class.configure do
+          consumer do
+            class_name 'ConsumerTest::MyConsumer'
+          end
+        end
+      }.not_to raise_error
     end
   end
 end