deimos-ruby 1.6.1 → 1.8.0.pre.beta1

data/spec/consumer_spec.rb

@@ -2,11 +2,11 @@
 
 # :nodoc:
 module ConsumerTest
-  describe Deimos::Consumer do
+  describe Deimos::Consumer, 'Message Consumer' do
 
     prepend_before(:each) do
       # :nodoc:
-      consumer_class = Class.new(Deimos::Consumer) do
+      consumer_class = Class.new(described_class) do
         schema 'MySchema'
         namespace 'com.my-namespace'
         key_config field: 'test_id'
@@ -90,7 +90,7 @@ module ConsumerTest
             'some_int' => 123 },
           { skip_expectation: true }
         ) { raise 'OH NOES' }
-      } .not_to raise_error
+      }.not_to raise_error
     end
 
     it 'should not fail when consume fails without reraising errors' do
@@ -101,7 +101,7 @@ module ConsumerTest
           { 'invalid' => 'key' },
           { skip_expectation: true }
         )
-      } .not_to raise_error
+      }.not_to raise_error
     end
 
     it 'should call original' do
@@ -121,7 +121,7 @@ module ConsumerTest
       end
 
       it 'should use the key schema if set' do
-        consumer_class = Class.new(Deimos::Consumer) do
+        consumer_class = Class.new(described_class) do
           schema 'MySchema'
           namespace 'com.my-namespace'
           key_config schema: 'MySchema_key'
@@ -132,7 +132,7 @@ module ConsumerTest
       end
 
       it 'should not decode if plain is set' do
-        consumer_class = Class.new(Deimos::Consumer) do
+        consumer_class = Class.new(described_class) do
           schema 'MySchema'
           namespace 'com.my-namespace'
           key_config plain: true
@@ -142,7 +142,7 @@ module ConsumerTest
       end
 
       it 'should error with nothing set' do
-        consumer_class = Class.new(Deimos::Consumer) do
+        consumer_class = Class.new(described_class) do
           schema 'MySchema'
           namespace 'com.my-namespace'
         end
@@ -156,7 +156,7 @@ module ConsumerTest
     describe 'timestamps' do
       before(:each) do
         # :nodoc:
-        consumer_class = Class.new(Deimos::Consumer) do
+        consumer_class = Class.new(described_class) do
           schema 'MySchemaWithDateTimes'
           namespace 'com.my-namespace'
           key_config plain: true

data/spec/deimos_spec.rb

@@ -68,11 +68,11 @@ describe Deimos do
 
   describe '#start_db_backend!' do
     it 'should start if backend is db and thread_count is > 0' do
-      signal_handler = instance_double(Deimos::Utils::SignalHandler)
+      signal_handler = instance_double(Sigurd::SignalHandler)
       allow(signal_handler).to receive(:run!)
-      expect(Deimos::Utils::Executor).to receive(:new).
+      expect(Sigurd::Executor).to receive(:new).
         with(anything, sleep_seconds: 5, logger: anything).and_call_original
-      expect(Deimos::Utils::SignalHandler).to receive(:new) do |executor|
+      expect(Sigurd::SignalHandler).to receive(:new) do |executor|
         expect(executor.runners.size).to eq(2)
         signal_handler
       end
@@ -83,7 +83,7 @@ describe Deimos do
     end
 
     it 'should not start if backend is not db' do
-      expect(Deimos::Utils::SignalHandler).not_to receive(:new)
+      expect(Sigurd::SignalHandler).not_to receive(:new)
       described_class.configure do |config|
         config.producers.backend = :kafka
       end
@@ -92,7 +92,7 @@ describe Deimos do
     end
 
     it 'should not start if thread_count is nil' do
-      expect(Deimos::Utils::SignalHandler).not_to receive(:new)
+      expect(Sigurd::SignalHandler).not_to receive(:new)
       described_class.configure do |config|
         config.producers.backend = :db
       end
@@ -101,61 +101,69 @@ describe Deimos do
     end
 
     it 'should not start if thread_count is 0' do
-      expect(Deimos::Utils::SignalHandler).not_to receive(:new)
+      expect(Sigurd::SignalHandler).not_to receive(:new)
       described_class.configure do |config|
         config.producers.backend = :db
       end
       expect { described_class.start_db_backend!(thread_count: 0) }.
         to raise_error('Thread count is not given or set to zero, exiting')
     end
+  end
 
-    describe 'delivery configuration' do
-      before(:each) do
-        allow(YAML).to receive(:load).and_return(phobos_configuration)
-      end
-
-      it 'should not raise an error with properly configured handlers' do
-        path = config_path # for scope issues in the block below
-        # Add explicit consumers
-        phobos_configuration['listeners'] << { 'handler' => 'ConsumerTest::MyConsumer',
-                                               'delivery' => 'message' }
-        phobos_configuration['listeners'] << { 'handler' => 'ConsumerTest::MyConsumer',
-                                               'delivery' => 'batch' }
-
-        expect {
-          described_class.configure { |c| c.phobos_config_file = path }
-        }.not_to raise_error
-      end
-
-      it 'should raise an error if BatchConsumers do not have inline_batch delivery' do
-        path = config_path # for scope issues in the block below
-        phobos_configuration['listeners'] = [{ 'handler' => 'ConsumerTest::MyBatchConsumer',
-                                               'delivery' => 'message' }]
-
-        expect {
-          described_class.configure { |c| c.phobos_config_file = path }
-        }.to raise_error('BatchConsumer ConsumerTest::MyBatchConsumer must have delivery set to `inline_batch`')
-      end
-
-      it 'should raise an error if Consumers do not have message or batch delivery' do
-        path = config_path # for scope issues in the block below
-        phobos_configuration['listeners'] = [{ 'handler' => 'ConsumerTest::MyConsumer',
-                                               'delivery' => 'inline_batch' }]
+  describe 'delivery configuration' do
+    before(:each) do
+      allow(YAML).to receive(:load).and_return(phobos_configuration)
+    end
 
-        expect {
-          described_class.configure { |c| c.phobos_config_file = path }
-        }.to raise_error('Non-batch Consumer ConsumerTest::MyConsumer must have delivery set to `message` or `batch`')
-      end
+    it 'should not raise an error with properly configured handlers' do
+      expect {
+        described_class.configure do
+          consumer do
+            class_name 'ConsumerTest::MyConsumer'
+            delivery :message
+          end
+          consumer do
+            class_name 'ConsumerTest::MyConsumer'
+            delivery :batch
+          end
+          consumer do
+            class_name 'ConsumerTest::MyBatchConsumer'
+            delivery :inline_batch
+          end
+        end
+      }.not_to raise_error
+    end
 
-      it 'should treat nil as `batch`' do
-        path = config_path # for scope issues in the block below
-        phobos_configuration['listeners'] = [{ 'handler' => 'ConsumerTest::MyConsumer' }]
+    it 'should raise an error if inline_batch listeners do not implement consume_batch' do
+      expect {
+        described_class.configure do
+          consumer do
+            class_name 'ConsumerTest::MyConsumer'
+            delivery :inline_batch
+          end
+        end
+      }.to raise_error('BatchConsumer ConsumerTest::MyConsumer does not implement `consume_batch`')
+    end
 
-        expect {
-          described_class.configure { |c| c.phobos_config_file = path }
-        }.not_to raise_error
-      end
+    it 'should raise an error if Consumers do not have message or batch delivery' do
+      expect {
+        described_class.configure do
+          consumer do
+            class_name 'ConsumerTest::MyBatchConsumer'
+            delivery :message
+          end
+        end
+      }.to raise_error('Non-batch Consumer ConsumerTest::MyBatchConsumer does not implement `consume`')
+    end
 
+    it 'should treat nil as `batch`' do
+      expect {
+        described_class.configure do
+          consumer do
+            class_name 'ConsumerTest::MyConsumer'
+          end
+        end
+      }.not_to raise_error
     end
   end
 end

data/spec/handlers/my_batch_consumer.rb

@@ -1,5 +1,10 @@
 # frozen_string_literal: true
 
 module ConsumerTest
-  class MyBatchConsumer < Deimos::BatchConsumer; end
+  # Mock consumer
+  class MyBatchConsumer < Deimos::Consumer
+    # :no-doc:
+    def consume_batch
+    end
+  end
 end

data/spec/handlers/my_consumer.rb

@@ -1,5 +1,10 @@
 # frozen_string_literal: true
 
 module ConsumerTest
-  class MyConsumer < Deimos::Consumer; end
+  # Mock consumer
+  class MyConsumer < Deimos::Consumer
+    # :no-doc:
+    def consume
+    end
+  end
 end

data/spec/kafka_source_spec.rb

@@ -126,6 +126,59 @@ module KafkaSourceSpec
                                       }, widgets[2].id)
     end
 
+    it 'should send events on import with on_duplicate_key_update and existing records' do
+      widget1 = Widget.create(widget_id: 1, name: 'Widget 1')
+      widget2 = Widget.create(widget_id: 2, name: 'Widget 2')
+      widget1.name = 'New Widget 1'
+      widget2.name = 'New Widget 2'
+      Widget.import([widget1, widget2], on_duplicate_key_update: %i(widget_id name))
+
+      expect('my-topic').to have_sent({
+                                        widget_id: 1,
+                                        name: 'New Widget 1',
+                                        id: widget1.id,
+                                        created_at: anything,
+                                        updated_at: anything
+                                      }, widget1.id)
+      expect('my-topic').to have_sent({
+                                        widget_id: 2,
+                                        name: 'New Widget 2',
+                                        id: widget2.id,
+                                        created_at: anything,
+                                        updated_at: anything
+                                      }, widget2.id)
+    end
+
+    it 'should not fail when mixing existing and new records for import :on_duplicate_key_update' do
+      widget1 = Widget.create(widget_id: 1, name: 'Widget 1')
+      expect('my-topic').to have_sent({
+                                        widget_id: 1,
+                                        name: 'Widget 1',
+                                        id: widget1.id,
+                                        created_at: anything,
+                                        updated_at: anything
+                                      }, widget1.id)
+
+      widget2 = Widget.new(widget_id: 2, name: 'Widget 2')
+      widget1.name = 'New Widget 1'
+      Widget.import([widget1, widget2], on_duplicate_key_update: %i(widget_id))
+      widgets = Widget.all
+      expect('my-topic').to have_sent({
+                                        widget_id: 1,
+                                        name: 'New Widget 1',
+                                        id: widgets[0].id,
+                                        created_at: anything,
+                                        updated_at: anything
+                                      }, widgets[0].id)
+      expect('my-topic').to have_sent({
+                                        widget_id: 2,
+                                        name: 'Widget 2',
+                                        id: widgets[1].id,
+                                        created_at: anything,
+                                        updated_at: anything
+                                      }, widgets[1].id)
+    end
+
     it 'should send events even if the save fails' do
       widget = Widget.create!(widget_id: 1, name: 'widget')
       expect('my-topic').to have_sent({

data/spec/message_spec.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+RSpec.describe(Deimos::Message) do
+  it 'should detect tombstones' do
+    expect(described_class.new(nil, nil, key: 'key1')).
+      to be_tombstone
+    expect(described_class.new({ v: 'val1' }, nil, key: 'key1')).
+      not_to be_tombstone
+    expect(described_class.new({ v: '' }, nil, key: 'key1')).
+      not_to be_tombstone
+    expect(described_class.new({ v: 'val1' }, nil, key: nil)).
+      not_to be_tombstone
+  end
+
+  it 'can support complex keys/values' do
+    expect { described_class.new({ a: 1, b: 2 }, nil, key: { c: 3, d: 4 }) }.
+      not_to raise_exception
+  end
+end

data/spec/producer_spec.rb

@@ -148,7 +148,7 @@ module ProducerTest
         Deimos.disable_producers do
           raise 'OH NOES'
         end
-      }      .to raise_error('OH NOES')
+      }.to raise_error('OH NOES')
       expect(Deimos).not_to be_producers_disabled
     end
 
@@ -246,7 +246,7 @@ module ProducerTest
         MyNonEncodedProducer.publish_list(
           [{ 'test_id' => 'foo', 'some_int' => 123 }]
         )
-      }      .to raise_error('No key given but a key is required! Use `key_config none: true` to avoid using keys.')
+      }.to raise_error('No key given but a key is required! Use `key_config none: true` to avoid using keys.')
     end
 
     it 'should allow nil keys if none: true is configured' do
@@ -254,7 +254,7 @@ module ProducerTest
         MyNoKeyProducer.publish_list(
           [{ 'test_id' => 'foo', 'some_int' => 123 }]
         )
-      }      .not_to raise_error
+      }.not_to raise_error
     end
 
     it 'should use a partition key' do

data/spec/rake_spec.rb

@@ -9,7 +9,7 @@ if Rake.application.lookup(:environment).nil?
   Rake::Task.define_task(:environment)
 end
 
-describe 'Rakefile' do # rubocop:disable RSpec/DescribeClass
+describe 'Rakefile' do
   it 'should start listeners' do
     runner = instance_double(Phobos::CLI::Runner)
     expect(Phobos::CLI::Runner).to receive(:new).and_return(runner)

data/spec/schemas/com/my-namespace/MySchemaCompound-key.avsc

@@ -0,0 +1,18 @@
+{
+  "namespace": "com.my-namespace",
+  "name": "MySchemaCompound-key",
+  "type": "record",
+  "doc": "Test schema",
+  "fields": [
+    {
+      "name": "part_one",
+      "type": "string",
+      "doc": "test string one"
+    },
+    {
+      "name": "part_two",
+      "type": "string",
+      "doc": "test string two"
+    }
+  ]
+}

data/spec/schemas/com/my-namespace/Wibble.avsc

@@ -0,0 +1,43 @@
+{
+  "namespace": "com.my-namespace",
+  "name": "Wibble",
+  "type": "record",
+  "fields": [
+    {
+      "name": "id",
+      "type": "long"
+    },
+    {
+      "name": "wibble_id",
+      "type": "long"
+    },
+    {
+      "name": "name",
+      "type": "string"
+    },
+    {
+      "name": "floop",
+      "type": "string"
+    },
+    {
+      "name": "birthday_int",
+      "type": "int"
+    },
+    {
+      "name": "birthday_long",
+      "type": "long"
+    },
+    {
+      "name": "birthday_optional",
+      "type": ["null", "int"]
+    },
+    {
+      "name": "updated_at",
+      "type": "long"
+    },
+    {
+      "name": "created_at",
+      "type": "long"
+    }
+  ]
+}

data/spec/spec_helper.rb

@@ -2,6 +2,7 @@
 
 $LOAD_PATH.unshift(File.expand_path('../lib', __dir__))
 require 'active_record'
+require 'database_cleaner'
 require 'deimos'
 require 'deimos/metrics/mock'
 require 'deimos/tracing/mock'
@@ -100,9 +101,8 @@ module DbConfigs
     end
   end
 
-  # Set up the given database.
-  def setup_db(options)
-    ActiveRecord::Base.establish_connection(options)
+  # :nodoc:
+  def run_db_backend_migration
     migration_class_name = 'DbBackendMigration'
     migration_version = '[5.2]'
     migration = ERB.new(
@@ -110,6 +110,24 @@ module DbConfigs
     ).result(binding)
     eval(migration) # rubocop:disable Security/Eval
     ActiveRecord::Migration.new.run(DbBackendMigration, direction: :up)
+  end
+
+  # :nodoc:
+  def run_db_poller_migration
+    migration_class_name = 'DbPollerMigration'
+    migration_version = '[5.2]'
+    migration = ERB.new(
+      File.read('lib/generators/deimos/db_poller/templates/migration')
+    ).result(binding)
+    eval(migration) # rubocop:disable Security/Eval
+    ActiveRecord::Migration.new.run(DbPollerMigration, direction: :up)
+  end
+
+  # Set up the given database.
+  def setup_db(options)
+    ActiveRecord::Base.establish_connection(options)
+    run_db_backend_migration
+    run_db_poller_migration
 
     ActiveRecord::Base.descendants.each do |klass|
       klass.reset_sequence_name if klass.respond_to?(:reset_sequence_name)
@@ -130,8 +148,11 @@ RSpec.configure do |config|
   # true by default for RSpec 4.0
   config.shared_context_metadata_behavior = :apply_to_host_groups
 
+  config.filter_run(focus: true)
+  config.run_all_when_everything_filtered = true
+
   config.before(:all) do
-    Time.zone = 'EST'
+    Time.zone = 'Eastern Time (US & Canada)'
     ActiveRecord::Base.logger = Logger.new('/dev/null')
     ActiveRecord::Base.establish_connection(
       'adapter' => 'sqlite3',
@@ -141,9 +162,10 @@ RSpec.configure do |config|
   config.include Deimos::TestHelpers
   config.include ActiveSupport::Testing::TimeHelpers
   config.before(:suite) do
-    Time.zone = 'EST'
-    ActiveRecord::Base.logger = Logger.new('/dev/null')
     setup_db(DbConfigs::DB_OPTIONS.last)
+
+    DatabaseCleaner.strategy = :transaction
+    DatabaseCleaner.clean_with(:truncation)
   end
 
   config.mock_with(:rspec) do |mocks|
@@ -164,6 +186,39 @@ RSpec.configure do |config|
       deimos_config.schema.backend = :avro_validation
     end
   end
+
+  config.around(:each) do |example|
+    use_cleaner = !example.metadata[:integration]
+
+    DatabaseCleaner.start if use_cleaner
+
+    example.run
+
+    DatabaseCleaner.clean if use_cleaner
+  end
+end
+
+RSpec.shared_context('with widgets') do
+  before(:all) do
+    ActiveRecord::Base.connection.create_table(:widgets, force: true) do |t|
+      t.string(:test_id)
+      t.integer(:some_int)
+      t.boolean(:some_bool)
+      t.timestamps
+    end
+
+    # :nodoc:
+    class Widget < ActiveRecord::Base
+      # @return [String]
+      def generated_id
+        'generated_id'
+      end
+    end
+  end
+
+  after(:all) do
+    ActiveRecord::Base.connection.drop_table(:widgets)
+  end
 end
 
 RSpec.shared_context('with DB') do