deimos-ruby 1.6.4 → 1.7.0.pre.beta1

data/lib/deimos/utils/db_producer.rb

@@ -2,7 +2,8 @@
 
 module Deimos
   module Utils
-    # Class which continually polls the database and sends Kafka messages.
+    # Class which continually polls the kafka_messages table
+    # in the database and sends Kafka messages.
     class DbProducer
       include Phobos::Producer
       attr_accessor :id, :current_topic

data/lib/deimos/utils/executor.rb

@@ -85,7 +85,7 @@ module Deimos
           retry unless @signal_to_stop
         end
       rescue Exception => e
-        @logger.error("Failed to run listener (#{e.message}) #{error_metadata(e)}")
+        @logger.error("Failed to run executor (#{e.message}) #{error_metadata(e)}")
         raise e
       end
 

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.6.4'
+  VERSION = '1.7.0-beta1'
 end

data/lib/deimos.rb

@@ -31,6 +31,7 @@ if defined?(ActiveRecord)
   require 'deimos/utils/signal_handler.rb'
   require 'deimos/utils/executor.rb'
   require 'deimos/utils/db_producer.rb'
+  require 'deimos/utils/db_poller'
 end
 
 require 'deimos/utils/inline_consumer'

data/lib/generators/deimos/db_poller/templates/migration

@@ -0,0 +1,11 @@
+class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
+  def change
+    create_table :deimos_poll_info, force: true do |t|
+      t.string :producer, null: false
+      t.datetime :last_sent
+      t.bigint :last_sent_id
+    end
+
+    add_index :deimos_poll_info, [:producer]
+  end
+end

data/lib/generators/deimos/db_poller/templates/rails3_migration

@@ -0,0 +1,16 @@
+class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
+  def self.up
+    create_table :deimos_poll_info, force: true do |t|
+      t.string :producer, null: false
+      t.datetime :last_sent
+      t.bigint :last_sent_id
+    end
+
+    add_index :deimos_poll_info, [:producer]
+  end
+
+  def self.down
+    drop_table :deimos_poll_info
+  end
+
+end

data/lib/generators/deimos/db_poller_generator.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'rails/generators'
+require 'rails/generators/active_record/migration'
+
+module Deimos
+  module Generators
+    # Generate the database backend migration.
+    class DbPollerGenerator < Rails::Generators::Base
+      include Rails::Generators::Migration
+      if Rails.version < '4'
+        extend(ActiveRecord::Generators::Migration)
+      else
+        include ActiveRecord::Generators::Migration
+      end
+      source_root File.expand_path('db_poller/templates', __dir__)
+      desc 'Add migrations for the database poller'
+
+      # @return [String]
+      def migration_version
+        "[#{ActiveRecord::Migration.current_version}]"
+      rescue StandardError
+        ''
+      end
+
+      # @return [String]
+      def db_migrate_path
+        if defined?(Rails.application) && Rails.application
+          paths = Rails.application.config.paths['db/migrate']
+          paths.respond_to?(:to_ary) ? paths.to_ary.first : paths.to_a.first
+        else
+          'db/migrate'
+        end
+      end
+
+      # Main method to create all the necessary files
+      def generate
+        if Rails.version < '4'
+          migration_template('rails3_migration',
+                             "#{db_migrate_path}/create_db_poller.rb")
+        else
+          migration_template('migration',
+                             "#{db_migrate_path}/create_db_poller.rb")
+        end
+      end
+    end
+  end
+end

data/lib/tasks/deimos.rake

@@ -24,4 +24,11 @@ namespace :deimos do
     Deimos.start_db_backend!(thread_count: thread_count)
   end
 
+  task db_poller: :environment do
+    ENV['DEIMOS_RAKE_TASK'] = 'true'
+    STDOUT.sync = true
+    Rails.logger.info('Running deimos:db_poller rake task.')
+    Deimos::Utils::DbPoller.start!
+  end
+
 end

data/spec/active_record_producer_spec.rb

@@ -1,107 +1,85 @@
 # frozen_string_literal: true
 
-# :nodoc:
-module ActiveRecordProducerTest
-  describe Deimos::ActiveRecordProducer do
+describe Deimos::ActiveRecordProducer do
 
-    before(:all) do
-      ActiveRecord::Base.connection.create_table(:widgets) do |t|
-        t.string(:test_id)
-        t.integer(:some_int)
-        t.boolean(:some_bool)
-        t.timestamps
-      end
+  include_context 'with widgets'
 
-      # :nodoc:
-      class Widget < ActiveRecord::Base
-        # @return [String]
-        def generated_id
-          'generated_id'
-        end
-      end
-    end
+  prepend_before(:each) do
 
-    after(:all) do
-      ActiveRecord::Base.connection.drop_table(:widgets)
+    producer_class = Class.new(Deimos::ActiveRecordProducer) do
+      schema 'MySchema'
+      namespace 'com.my-namespace'
+      topic 'my-topic'
+      key_config none: true
     end
+    stub_const('MyProducer', producer_class)
 
-    prepend_before(:each) do
-
-      producer_class = Class.new(Deimos::ActiveRecordProducer) do
-        schema 'MySchema'
-        namespace 'com.my-namespace'
-        topic 'my-topic'
-        key_config none: true
-      end
-      stub_const('MyProducer', producer_class)
-
-      producer_class = Class.new(Deimos::ActiveRecordProducer) do
-        schema 'MySchemaWithBooleans'
-        namespace 'com.my-namespace'
-        topic 'my-topic-with-boolean'
-        key_config none: true
-      end
-      stub_const('MyBooleanProducer', producer_class)
-
-      producer_class = Class.new(Deimos::ActiveRecordProducer) do
-        schema 'MySchemaWithId'
-        namespace 'com.my-namespace'
-        topic 'my-topic-with-id'
-        key_config none: true
-        record_class Widget
+    producer_class = Class.new(Deimos::ActiveRecordProducer) do
+      schema 'MySchemaWithBooleans'
+      namespace 'com.my-namespace'
+      topic 'my-topic-with-boolean'
+      key_config none: true
+    end
+    stub_const('MyBooleanProducer', producer_class)
 
-        # :nodoc:
-        def self.generate_payload(attrs, widget)
-          super.merge(message_id: widget.generated_id)
-        end
+    producer_class = Class.new(Deimos::ActiveRecordProducer) do
+      schema 'MySchemaWithId'
+      namespace 'com.my-namespace'
+      topic 'my-topic-with-id'
+      key_config none: true
+      record_class Widget
 
+      # :nodoc:
+      def self.generate_payload(attrs, widget)
+        super.merge(message_id: widget.generated_id)
       end
-      stub_const('MyProducerWithID', producer_class)
 
-      producer_class = Class.new(Deimos::ActiveRecordProducer) do
-        schema 'MySchemaWithUniqueId'
-        namespace 'com.my-namespace'
-        topic 'my-topic-with-unique-id'
-        key_config field: :id
-        record_class Widget
-      end
-      stub_const('MyProducerWithUniqueID', producer_class)
     end
-
-    it 'should send events correctly' do
-      MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 3))
-      expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
+    stub_const('MyProducerWithID', producer_class)
+
+    producer_class = Class.new(Deimos::ActiveRecordProducer) do
+      schema 'MySchemaWithUniqueId'
+      namespace 'com.my-namespace'
+      topic 'my-topic-with-unique-id'
+      key_config field: :id
+      record_class Widget
     end
+    stub_const('MyProducerWithUniqueID', producer_class)
+  end
 
-    it 'should coerce values' do
-      MyProducer.send_event(Widget.new(test_id: 'abc', some_int: '3'))
-      MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 4.5))
-      expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
-      expect('my-topic').to have_sent(test_id: 'abc', some_int: 4)
-      expect {
-        MyProducer.send_event(Widget.new(test_id: 'abc', some_int: nil))
-      }.to raise_error(Avro::SchemaValidator::ValidationError)
-
-      MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: nil))
-      MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: true))
-      expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: false)
-      expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: true)
-    end
+  it 'should send events correctly' do
+    MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 3))
+    expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
+  end
 
-    it 'should be able to call the record' do
-      widget = Widget.create!(test_id: 'abc2', some_int: 3)
-      MyProducerWithID.send_event(id: widget.id, test_id: 'abc2', some_int: 3)
-      expect('my-topic-with-id').to have_sent(
-        test_id: 'abc2',
-        some_int: 3,
-        message_id: 'generated_id',
-        timestamp: anything
-      )
-    end
+  it 'should coerce values' do
+    MyProducer.send_event(Widget.new(test_id: 'abc', some_int: '3'))
+    MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 4.5))
+    expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
+    expect('my-topic').to have_sent(test_id: 'abc', some_int: 4)
+    expect {
+      MyProducer.send_event(Widget.new(test_id: 'abc', some_int: nil))
+    }.to raise_error(Avro::SchemaValidator::ValidationError)
+
+    MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: nil))
+    MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: true))
+    expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: false)
+    expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: true)
+  end
 
-    specify '#watched_attributes' do
-      expect(MyProducer.watched_attributes).to eq(%w(test_id some_int))
-    end
+  it 'should be able to call the record' do
+    widget = Widget.create!(test_id: 'abc2', some_int: 3)
+    MyProducerWithID.send_event(id: widget.id, test_id: 'abc2', some_int: 3)
+    expect('my-topic-with-id').to have_sent(
+      test_id: 'abc2',
+      some_int: 3,
+      message_id: 'generated_id',
+      timestamp: anything
+    )
+  end
 
+  specify '#watched_attributes' do
+    expect(MyProducer.watched_attributes).to eq(%w(test_id some_int))
   end
+
 end

data/spec/consumer_spec.rb

@@ -90,7 +90,7 @@ module ConsumerTest
             'some_int' => 123 },
           { skip_expectation: true }
         ) { raise 'OH NOES' }
-      } .not_to raise_error
+      }.not_to raise_error
     end
 
     it 'should not fail when consume fails without reraising errors' do
@@ -101,7 +101,7 @@ module ConsumerTest
           { 'invalid' => 'key' },
           { skip_expectation: true }
         )
-      } .not_to raise_error
+      }.not_to raise_error
     end
 
     it 'should call original' do

data/spec/producer_spec.rb

@@ -148,7 +148,7 @@ module ProducerTest
         Deimos.disable_producers do
           raise 'OH NOES'
         end
-      }      .to raise_error('OH NOES')
+      }.to raise_error('OH NOES')
       expect(Deimos).not_to be_producers_disabled
     end
 
@@ -246,7 +246,7 @@ module ProducerTest
         MyNonEncodedProducer.publish_list(
           [{ 'test_id' => 'foo', 'some_int' => 123 }]
         )
-      }      .to raise_error('No key given but a key is required! Use `key_config none: true` to avoid using keys.')
+      }.to raise_error('No key given but a key is required! Use `key_config none: true` to avoid using keys.')
     end
 
     it 'should allow nil keys if none: true is configured' do
@@ -254,7 +254,7 @@ module ProducerTest
         MyNoKeyProducer.publish_list(
           [{ 'test_id' => 'foo', 'some_int' => 123 }]
         )
-      }      .not_to raise_error
+      }.not_to raise_error
     end
 
     it 'should use a partition key' do

data/spec/rake_spec.rb

@@ -9,7 +9,7 @@ if Rake.application.lookup(:environment).nil?
   Rake::Task.define_task(:environment)
 end
 
-describe 'Rakefile' do # rubocop:disable RSpec/DescribeClass
+describe 'Rakefile' do
   it 'should start listeners' do
     runner = instance_double(Phobos::CLI::Runner)
     expect(Phobos::CLI::Runner).to receive(:new).and_return(runner)

data/spec/spec_helper.rb

@@ -100,9 +100,8 @@ module DbConfigs
     end
   end
 
-  # Set up the given database.
-  def setup_db(options)
-    ActiveRecord::Base.establish_connection(options)
+  # :nodoc:
+  def run_db_backend_migration
     migration_class_name = 'DbBackendMigration'
     migration_version = '[5.2]'
     migration = ERB.new(
@@ -110,6 +109,24 @@ module DbConfigs
     ).result(binding)
     eval(migration) # rubocop:disable Security/Eval
     ActiveRecord::Migration.new.run(DbBackendMigration, direction: :up)
+  end
+
+  # :nodoc:
+  def run_db_poller_migration
+    migration_class_name = 'DbPollerMigration'
+    migration_version = '[5.2]'
+    migration = ERB.new(
+      File.read('lib/generators/deimos/db_poller/templates/migration')
+    ).result(binding)
+    eval(migration) # rubocop:disable Security/Eval
+    ActiveRecord::Migration.new.run(DbPollerMigration, direction: :up)
+  end
+
+  # Set up the given database.
+  def setup_db(options)
+    ActiveRecord::Base.establish_connection(options)
+    run_db_backend_migration
+    run_db_poller_migration
 
     ActiveRecord::Base.descendants.each do |klass|
       klass.reset_sequence_name if klass.respond_to?(:reset_sequence_name)
@@ -131,7 +148,7 @@ RSpec.configure do |config|
   config.shared_context_metadata_behavior = :apply_to_host_groups
 
   config.before(:all) do
-    Time.zone = 'EST'
+    Time.zone = 'Eastern Time (US & Canada)'
     ActiveRecord::Base.logger = Logger.new('/dev/null')
     ActiveRecord::Base.establish_connection(
       'adapter' => 'sqlite3',
@@ -141,8 +158,6 @@ RSpec.configure do |config|
   config.include Deimos::TestHelpers
   config.include ActiveSupport::Testing::TimeHelpers
   config.before(:suite) do
-    Time.zone = 'EST'
-    ActiveRecord::Base.logger = Logger.new('/dev/null')
     setup_db(DbConfigs::DB_OPTIONS.last)
   end
 
@@ -166,6 +181,29 @@ RSpec.configure do |config|
   end
 end
 
+RSpec.shared_context('with widgets') do
+  before(:all) do
+    ActiveRecord::Base.connection.create_table(:widgets, force: true) do |t|
+      t.string(:test_id)
+      t.integer(:some_int)
+      t.boolean(:some_bool)
+      t.timestamps
+    end
+
+    # :nodoc:
+    class Widget < ActiveRecord::Base
+      # @return [String]
+      def generated_id
+        'generated_id'
+      end
+    end
+  end
+
+  after(:all) do
+    ActiveRecord::Base.connection.drop_table(:widgets)
+  end
+end
+
 RSpec.shared_context('with DB') do
   before(:all) do
     setup_db(self.class.metadata[:db_config] || DbConfigs::DB_OPTIONS.last)