deimos-ruby 1.0.0.pre.beta22

data/lib/generators/deimos/db_backend/templates/rails3_migration

@@ -0,0 +1,30 @@
+class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
+  def self.up
+    create_table :kafka_messages, force: true do |t|
+      t.string :topic, null: false
+      t.binary :message, limit: 10.megabytes
+      t.binary :key
+      t.string :partition_key
+      t.timestamps
+    end
+
+    add_index :kafka_messages, [:topic, :id]
+
+    create_table :kafka_topic_info, force: true do |t| # rubocop:disable Rails/CreateTableWithTimestamps
+      t.string :topic, null: false
+      t.string :locked_by
+      t.datetime :locked_at
+      t.boolean :error, null: false, default: false
+      t.integer :retries, null: false, default: 0
+    end
+    add_index :kafka_topic_info, :topic, unique: true
+    add_index :kafka_topic_info, [:locked_by, :error]
+    add_index :kafka_topic_info, :locked_at
+  end
+
+  def self.down
+    drop_table :kafka_messages
+    drop_table :kafka_topic_info
+  end
+
+end

data/lib/generators/deimos/db_backend_generator.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'rails/generators'
+require 'rails/generators/active_record/migration'
+
+module Deimos
+  module Generators
+    # Generate the database backend migration.
+    class DbBackendGenerator < Rails::Generators::Base
+      include Rails::Generators::Migration
+      if Rails.version < '4'
+        extend(ActiveRecord::Generators::Migration)
+      else
+        include ActiveRecord::Generators::Migration
+      end
+      source_root File.expand_path('db_backend/templates', __dir__)
+      desc 'Add migrations for the database backend'
+
+      # @return [String]
+      def migration_version
+        "[#{ActiveRecord::Migration.current_version}]"
+      rescue StandardError
+        ''
+      end
+
+      # @return [String]
+      def db_migrate_path
+        if defined?(Rails.application) && Rails.application
+          paths = Rails.application.config.paths['db/migrate']
+          paths.respond_to?(:to_ary) ? paths.to_ary.first : paths.to_a.first
+        else
+          'db/migrate'
+        end
+      end
+
+      # Main method to create all the necessary files
+      def generate
+        if Rails.version < '4'
+          migration_template('rails3_migration',
+                             "#{db_migrate_path}/create_db_backend.rb")
+        else
+          migration_template('migration',
+                             "#{db_migrate_path}/create_db_backend.rb")
+        end
+      end
+    end
+  end
+end

data/lib/tasks/deimos.rake

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'phobos'
+require 'phobos/cli'
+
+namespace :deimos do
+  desc 'Starts Deimos in the rails environment'
+  task start: :environment do
+    Deimos.configure do |config|
+      config.publish_backend = :kafka_sync if config.publish_backend == :kafka_async
+    end
+    ENV['DEIMOS_RAKE_TASK'] = 'true'
+    STDOUT.sync = true
+    Rails.logger.info('Running deimos:start rake task.')
+    Phobos::CLI::Commands.start(%w(start --skip_config))
+  end
+
+  desc 'Starts the Deimos database producer'
+  task db_producer: :environment do
+    ENV['DEIMOS_RAKE_TASK'] = 'true'
+    STDOUT.sync = true
+    Rails.logger.info('Running deimos:db_producer rake task.')
+    thread_count = ENV['THREADS'].presence || 1
+    Deimos.start_db_backend!(thread_count: thread_count)
+  end
+
+end

data/spec/active_record_consumer_spec.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require 'date'
+
+# :nodoc:
+module ActiveRecordProducerTest
+  describe Deimos::ActiveRecordConsumer do
+
+    before(:all) do
+      ActiveRecord::Base.connection.create_table(:widgets, force: true) do |t|
+        t.string(:test_id)
+        t.integer(:some_int)
+        t.boolean(:some_bool)
+        t.datetime(:some_datetime_int)
+        t.timestamps
+      end
+
+      # :nodoc:
+      class Widget < ActiveRecord::Base
+      end
+      Widget.reset_column_information
+    end
+
+    after(:all) do
+      ActiveRecord::Base.connection.drop_table(:widgets)
+    end
+
+    prepend_before(:each) do
+
+      consumer_class = Class.new(Deimos::ActiveRecordConsumer) do
+        schema 'MySchemaWithDateTimes'
+        namespace 'com.my-namespace'
+        key_config plain: true
+        record_class Widget
+      end
+      stub_const('MyConsumer', consumer_class)
+
+      Time.zone = 'Eastern Time (US & Canada)'
+    end
+
+    it 'should receive events correctly' do
+      travel 1.day do
+        expect(Widget.count).to eq(0)
+        test_consume_message(MyConsumer, {
+                               test_id: 'abc',
+                               some_int: 3,
+                               updated_at: 1.day.ago.to_i,
+                               some_datetime_int: Time.zone.now.to_i,
+                               timestamp: 2.minutes.ago.to_s
+                             }, { call_original: true, key: 5 })
+
+        expect(Widget.count).to eq(1)
+        widget = Widget.last
+        expect(widget.id).to eq(5)
+        expect(widget.test_id).to eq('abc')
+        expect(widget.some_int).to eq(3)
+        expect(widget.some_datetime_int).to eq(Time.zone.now)
+        expect(widget.updated_at).to eq(Time.zone.now)
+
+        # test update
+        test_consume_message(MyConsumer, {
+                               test_id: 'abcd',
+                               some_int: 3,
+                               some_datetime_int: Time.zone.now.to_i,
+                               timestamp: 2.minutes.ago.to_s
+                             }, { call_original: true, key: 5 })
+        widget = Widget.last
+        expect(widget.id).to eq(5)
+        expect(widget.test_id).to eq('abcd')
+        expect(widget.some_int).to eq(3)
+
+        # test delete
+        test_consume_message(MyConsumer, nil, call_original: true, key: 5)
+        expect(Widget.count).to eq(0)
+
+      end
+
+    end
+
+  end
+end

data/spec/active_record_producer_spec.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+# :nodoc:
+module ActiveRecordProducerTest
+  describe Deimos::ActiveRecordProducer do
+
+    before(:all) do
+      ActiveRecord::Base.connection.create_table(:widgets) do |t|
+        t.string(:test_id)
+        t.integer(:some_int)
+        t.boolean(:some_bool)
+        t.timestamps
+      end
+
+      # :nodoc:
+      class Widget < ActiveRecord::Base
+        # @return [String]
+        def generated_id
+          'generated_id'
+        end
+      end
+    end
+
+    after(:all) do
+      ActiveRecord::Base.connection.drop_table(:widgets)
+    end
+
+    prepend_before(:each) do
+
+      producer_class = Class.new(Deimos::ActiveRecordProducer) do
+        schema 'MySchema'
+        namespace 'com.my-namespace'
+        topic 'my-topic'
+        key_config none: true
+      end
+      stub_const('MyProducer', producer_class)
+
+      producer_class = Class.new(Deimos::ActiveRecordProducer) do
+        schema 'MySchemaWithBooleans'
+        namespace 'com.my-namespace'
+        topic 'my-topic-with-boolean'
+        key_config none: true
+      end
+      stub_const('MyBooleanProducer', producer_class)
+
+      producer_class = Class.new(Deimos::ActiveRecordProducer) do
+        schema 'MySchemaWithId'
+        namespace 'com.my-namespace'
+        topic 'my-topic-with-id'
+        key_config none: true
+        record_class Widget
+
+        # :nodoc:
+        def self.generate_payload(attrs, widget)
+          super.merge(message_id: widget.generated_id)
+        end
+
+      end
+      stub_const('MyProducerWithID', producer_class)
+
+      producer_class = Class.new(Deimos::ActiveRecordProducer) do
+        schema 'MySchemaWithUniqueId'
+        namespace 'com.my-namespace'
+        topic 'my-topic-with-unique-id'
+        key_config field: :id
+        record_class Widget
+      end
+      stub_const('MyProducerWithUniqueID', producer_class)
+    end
+
+    it 'should send events correctly' do
+      MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 3))
+      expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
+    end
+
+    it 'should coerce values' do
+      MyProducer.send_event(Widget.new(test_id: 'abc', some_int: '3'))
+      MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 4.5))
+      expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
+      expect('my-topic').to have_sent(test_id: 'abc', some_int: 4)
+      expect {
+        MyProducer.send_event(Widget.new(test_id: 'abc', some_int: nil))
+      }.to raise_error(Avro::SchemaValidator::ValidationError)
+
+      MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: nil))
+      MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: true))
+      expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: false)
+      expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: true)
+    end
+
+    it 'should be able to call the record' do
+      widget = Widget.create!(test_id: 'abc2', some_int: 3)
+      MyProducerWithID.send_event(id: widget.id, test_id: 'abc2', some_int: 3)
+      expect('my-topic-with-id').to have_sent(
+        test_id: 'abc2',
+        some_int: 3,
+        message_id: 'generated_id',
+        timestamp: anything
+      )
+    end
+
+    specify '#watched_attributes' do
+      expect(MyProducer.watched_attributes).to eq(%w(test_id some_int))
+    end
+
+  end
+end

data/spec/avro_data_decoder_spec.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+describe Deimos::AvroDataDecoder do
+
+  let(:decoder) do
+    decoder = described_class.new(schema: 'MySchema',
+                                  namespace: 'com.my-namespace')
+    allow(decoder).to(receive(:decode)) { |payload| payload }
+    decoder
+  end
+
+  it 'should decode a key' do
+    # reset stub from TestHelpers
+    allow(described_class).to receive(:new).and_call_original
+    expect(decoder.decode_key({ 'test_id' => '123' }, 'test_id')).to eq('123')
+  end
+
+end

data/spec/avro_data_encoder_spec.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'avro_turf/messaging'
+
+describe Deimos::AvroDataEncoder do
+
+  let(:encoder) do
+    encoder = described_class.new(schema: 'MySchema',
+                                  namespace: 'com.my-namespace')
+    allow(encoder).to(receive(:encode)) { |payload| payload }
+    encoder
+  end
+
+  specify 'generate_key_schema' do
+    expect_any_instance_of(AvroTurf::SchemaStore).
+      to receive(:add_schema).with(
+        'type' => 'record',
+        'name' => 'MySchema_key',
+        'namespace' => 'com.my-namespace',
+        'doc' => 'Key for com.my-namespace.MySchema',
+        'fields' => [
+          {
+            'name' => 'test_id',
+            'type' => 'string'
+          }
+        ]
+      )
+    encoder.send(:_generate_key_schema, 'test_id')
+  end
+
+  it 'should encode a key' do
+    # reset stub from TestHelpers
+    allow(described_class).to receive(:new).and_call_original
+    expect(encoder.encode_key('test_id', '123')).to eq('test_id' => '123')
+  end
+
+end

data/spec/backends/db_spec.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+each_db_config(Deimos::Backends::Db) do
+  include_context 'with publish_backend'
+
+  it 'should save to the database' do
+    described_class.publish(producer_class: MyProducer, messages: messages)
+    records = Deimos::KafkaMessage.all
+    expect(records.size).to eq(3)
+    expect(records[0].attributes.to_h).to include(
+      'message' => '{"foo"=>1}',
+      'topic' => 'my-topic',
+      'key' => 'foo1'
+    )
+    expect(records[1].attributes.to_h).to include(
+      'message' => '{"foo"=>2}',
+      'topic' => 'my-topic',
+      'key' => 'foo2'
+    )
+    expect(records[2].attributes.to_h).to include(
+      'message' => '{"foo"=>3}',
+      'topic' => 'my-topic',
+      'key' => 'foo3'
+    )
+  end
+  it 'should add to non-keyed messages' do
+    described_class.publish(producer_class: MyNoKeyProducer,
+                            messages: messages)
+    expect(Deimos::KafkaMessage.count).to eq(3)
+    described_class.publish(producer_class: MyNoKeyProducer,
+                            messages: [messages.first])
+    expect(Deimos::KafkaMessage.count).to eq(4)
+
+  end
+end

data/spec/backends/kafka_async_spec.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+RSpec.describe Deimos::Backends::KafkaAsync do
+  include_context 'with publish_backend'
+  it 'should publish to Kafka asynchronously' do
+    producer = instance_double(Phobos::Producer::ClassMethods::PublicAPI)
+    expect(producer).to receive(:async_publish_list).with(messages.map(&:encoded_hash))
+    expect(described_class).to receive(:producer).and_return(producer)
+    described_class.publish(producer_class: MyProducer, messages: messages)
+  end
+end

data/spec/backends/kafka_spec.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+RSpec.describe Deimos::Backends::Kafka do
+  include_context 'with publish_backend'
+  it 'should publish to Kafka synchronously' do
+    producer = instance_double(Phobos::Producer::ClassMethods::PublicAPI)
+    expect(producer).to receive(:publish_list).with(messages.map(&:encoded_hash))
+    expect(described_class).to receive(:producer).and_return(producer)
+    described_class.publish(producer_class: MyProducer, messages: messages)
+  end
+end

data/spec/consumer_spec.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+# :nodoc:
+module ConsumerTest
+  describe Deimos::Consumer do
+
+    prepend_before(:each) do
+      # :nodoc:
+      consumer_class = Class.new(Deimos::Consumer) do
+        schema 'MySchema'
+        namespace 'com.my-namespace'
+        key_config field: 'test_id'
+
+        # :nodoc:
+        def consume(_payload, _metadata)
+          raise 'This should not be called unless call_original is set'
+        end
+      end
+      stub_const('ConsumerTest::MyConsumer', consumer_class)
+    end
+
+    it 'should consume a message' do
+      test_consume_message(MyConsumer,
+                           'test_id' => 'foo',
+                           'some_int' => 123) do |payload, _metadata|
+                             expect(payload['test_id']).to eq('foo')
+                           end
+    end
+
+    it 'should consume a message on a topic' do
+      test_consume_message('my_consume_topic',
+                           'test_id' => 'foo',
+                           'some_int' => 123) do |payload, _metadata|
+                             expect(payload['test_id']).to eq('foo')
+                           end
+    end
+
+    it 'should fail on invalid message' do
+      test_consume_invalid_message(MyConsumer, 'invalid' => 'key')
+    end
+
+    it 'should fail on message with extra fields' do
+      test_consume_invalid_message(MyConsumer,
+                                   'test_id' => 'foo',
+                                   'some_int' => 123,
+                                   'extra_field' => 'field name')
+    end
+
+    it 'should not fail when before_consume fails without reraising errors' do
+      Deimos.configure { |config| config.reraise_consumer_errors = false }
+      expect {
+        test_consume_message(
+          MyConsumer,
+          { 'test_id' => 'foo',
+            'some_int' => 123 },
+          { skip_expectation: true }
+        ) { raise 'OH NOES' }
+      } .not_to raise_error
+    end
+
+    it 'should not fail when consume fails without reraising errors' do
+      Deimos.configure { |config| config.reraise_consumer_errors = false }
+      expect {
+        test_consume_message(
+          MyConsumer,
+          { 'invalid' => 'key' },
+          { skip_expectation: true }
+        )
+      } .not_to raise_error
+    end
+
+    it 'should call original' do
+      expect {
+        test_consume_message(MyConsumer,
+                             { 'test_id' => 'foo', 'some_int' => 123 },
+                             { call_original: true })
+      }.to raise_error('This should not be called unless call_original is set')
+    end
+
+    describe 'decode_key' do
+
+      it 'should use the key field in the value if set' do
+        # actual decoding is disabled in test
+        expect(MyConsumer.new.decode_key('test_id' => '123')).to eq('123')
+        expect { MyConsumer.new.decode_key(123) }.to raise_error(NoMethodError)
+      end
+
+      it 'should use the key schema if set' do
+        consumer_class = Class.new(Deimos::Consumer) do
+          schema 'MySchema'
+          namespace 'com.my-namespace'
+          key_config schema: 'MySchema_key'
+        end
+        stub_const('ConsumerTest::MySchemaConsumer', consumer_class)
+        expect(MyConsumer.new.decode_key('test_id' => '123')).to eq('123')
+        expect { MyConsumer.new.decode_key(123) }.to raise_error(NoMethodError)
+      end
+
+      it 'should not encode if plain is set' do
+        consumer_class = Class.new(Deimos::Consumer) do
+          schema 'MySchema'
+          namespace 'com.my-namespace'
+          key_config plain: true
+        end
+        stub_const('ConsumerTest::MyNonEncodedConsumer', consumer_class)
+        expect(MyNonEncodedConsumer.new.decode_key('123')).to eq('123')
+      end
+
+      it 'should error with nothing set' do
+        consumer_class = Class.new(Deimos::Consumer) do
+          schema 'MySchema'
+          namespace 'com.my-namespace'
+        end
+        stub_const('ConsumerTest::MyErrorConsumer', consumer_class)
+        expect { MyErrorConsumer.new.decode_key('123') }.
+          to raise_error('No key config given - if you are not decoding keys, please use `key_config plain: true`')
+      end
+
+    end
+
+    describe 'timestamps' do
+      before(:each) do
+        # :nodoc:
+        consumer_class = Class.new(Deimos::Consumer) do
+          schema 'MySchemaWithDateTimes'
+          namespace 'com.my-namespace'
+          key_config plain: true
+
+          # :nodoc:
+          def consume(_payload, _metadata)
+            raise 'This should not be called unless call_original is set'
+          end
+        end
+        stub_const('ConsumerTest::MyConsumer', consumer_class)
+        stub_consumer(consumer_class)
+      end
+
+      it 'should consume a message' do
+        expect(Deimos.config.metrics).to receive(:histogram).twice
+        test_consume_message('my_consume_topic',
+                             'test_id' => 'foo',
+                             'some_int' => 123,
+                             'updated_at' => Time.now.to_i,
+                             'timestamp' => 2.minutes.ago.to_s) do |payload, _metadata|
+                               expect(payload['test_id']).to eq('foo')
+                             end
+      end
+
+      it 'should fail nicely when timestamp wrong format' do
+        expect(Deimos.config.metrics).to receive(:histogram).twice
+        test_consume_message('my_consume_topic',
+                             'test_id' => 'foo',
+                             'some_int' => 123,
+                             'updated_at' => Time.now.to_i,
+                             'timestamp' => 'dffdf') do |payload, _metadata|
+                               expect(payload['test_id']).to eq('foo')
+                             end
+        test_consume_message('my_consume_topic',
+                             'test_id' => 'foo',
+                             'some_int' => 123,
+                             'updated_at' => Time.now.to_i,
+                             'timestamp' => '') do |payload, _metadata|
+                               expect(payload['test_id']).to eq('foo')
+                             end
+      end
+
+    end
+  end
+end