deimos-ruby 1.0.0.pre.beta22

data/spec/deimos_spec.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+describe Deimos do
+
+  it 'should have a version number' do
+    expect(Deimos::VERSION).not_to be_nil
+  end
+
+  specify 'configure' do
+    phobos_configuration = { 'logger' =>
+  { 'file' => 'log/phobos.log',
+    'stdout_json' => false,
+    'level' => 'debug',
+    'ruby_kafka' =>
+     { 'level' => 'debug' } },
+                             'kafka' =>
+  { 'client_id' => 'phobos',
+    'connect_timeout' => 15,
+    'socket_timeout' => 15,
+    'seed_brokers' => 'my_seed_broker.com',
+    'ssl_ca_cert' => 'my_ssl_ca_cert',
+    'ssl_client_cert' => 'my_ssl_client_cert',
+    'ssl_client_cert_key' => 'my_ssl_client_cert_key' },
+                             'producer' =>
+  { 'ack_timeout' => 5,
+    'required_acks' => :all,
+    'max_retries' => 2,
+    'retry_backoff' => 1,
+    'max_buffer_size' => 10_000,
+    'max_buffer_bytesize' => 10_000_000,
+    'compression_codec' => nil,
+    'compression_threshold' => 1,
+    'max_queue_size' => 10_000,
+    'delivery_threshold' => 0,
+    'delivery_interval' => 0 },
+                             'consumer' =>
+  { 'session_timeout' => 300,
+    'offset_commit_interval' => 10,
+    'offset_commit_threshold' => 0,
+    'heartbeat_interval' => 10 },
+                             'backoff' =>
+                              { 'min_ms' => 1000,
+                                'max_ms' => 60_000 },
+                             'listeners' => [
+                               { 'handler' => 'ConsumerTest::MyConsumer',
+                                 'topic' => 'my_consume_topic',
+                                 'group_id' => 'my_group_id',
+                                 'max_bytes_per_partition' => 524_288 }
+                             ],
+                             'custom_logger' => nil,
+                             'custom_kafka_logger' => nil }
+
+    expect(Phobos).to receive(:configure).with(phobos_configuration)
+    allow(described_class).to receive(:ssl_var_contents) { |key| key }
+    described_class.configure do |config|
+      config.phobos_config_file = File.join(File.dirname(__FILE__), 'phobos.yml')
+      config.seed_broker = 'my_seed_broker.com'
+      config.ssl_enabled = true
+      config.ssl_ca_cert = 'my_ssl_ca_cert'
+      config.ssl_client_cert = 'my_ssl_client_cert'
+      config.ssl_client_cert_key = 'my_ssl_client_cert_key'
+    end
+  end
+
+  it 'should error if required_acks is not all' do
+    expect {
+      described_class.configure do |config|
+        config.publish_backend = :db
+        config.phobos_config_file = File.join(File.dirname(__FILE__), 'phobos.bad_db.yml')
+      end
+    }.to raise_error('Cannot set publish_backend to :db unless required_acks is set to ":all" in phobos.yml!')
+  end
+
+  describe '#start_db_backend!' do
+    before(:each) do
+      allow(described_class).to receive(:run_db_backend)
+    end
+
+    it 'should start if backend is db and num_producer_threads is > 0' do
+      signal_handler = instance_double(Deimos::Utils::SignalHandler)
+      allow(signal_handler).to receive(:run!)
+      expect(Deimos::Utils::SignalHandler).to receive(:new) do |executor|
+        expect(executor.runners.size).to eq(2)
+        signal_handler
+      end
+      described_class.configure do |config|
+        config.publish_backend = :db
+      end
+      described_class.start_db_backend!(thread_count: 2)
+    end
+
+    it 'should not start if backend is not db' do
+      expect(Deimos::Utils::SignalHandler).not_to receive(:new)
+      described_class.configure do |config|
+        config.publish_backend = :kafka
+      end
+      expect { described_class.start_db_backend!(thread_count: 2) }.
+        to raise_error('Publish backend is not set to :db, exiting')
+    end
+
+    it 'should not start if num_producer_threads is nil' do
+      expect(Deimos::Utils::SignalHandler).not_to receive(:new)
+      described_class.configure do |config|
+        config.publish_backend = :db
+      end
+      expect { described_class.start_db_backend!(thread_count: nil) }.
+        to raise_error('Thread count is not given or set to zero, exiting')
+    end
+
+    it 'should not start if num_producer_threads is 0' do
+      expect(Deimos::Utils::SignalHandler).not_to receive(:new)
+      described_class.configure do |config|
+        config.publish_backend = :db
+      end
+      expect { described_class.start_db_backend!(thread_count: 0) }.
+        to raise_error('Thread count is not given or set to zero, exiting')
+    end
+
+  end
+end

data/spec/kafka_source_spec.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+require 'activerecord-import'
+
+# Wrap in a module so our classes don't leak out afterwards
+module KafkaSourceSpec
+  RSpec.describe Deimos::KafkaSource do
+    before(:all) do
+      ActiveRecord::Base.connection.create_table(:widgets, force: true) do |t|
+        t.integer(:widget_id)
+        t.string(:description)
+        t.string(:model_id, default: '')
+        t.string(:name)
+        t.timestamps
+      end
+      ActiveRecord::Base.connection.add_index(:widgets, :widget_id)
+
+      # Dummy producer which mimicks the behavior of a real producer
+      class WidgetProducer < Deimos::ActiveRecordProducer
+        topic 'my-topic'
+        namespace 'com.my-namespace'
+        schema 'Widget'
+        key_config field: :id
+      end
+
+      # Dummy producer which mimicks the behavior of a real producer
+      class WidgetProducerTheSecond < Deimos::ActiveRecordProducer
+        topic 'my-topic-the-second'
+        namespace 'com.my-namespace'
+        schema 'WidgetTheSecond'
+        key_config field: :id
+      end
+
+      # Dummy class we can include the mixin in. Has a backing table created
+      # earlier.
+      class Widget < ActiveRecord::Base
+        include Deimos::KafkaSource
+
+        # :nodoc:
+        def self.kafka_producers
+          [WidgetProducer, WidgetProducerTheSecond]
+        end
+      end
+      Widget.reset_column_information
+
+    end
+
+    after(:all) do
+      ActiveRecord::Base.connection.drop_table(:widgets)
+    end
+
+    before(:each) do
+      Widget.delete_all
+    end
+
+    it 'should send events on creation, update, and deletion' do
+      widget = Widget.create!(widget_id: 1, name: 'widget')
+      expect('my-topic').to have_sent({
+                                        widget_id: 1,
+                                        name: 'widget',
+                                        id: widget.id,
+                                        created_at: anything,
+                                        updated_at: anything
+                                      }, 1)
+      expect('my-topic-the-second').to have_sent({
+                                                   widget_id: 1,
+                                                   model_id: '',
+                                                   id: widget.id,
+                                                   created_at: anything,
+                                                   updated_at: anything
+                                                 }, 1)
+      widget.update_attribute(:name, 'widget 2')
+      expect('my-topic').to have_sent({
+                                        widget_id: 1,
+                                        name: 'widget 2',
+                                        id: widget.id,
+                                        created_at: anything,
+                                        updated_at: anything
+                                      }, 1)
+      expect('my-topic-the-second').to have_sent({
+                                                   widget_id: 1,
+                                                   model_id: '',
+                                                   id: widget.id,
+                                                   created_at: anything,
+                                                   updated_at: anything
+                                                 }, 1)
+      widget.destroy
+      expect('my-topic').to have_sent(nil, 1)
+      expect('my-topic-the-second').to have_sent(nil, 1)
+    end
+
+    it 'should send events on import' do
+      widgets = (1..3).map do |i|
+        Widget.new(widget_id: i, name: "Widget #{i}")
+      end
+      Widget.import(widgets)
+      widgets = Widget.all
+      expect('my-topic').to have_sent({
+                                        widget_id: 1,
+                                        name: 'Widget 1',
+                                        id: widgets[0].id,
+                                        created_at: anything,
+                                        updated_at: anything
+                                      }, widgets[0].id)
+      expect('my-topic').to have_sent({
+                                        widget_id: 2,
+                                        name: 'Widget 2',
+                                        id: widgets[1].id,
+                                        created_at: anything,
+                                        updated_at: anything
+                                      }, widgets[1].id)
+      expect('my-topic').to have_sent({
+                                        widget_id: 3,
+                                        name: 'Widget 3',
+                                        id: widgets[2].id,
+                                        created_at: anything,
+                                        updated_at: anything
+                                      }, widgets[2].id)
+    end
+
+    it 'should send events even if the save fails' do
+      widget = Widget.create!(widget_id: 1, name: 'widget')
+      expect('my-topic').to have_sent({
+                                        widget_id: 1,
+                                        name: widget.name,
+                                        id: widget.id,
+                                        created_at: anything,
+                                        updated_at: anything
+                                      }, widget.id)
+      clear_kafka_messages!
+      Widget.transaction do
+        widget.update_attribute(:name, 'widget 3')
+        raise ActiveRecord::Rollback
+      end
+      expect('my-topic').to have_sent(anything)
+    end
+
+    it 'should not send events if an unrelated field changes' do
+      widget = Widget.create!(widget_id: 1, name: 'widget')
+      clear_kafka_messages!
+      widget.update_attribute(:description, 'some description')
+      expect('my-topic').not_to have_sent(anything)
+    end
+
+    context 'with DB backend' do
+      before(:each) do
+        Deimos.configure do |config|
+          config.publish_backend = :db
+        end
+        setup_db(DB_OPTIONS.last) # sqlite
+        allow(Deimos::Producer).to receive(:produce_batch).and_call_original
+      end
+
+      it 'should save to the DB' do
+        Widget.create!(widget_id: 1, name: 'widget')
+        expect(Deimos::KafkaMessage.count).to eq(2) # 2 producers
+      end
+
+      it 'should not save with a rollback' do
+        Widget.transaction do
+          Widget.create!(widget_id: 1, name: 'widget')
+          raise ActiveRecord::Rollback
+        end
+        expect(Deimos::KafkaMessage.count).to eq(0)
+      end
+    end
+  end
+end

data/spec/kafka_topic_info_spec.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+each_db_config(Deimos::KafkaTopicInfo) do
+
+  it 'should lock the topic' do
+    expect(described_class.lock('my-topic', 'abc')).to be_truthy
+    expect(described_class.lock('my-topic', 'def')).to be_falsey
+    expect(described_class.lock('my-topic2', 'def')).to be_truthy
+    expect(described_class.count).to eq(2)
+    expect(described_class.first.locked_by).to eq('abc')
+    expect(described_class.last.locked_by).to eq('def')
+  end
+
+  it "should lock the topic if it's old" do
+    described_class.create!(topic: 'my-topic', locked_by: 'abc', error: true,
+                            locked_at: 2.minutes.ago)
+    expect(described_class.lock('my-topic', 'abc')).to be_truthy
+    expect(described_class.count).to eq(1)
+    expect(described_class.first.locked_by).to eq('abc')
+
+  end
+
+  it "should lock the topic if it's not currently locked" do
+    described_class.create!(topic: 'my-topic', locked_by: nil,
+                            locked_at: nil)
+    expect(described_class.lock('my-topic', 'abc')).to be_truthy
+    expect(described_class.count).to eq(1)
+    expect(described_class.first.locked_by).to eq('abc')
+  end
+
+  it "should not lock the topic if it's errored" do
+    described_class.create!(topic: 'my-topic', locked_by: nil,
+                            locked_at: nil, error: true)
+    expect(described_class.lock('my-topic', 'abc')).to be_falsey
+    expect(described_class.count).to eq(1)
+    expect(described_class.first.locked_by).to eq(nil)
+  end
+
+  specify '#clear_lock' do
+    described_class.create!(topic: 'my-topic', locked_by: 'abc',
+                            locked_at: 10.seconds.ago, error: true, retries: 1)
+    described_class.create!(topic: 'my-topic2', locked_by: 'def',
+                            locked_at: 10.seconds.ago, error: true, retries: 1)
+    described_class.clear_lock('my-topic', 'abc')
+    expect(described_class.count).to eq(2)
+    record = described_class.first
+    expect(record.locked_by).to eq(nil)
+    expect(record.locked_at).to eq(nil)
+    expect(record.error).to eq(false)
+    expect(record.retries).to eq(0)
+    record = described_class.last
+    expect(record.locked_by).not_to eq(nil)
+    expect(record.locked_at).not_to eq(nil)
+    expect(record.error).not_to eq(false)
+    expect(record.retries).not_to eq(0)
+  end
+
+  specify '#register_error' do
+    freeze_time do
+      described_class.create!(topic: 'my-topic', locked_by: 'abc',
+                              locked_at: 10.seconds.ago)
+      described_class.create!(topic: 'my-topic2', locked_by: 'def',
+                              locked_at: 10.seconds.ago, error: true, retries: 1)
+      described_class.register_error('my-topic', 'abc')
+      record = described_class.first
+      expect(record.locked_by).to be_nil
+      expect(record.locked_at).to eq(Time.zone.now)
+      expect(record.error).to be_truthy
+      expect(record.retries).to eq(1)
+
+      described_class.register_error('my-topic2', 'def')
+      record = described_class.last
+      expect(record.error).to be_truthy
+      expect(record.retries).to eq(2)
+      expect(record.locked_at).to eq(Time.zone.now)
+    end
+  end
+
+  specify '#heartbeat' do
+    freeze_time do
+      described_class.create!(topic: 'my-topic', locked_by: 'abc',
+                              locked_at: 10.seconds.ago)
+      described_class.heartbeat('my-topic', 'abc')
+      expect(described_class.last.locked_at).to eq(Time.zone.now)
+    end
+  end
+
+end

data/spec/phobos.bad_db.yml

@@ -0,0 +1,73 @@
+logger:
+  # Optional log file, set to false or remove to disable it
+  file: log/phobos.log
+  # Optional output format for stdout, default is false (human readable).
+  # Set to true to enable json output.
+  stdout_json: false
+  level: debug
+  # Comment the block to disable ruby-kafka logs
+  ruby_kafka:
+    level: debug
+
+kafka:
+  # identifier for this application
+  client_id: phobos
+  # timeout setting for connecting to brokers
+  connect_timeout: 15
+  # timeout setting for socket connections
+  socket_timeout: 15
+
+producer:
+  # number of seconds a broker can wait for replicas to acknowledge
+  # a write before responding with a timeout
+  ack_timeout: 5
+  # number of replicas that must acknowledge a write, or `:all`
+  # if all in-sync replicas must acknowledge
+  required_acks: 1
+  # number of retries that should be attempted before giving up sending
+  # messages to the cluster. Does not include the original attempt
+  max_retries: 2
+  # number of seconds to wait between retries
+  retry_backoff: 1
+  # number of messages allowed in the buffer before new writes will
+  # raise {BufferOverflow} exceptions
+  max_buffer_size: 10000
+  # maximum size of the buffer in bytes. Attempting to produce messages
+  # when the buffer reaches this size will result in {BufferOverflow} being raised
+  max_buffer_bytesize: 10000000
+  # name of the compression codec to use, or nil if no compression should be performed.
+  # Valid codecs: `:snappy` and `:gzip`
+  compression_codec:
+  # number of messages that needs to be in a message set before it should be compressed.
+  # Note that message sets are per-partition rather than per-topic or per-producer
+  compression_threshold: 1
+  # maximum number of messages allowed in the queue. Only used for async_producer
+  max_queue_size: 10000
+  # if greater than zero, the number of buffered messages that will automatically
+  # trigger a delivery. Only used for async_producer
+  delivery_threshold: 0
+  # if greater than zero, the number of seconds between automatic message
+  # deliveries. Only used for async_producer
+  delivery_interval: 0
+
+consumer:
+  # number of seconds after which, if a client hasn't contacted the Kafka cluster,
+  # it will be kicked out of the group
+  session_timeout: 300
+  # interval between offset commits, in seconds
+  offset_commit_interval: 10
+  # number of messages that can be processed before their offsets are committed.
+  # If zero, offset commits are not triggered by message processing
+  offset_commit_threshold: 0
+  # interval between heartbeats; must be less than the session window
+  heartbeat_interval: 10
+
+backoff:
+  min_ms: 1000
+  max_ms: 60000
+
+listeners:
+  - handler: ConsumerTest::MyConsumer
+    topic: my_consume_topic
+    group_id: my_group_id
+    max_bytes_per_partition: 524288 # 512 KB

data/spec/phobos.yml

@@ -0,0 +1,73 @@
+logger:
+  # Optional log file, set to false or remove to disable it
+  file: log/phobos.log
+  # Optional output format for stdout, default is false (human readable).
+  # Set to true to enable json output.
+  stdout_json: false
+  level: debug
+  # Comment the block to disable ruby-kafka logs
+  ruby_kafka:
+    level: debug
+
+kafka:
+  # identifier for this application
+  client_id: phobos
+  # timeout setting for connecting to brokers
+  connect_timeout: 15
+  # timeout setting for socket connections
+  socket_timeout: 15
+
+producer:
+  # number of seconds a broker can wait for replicas to acknowledge
+  # a write before responding with a timeout
+  ack_timeout: 5
+  # number of replicas that must acknowledge a write, or `:all`
+  # if all in-sync replicas must acknowledge
+  required_acks: :all
+  # number of retries that should be attempted before giving up sending
+  # messages to the cluster. Does not include the original attempt
+  max_retries: 2
+  # number of seconds to wait between retries
+  retry_backoff: 1
+  # number of messages allowed in the buffer before new writes will
+  # raise {BufferOverflow} exceptions
+  max_buffer_size: 10000
+  # maximum size of the buffer in bytes. Attempting to produce messages
+  # when the buffer reaches this size will result in {BufferOverflow} being raised
+  max_buffer_bytesize: 10000000
+  # name of the compression codec to use, or nil if no compression should be performed.
+  # Valid codecs: `:snappy` and `:gzip`
+  compression_codec:
+  # number of messages that needs to be in a message set before it should be compressed.
+  # Note that message sets are per-partition rather than per-topic or per-producer
+  compression_threshold: 1
+  # maximum number of messages allowed in the queue. Only used for async_producer
+  max_queue_size: 10000
+  # if greater than zero, the number of buffered messages that will automatically
+  # trigger a delivery. Only used for async_producer
+  delivery_threshold: 0
+  # if greater than zero, the number of seconds between automatic message
+  # deliveries. Only used for async_producer
+  delivery_interval: 0
+
+consumer:
+  # number of seconds after which, if a client hasn't contacted the Kafka cluster,
+  # it will be kicked out of the group
+  session_timeout: 300
+  # interval between offset commits, in seconds
+  offset_commit_interval: 10
+  # number of messages that can be processed before their offsets are committed.
+  # If zero, offset commits are not triggered by message processing
+  offset_commit_threshold: 0
+  # interval between heartbeats; must be less than the session window
+  heartbeat_interval: 10
+
+backoff:
+  min_ms: 1000
+  max_ms: 60000
+
+listeners:
+  - handler: ConsumerTest::MyConsumer
+    topic: my_consume_topic
+    group_id: my_group_id
+    max_bytes_per_partition: 524288 # 512 KB