deimos-ruby 1.0.0.pre.beta22

data/spec/spec_helper.rb

@@ -0,0 +1,207 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift(File.expand_path('../lib', __dir__))
+require 'active_record'
+require 'deimos'
+require 'deimos/metrics/mock'
+require 'deimos/tracing/mock'
+require 'deimos/test_helpers'
+require 'active_support/testing/time_helpers'
+
+# Helpers for Executor/DbProducer
+module TestRunners
+  # Execute a block until it stops failing. This is helpful for testing threads
+  # where we need to wait for them to continue but don't want to rely on
+  # sleeping for X seconds, which is crazy brittle and slow.
+  def wait_for
+    start_time = Time.now
+    begin
+      yield
+    rescue Exception # rubocop:disable Lint/RescueException
+      raise if Time.now - start_time > 2 # 2 seconds is probably plenty of time! <_<
+
+      sleep(0.1)
+      retry
+    end
+  end
+
+  # Test runner
+  class TestRunner
+    attr_accessor :id, :started, :stopped, :should_error
+    # :nodoc:
+    def initialize(id=nil)
+      @id = id
+    end
+
+    # :nodoc:
+    def start
+      if @should_error
+        @should_error = false
+        raise 'OH NOES'
+      end
+      @started = true
+    end
+
+    # :nodoc:
+    def stop
+      @stopped = true
+    end
+  end
+end
+
+# :nodoc:
+module DbConfigs
+  # @param payload [Hash]
+  # @param topic [String]
+  # @param key [String]
+  def build_message(payload, topic, key)
+    message = Deimos::Message.new(payload, Deimos::Producer,
+                                  topic: topic, key: key)
+    message.encoded_payload = message.payload
+    message.encoded_key = message.key
+    message
+  end
+
+  DB_OPTIONS = [
+    {
+      adapter: 'postgresql',
+      port: 5432,
+      username: 'postgres',
+      password: 'root',
+      database: 'postgres',
+      host: ENV['PG_HOST'] || 'localhost'
+    },
+    {
+      adapter: 'mysql2',
+      port: 3306,
+      username: 'root',
+      database: 'test',
+      host: ENV['MYSQL_HOST'] || 'localhost'
+    },
+    {
+      adapter: 'sqlite3',
+      database: 'test.sqlite3'
+    } # this one always needs to be last for non-integration tests
+  ].freeze
+
+  # For each config, run some tests.
+  def each_db_config(subject, &block)
+    DB_OPTIONS.each do |options|
+      describe subject, :integration, db_config: options do
+
+        include_context 'with DB'
+        describe options[:adapter] do # rubocop:disable RSpec/EmptyExampleGroup
+          self.instance_eval(&block)
+        end
+      end
+    end
+  end
+
+  # Set up the given database.
+  def setup_db(options)
+    ActiveRecord::Base.establish_connection(options)
+    migration_class_name = 'DbBackendMigration'
+    migration_version = '[5.2]'
+    migration = ERB.new(
+      File.read('lib/generators/deimos/db_backend/templates/migration')
+    ).result(binding)
+    eval(migration) # rubocop:disable Security/Eval
+    ActiveRecord::Migration.new.run(DbBackendMigration, direction: :up)
+
+    ActiveRecord::Base.descendants.each do |klass|
+      klass.reset_sequence_name if klass.respond_to?(:reset_sequence_name)
+      # reset internal variables - terrible hack to trick Rails into doing this
+      table_name = klass.table_name
+      klass.table_name = "#{table_name}2"
+      klass.table_name = table_name
+    end
+  end
+end
+
+RSpec.configure do |config|
+  config.extend(DbConfigs)
+  include DbConfigs
+  config.include TestRunners
+  config.full_backtrace = true
+
+  # true by default for RSpec 4.0
+  config.shared_context_metadata_behavior = :apply_to_host_groups
+
+  config.before(:all) do
+    Time.zone = 'EST'
+    ActiveRecord::Base.logger = Logger.new('/dev/null')
+    ActiveRecord::Base.establish_connection(
+      'adapter' => 'sqlite3',
+      'database' => 'test.sqlite3'
+    )
+  end
+  config.include Deimos::TestHelpers
+  config.include ActiveSupport::Testing::TimeHelpers
+  config.before(:suite) do
+    Time.zone = 'EST'
+    ActiveRecord::Base.logger = Logger.new('/dev/null')
+    setup_db(DbConfigs::DB_OPTIONS.last)
+    Deimos.configure do |fr_config|
+      fr_config.phobos_config_file = File.join(File.dirname(__FILE__), 'phobos.yml')
+      fr_config.schema_path = File.join(File.expand_path(__dir__), 'schemas')
+      fr_config.reraise_consumer_errors = true
+      fr_config.schema_registry_url = ENV['SCHEMA_REGISTRY'] || 'http://localhost:8081'
+      fr_config.seed_broker = ENV['KAFKA_SEED_BROKER'] || 'localhost:9092'
+      fr_config.logger = Logger.new('/dev/null')
+
+      # Use Mock Metrics and Tracing for rspecs
+      fr_config.metrics = Deimos::Metrics::Mock.new
+      fr_config.tracer = Deimos::Tracing::Mock.new
+    end
+  end
+
+  config.before(:each) do |ex|
+    stub_producers_and_consumers! unless ex.metadata[:integration]
+
+    @previous_config = Deimos.config.dup
+    @previous_phobos_config = Phobos.config.dup
+  end
+
+  config.after(:each) do
+    Deimos.config = @previous_config
+    Phobos.instance_variable_set(:@config, @previous_phobos_config)
+  end
+
+end
+
+RSpec.shared_context('with DB') do
+  before(:all) do
+    setup_db(self.class.metadata[:db_config] || DbConfigs::DB_OPTIONS.last)
+  end
+
+  after(:each) do
+    Deimos::KafkaMessage.delete_all
+    Deimos::KafkaTopicInfo.delete_all
+  end
+end
+
+RSpec.shared_context('with publish_backend') do
+  before(:each) do
+    producer_class = Class.new(Deimos::Producer) do
+      schema 'MySchema'
+      namespace 'com.my-namespace'
+      topic 'my-topic'
+      key_config field: 'test_id'
+    end
+    stub_const('MyProducer', producer_class)
+
+    producer_class = Class.new(Deimos::Producer) do
+      schema 'MySchema'
+      namespace 'com.my-namespace'
+      topic 'my-topic'
+      key_config none: true
+    end
+    stub_const('MyNoKeyProducer', producer_class)
+  end
+
+  let(:messages) do
+    (1..3).map do |i|
+      build_message({ foo: i }, 'my-topic', "foo#{i}")
+    end
+  end
+end

data/spec/updateable_schema_store_spec.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+describe AvroTurf::SchemaStore do
+
+  it 'should add an in-memory schema' do
+    schema_store = described_class.new(path: Deimos.config.schema_path)
+    schema_store.load_schemas!
+    found_schema = schema_store.find('MySchema', 'com.my-namespace').as_json
+    expect(found_schema['name']).to eq('MySchema')
+    expect(found_schema['namespace']).to eq('com.my-namespace')
+    expect(found_schema['fields'].size).to eq(2)
+    expect(found_schema['fields'][0]['type']['type_sym']).to eq('string')
+    expect(found_schema['fields'][0]['name']).to eq('test_id')
+    new_schema = {
+      'namespace' => 'com.my-namespace',
+      'name' => 'MyNewSchema',
+      'type' => 'record',
+      'doc' => 'Test schema',
+      'fields' => [
+        {
+          'name' => 'my_id',
+          'type' => 'int',
+          'doc' => 'test int'
+        }
+      ]
+    }
+    schema_store.add_schema(new_schema)
+    found_schema = schema_store.find('MyNewSchema', 'com.my-namespace').
+      as_json
+    expect(found_schema['name']).to eq('MyNewSchema')
+    expect(found_schema['namespace']).to eq('com.my-namespace')
+    expect(found_schema['fields'].size).to eq(1)
+    expect(found_schema['fields'][0]['type']['type_sym']).to eq('int')
+    expect(found_schema['fields'][0]['name']).to eq('my_id')
+  end
+end

data/spec/utils/db_producer_spec.rb

@@ -0,0 +1,259 @@
+# frozen_string_literal: true
+
+each_db_config(Deimos::Utils::DbProducer) do
+  let(:producer) do
+    producer = described_class.new
+    allow(producer).to receive(:sleep)
+    allow(producer).to receive(:producer).and_return(phobos_producer)
+    producer
+  end
+
+  let(:phobos_producer) do
+    pp = instance_double(Phobos::Producer::PublicAPI)
+    allow(pp).to receive(:publish_list)
+    pp
+  end
+
+  before(:each) do
+    stub_const('Deimos::Utils::DbProducer::BATCH_SIZE', 2)
+  end
+
+  specify '#process_next_messages' do
+    expect(producer).to receive(:retrieve_topics).and_return(%w(topic1 topic2))
+    expect(producer).to receive(:process_topic).twice
+    expect(producer).to receive(:sleep).with(0.5)
+    producer.process_next_messages
+  end
+
+  specify '#retrieve_topics' do
+    (1..3).each do |i|
+      Deimos::KafkaMessage.create!(topic: "topic#{i}",
+                                   key: 'blergkey',
+                                   message: 'blerg')
+    end
+    expect(producer.retrieve_topics).
+      to contain_exactly('topic1', 'topic2', 'topic3')
+  end
+
+  specify '#retrieve_messages' do
+    (1..3).each do |i|
+      Deimos::KafkaMessage.create!(topic: 'topic1',
+                                   message: 'blah',
+                                   key: "key#{i}")
+    end
+    stub_const('Deimos::Utils::DbProducer::BATCH_SIZE', 5)
+    producer.current_topic = 'topic1'
+    messages = producer.retrieve_messages
+    expect(messages.size).to eq(3)
+    expect(messages).to all(be_a_kind_of(Deimos::KafkaMessage))
+  end
+
+  describe '#produce_messages' do
+
+    it 'should produce normally' do
+      batch = ['A'] * 1000
+      expect(phobos_producer).to receive(:publish_list).with(batch).once
+      expect(Deimos.config.metrics).to receive(:increment).with('publish',
+                                                                tags: %w(status:success topic:),
+                                                                by: 1000).once
+      producer.produce_messages(batch)
+    end
+
+    it 'should split the batch size on buffer overflow' do
+      class_producer = double(Phobos::Producer::ClassMethods::PublicAPI, # rubocop:disable RSpec/VerifiedDoubles
+                              sync_producer_shutdown: nil)
+      allow(producer.class).to receive(:producer).and_return(class_producer)
+      expect(class_producer).to receive(:sync_producer_shutdown).twice
+      count = 0
+      allow(phobos_producer).to receive(:publish_list) do
+        count += 1
+        raise Kafka::BufferOverflow if count < 3
+      end
+      allow(Deimos.config.metrics).to receive(:increment)
+      batch = ['A'] * 1000
+      producer.produce_messages(batch)
+      expect(phobos_producer).to have_received(:publish_list).with(batch)
+      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 100)
+      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 10).exactly(100).times
+      expect(Deimos.config.metrics).to have_received(:increment).with('publish',
+                                                                      tags: %w(status:success topic:),
+                                                                      by: 10).exactly(100).times
+    end
+
+    it "should raise an error if it can't split any more" do
+      allow(phobos_producer).to receive(:publish_list) do
+        raise Kafka::BufferOverflow
+      end
+      expect(Deimos.config.metrics).not_to receive(:increment)
+      batch = ['A'] * 1000
+      expect { producer.produce_messages(batch) }.to raise_error(Kafka::BufferOverflow)
+      expect(phobos_producer).to have_received(:publish_list).with(batch)
+      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 100).once
+      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 10).once
+      expect(phobos_producer).to have_received(:publish_list).with(['A']).once
+
+    end
+  end
+
+  describe '#process_topic' do
+    before(:each) do
+      producer.id = 'abc'
+    end
+
+    it 'should do nothing if lock fails' do
+      expect(Deimos::KafkaTopicInfo).to receive(:lock).
+        with('my-topic', 'abc').and_return(false)
+      expect(producer).not_to receive(:retrieve_messages)
+      producer.process_topic('my-topic')
+    end
+
+    it 'should complete successfully' do
+      messages = (1..4).map do |i|
+        Deimos::KafkaMessage.new(
+          topic: 'my-topic',
+          message: "mess#{i}",
+          partition_key: "key#{i}"
+        )
+      end
+      expect(Deimos::KafkaTopicInfo).to receive(:lock).
+        with('my-topic', 'abc').and_return(true)
+      expect(producer).to receive(:retrieve_messages).ordered.
+        and_return(messages[0..1])
+      expect(producer).to receive(:produce_messages).ordered.with([
+                                                                    {
+                                                                      payload: 'mess1',
+                                                                      key: nil,
+                                                                      partition_key: 'key1',
+                                                                      topic: 'my-topic'
+                                                                    },
+                                                                    {
+                                                                      payload: 'mess2',
+                                                                      key: nil,
+                                                                      partition_key: 'key2',
+                                                                      topic: 'my-topic'
+                                                                    }
+                                                                  ])
+      expect(producer).to receive(:retrieve_messages).ordered.
+        and_return(messages[2..3])
+      expect(producer).to receive(:produce_messages).ordered.with([
+                                                                    {
+                                                                      payload: 'mess3',
+                                                                      partition_key: 'key3',
+                                                                      key: nil,
+                                                                      topic: 'my-topic'
+                                                                    },
+                                                                    {
+                                                                      payload: 'mess4',
+                                                                      partition_key: 'key4',
+                                                                      key: nil,
+                                                                      topic: 'my-topic'
+                                                                    }
+                                                                  ])
+      expect(producer).to receive(:retrieve_messages).ordered.
+        and_return([])
+      expect(Deimos::KafkaTopicInfo).to receive(:heartbeat).
+        with('my-topic', 'abc').twice
+      expect(Deimos::KafkaTopicInfo).to receive(:clear_lock).
+        with('my-topic', 'abc').once
+      producer.process_topic('my-topic')
+    end
+
+    it 'should register an error if it gets an error' do
+      expect(producer).to receive(:retrieve_messages).and_raise('OH NOES')
+      expect(Deimos::KafkaTopicInfo).to receive(:register_error).
+        with('my-topic', 'abc')
+      expect(producer).not_to receive(:produce_messages)
+      producer.process_topic('my-topic')
+    end
+
+    it 'should move on if it gets a partial batch' do
+      expect(producer).to receive(:retrieve_messages).ordered.
+        and_return([Deimos::KafkaMessage.new(
+          topic: 'my-topic',
+          message: 'mess1'
+        )])
+      expect(producer).to receive(:produce_messages).once
+      producer.process_topic('my-topic')
+    end
+
+  end
+
+  example 'Full integration test' do
+    (1..4).each do |i|
+      (1..2).each do |j|
+        Deimos::KafkaMessage.create!(topic: "topic#{j}",
+                                     message: "mess#{i}",
+                                     partition_key: "key#{i}")
+        Deimos::KafkaMessage.create!(topic: "topic#{j + 2}",
+                                     key: "key#{i}",
+                                     partition_key: "key#{i}",
+                                     message: "mess#{i}")
+      end
+    end
+    allow(producer).to receive(:produce_messages)
+
+    producer.process_next_messages
+    expect(Deimos::KafkaTopicInfo.count).to eq(4)
+    topics = Deimos::KafkaTopicInfo.select('distinct topic').map(&:topic)
+    expect(topics).to contain_exactly('topic1', 'topic2', 'topic3', 'topic4')
+    expect(Deimos::KafkaMessage.count).to eq(0)
+
+    expect(producer).to have_received(:produce_messages).with([
+                                                                {
+                                                                  payload: 'mess1',
+                                                                  partition_key: 'key1',
+                                                                  key: nil,
+                                                                  topic: 'topic1'
+                                                                },
+                                                                {
+                                                                  payload: 'mess2',
+                                                                  key: nil,
+                                                                  partition_key: 'key2',
+                                                                  topic: 'topic1'
+                                                                }
+                                                              ])
+    expect(producer).to have_received(:produce_messages).with([
+                                                                {
+                                                                  payload: 'mess3',
+                                                                  key: nil,
+                                                                  partition_key: 'key3',
+                                                                  topic: 'topic1'
+                                                                },
+                                                                {
+                                                                  payload: 'mess4',
+                                                                  key: nil,
+                                                                  partition_key: 'key4',
+                                                                  topic: 'topic1'
+                                                                }
+                                                              ])
+    expect(producer).to have_received(:produce_messages).with([
+                                                                {
+                                                                  payload: 'mess1',
+                                                                  key: 'key1',
+                                                                  partition_key: 'key1',
+                                                                  topic: 'topic3'
+                                                                },
+                                                                {
+                                                                  payload: 'mess2',
+                                                                  partition_key: 'key2',
+                                                                  key: 'key2',
+                                                                  topic: 'topic3'
+                                                                }
+                                                              ])
+    expect(producer).to have_received(:produce_messages).with([
+                                                                {
+                                                                  payload: 'mess3',
+                                                                  key: 'key3',
+                                                                  partition_key: 'key3',
+                                                                  topic: 'topic3'
+                                                                },
+                                                                {
+                                                                  payload: 'mess4',
+                                                                  partition_key: 'key4',
+                                                                  key: 'key4',
+                                                                  topic: 'topic3'
+                                                                }
+                                                              ])
+  end
+
+end

data/spec/utils/executor_spec.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Deimos::Utils::Executor do
+
+  let(:executor) { described_class.new(runners) }
+  let(:runners) { (1..2).map { |i| TestRunners::TestRunner.new(i) } }
+
+  it 'starts and stops configured runners' do
+    runners.each do |r|
+      expect(r.started).to be_falsey
+      expect(r.stopped).to be_falsey
+    end
+    executor.start
+    wait_for do
+      runners.each do |r|
+        expect(r.started).to be_truthy
+        expect(r.stopped).to be_falsey
+      end
+      executor.stop
+      runners.each do |r|
+        expect(r.started).to be_truthy
+        expect(r.stopped).to be_truthy
+      end
+    end
+  end
+
+  it 'reconnects crashed runners' do
+    allow(executor).to receive(:handle_crashed_runner).and_call_original
+    allow(executor).to receive(:handle_crashed_runner).and_call_original
+    runners.each { |r| r.should_error = true }
+    executor.start
+    wait_for do
+      expect(executor).to have_received(:handle_crashed_runner).with(runners[0], anything, 0).once
+      expect(executor).to have_received(:handle_crashed_runner).with(runners[1], anything, 0).once
+      runners.each { |r| expect(r.started).to be_truthy }
+      executor.stop
+    end
+  end
+
+end

data/spec/utils/lag_reporter_spec.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+describe Deimos::Utils::LagReporter do
+
+  before(:each) do
+    kafka_client = instance_double(Kafka::Client)
+    allow(kafka_client).to receive(:last_offset_for).and_return(100)
+    allow(Phobos).to receive(:create_kafka_client).and_return(kafka_client)
+    Deimos.configure { |c| c.report_lag = true }
+  end
+
+  after(:each) do
+    described_class.reset
+    Deimos.configure { |c| c.report_lag = false }
+  end
+
+  it 'should not report lag before ready' do
+    expect(Deimos.config.metrics).not_to receive(:gauge)
+    ActiveSupport::Notifications.instrument(
+      'heartbeat.consumer.kafka',
+      group_id: 'group1', topic_partitions: { 'my-topic': [1] }
+    )
+
+  end
+
+  it 'should report lag' do
+    expect(Deimos.config.metrics).to receive(:gauge).ordered.twice.
+      with('consumer_lag', 95,
+           tags: %w(
+             consumer_group:group1
+             partition:1
+             topic:my-topic
+           ))
+    expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
+      with('consumer_lag', 80,
+           tags: %w(
+             consumer_group:group1
+             partition:2
+             topic:my-topic
+           ))
+    expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
+      with('consumer_lag', 0,
+           tags: %w(
+             consumer_group:group1
+             partition:2
+             topic:my-topic
+           ))
+    ActiveSupport::Notifications.instrument(
+      'seek.consumer.kafka',
+      offset: 5, topic: 'my-topic', group_id: 'group1', partition: 1
+    )
+    ActiveSupport::Notifications.instrument(
+      'start_process_message.consumer.kafka',
+      offset_lag: 80, topic: 'my-topic', group_id: 'group1', partition: 2
+    )
+    ActiveSupport::Notifications.instrument(
+      'heartbeat.consumer.kafka',
+      group_id: 'group1', topic_partitions: { 'my-topic': [1, 2] }
+    )
+    ActiveSupport::Notifications.instrument(
+      'start_process_batch.consumer.kafka',
+      offset_lag: 0, topic: 'my-topic', group_id: 'group1', partition: 2
+    )
+    ActiveSupport::Notifications.instrument(
+      'heartbeat.consumer.kafka',
+      group_id: 'group1', topic_partitions: { 'my-topic': [1, 2] }
+    )
+  end
+end

data/spec/utils/signal_handler_spec.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+RSpec.describe Deimos::Utils::SignalHandler do
+  describe '#run!' do
+
+    it 'starts and stops the runner' do
+      runner = TestRunners::TestRunner.new
+      expect(runner).to receive(:start)
+      expect(runner).to receive(:stop)
+
+      signal_handler = described_class.new(runner)
+      signal_handler.send(:unblock, described_class::SIGNALS.first)
+      signal_handler.run!
+    end
+  end
+end