deimos-temp-fork 0.0.1

data/spec/consumer_spec.rb

@@ -0,0 +1,209 @@
+# frozen_string_literal: true
+
+# :nodoc:
+module ConsumerTest
+  describe Deimos::Consumer, 'Message Consumer' do
+
+    prepend_before(:each) do
+      # :nodoc:
+      consumer_class = Class.new(described_class) do
+        schema 'MySchema'
+        namespace 'com.my-namespace'
+        key_config field: 'test_id'
+
+        # :nodoc:
+        def fatal_error?(_exception, payload, _metadata)
+          payload == 'fatal'
+        end
+
+        # :nodoc:
+        def consume(_payload, _metadata)
+          raise 'This should not be called unless call_original is set'
+        end
+      end
+      stub_const('ConsumerTest::MyConsumer', consumer_class)
+    end
+
+    it 'should consume a message' do
+      test_consume_message(MyConsumer,
+                           'test_id' => 'foo',
+                           'some_int' => 123) do |payload, _metadata|
+                             expect(payload['test_id']).to eq('foo')
+                           end
+    end
+
+    it 'should consume a nil message' do
+      test_consume_message(MyConsumer, nil) do |payload, _metadata|
+        expect(payload).to be_nil
+      end
+    end
+
+    it 'should consume a message idempotently' do
+      # testing for a crash and re-consuming the same message/metadata
+      key = { 'test_id' => 'foo' }
+      test_metadata = { key: key }
+      allow_any_instance_of(MyConsumer).to(receive(:decode_key)) do |_instance, k|
+        k['test_id']
+      end
+      MyConsumer.new.around_consume({ 'test_id' => 'foo',
+                                      'some_int' => 123 }, test_metadata) do |_payload, metadata|
+                                        expect(metadata[:key]).to eq('foo')
+                                      end
+      MyConsumer.new.around_consume({ 'test_id' => 'foo',
+                                      'some_int' => 123 }, test_metadata) do |_payload, metadata|
+                                        expect(metadata[:key]).to eq('foo')
+                                      end
+    end
+
+    it 'should consume a message on a topic' do
+      test_consume_message('my_consume_topic',
+                           'test_id' => 'foo',
+                           'some_int' => 123) do |payload, _metadata|
+                             expect(payload['test_id']).to eq('foo')
+                           end
+    end
+
+    it 'should fail on invalid message' do
+      test_consume_invalid_message(MyConsumer, 'invalid' => 'key')
+    end
+
+    it 'should fail if reraise is false but fatal_error is true' do
+      Deimos.configure { |config| config.consumers.reraise_errors = false }
+      test_consume_invalid_message(MyConsumer, 'fatal')
+    end
+
+    it 'should fail if fatal_error is true globally' do
+      Deimos.configure do |config|
+        config.consumers.fatal_error = proc { true }
+        config.consumers.reraise_errors = false
+      end
+      test_consume_invalid_message(MyConsumer, 'invalid' => 'key')
+    end
+
+    it 'should fail on message with extra fields' do
+      test_consume_invalid_message(MyConsumer,
+                                   'test_id' => 'foo',
+                                   'some_int' => 123,
+                                   'extra_field' => 'field name')
+    end
+
+    it 'should not fail when before_consume fails without reraising errors' do
+      Deimos.configure { |config| config.consumers.reraise_errors = false }
+      expect {
+        test_consume_message(
+          MyConsumer,
+          { 'test_id' => 'foo',
+            'some_int' => 123 },
+          { skip_expectation: true }
+        ) { raise 'OH NOES' }
+      }.not_to raise_error
+    end
+
+    it 'should not fail when consume fails without reraising errors' do
+      Deimos.configure { |config| config.consumers.reraise_errors = false }
+      expect {
+        test_consume_message(
+          MyConsumer,
+          { 'invalid' => 'key' },
+          { skip_expectation: true }
+        )
+      }.not_to raise_error
+    end
+
+    it 'should call original' do
+      expect {
+        test_consume_message(MyConsumer,
+                             { 'test_id' => 'foo', 'some_int' => 123 },
+                             { call_original: true })
+      }.to raise_error('This should not be called unless call_original is set')
+    end
+
+    describe 'decode_key' do
+
+      it 'should use the key field in the value if set' do
+        # actual decoding is disabled in test
+        expect(MyConsumer.new.decode_key('test_id' => '123')).to eq('123')
+        expect { MyConsumer.new.decode_key(123) }.to raise_error(NoMethodError)
+      end
+
+      it 'should use the key schema if set' do
+        consumer_class = Class.new(described_class) do
+          schema 'MySchema'
+          namespace 'com.my-namespace'
+          key_config schema: 'MySchema_key'
+        end
+        stub_const('ConsumerTest::MySchemaConsumer', consumer_class)
+        expect(MyConsumer.new.decode_key('test_id' => '123')).to eq('123')
+        expect { MyConsumer.new.decode_key(123) }.to raise_error(NoMethodError)
+      end
+
+      it 'should not decode if plain is set' do
+        consumer_class = Class.new(described_class) do
+          schema 'MySchema'
+          namespace 'com.my-namespace'
+          key_config plain: true
+        end
+        stub_const('ConsumerTest::MyNonEncodedConsumer', consumer_class)
+        expect(MyNonEncodedConsumer.new.decode_key('123')).to eq('123')
+      end
+
+      it 'should error with nothing set' do
+        consumer_class = Class.new(described_class) do
+          schema 'MySchema'
+          namespace 'com.my-namespace'
+        end
+        stub_const('ConsumerTest::MyErrorConsumer', consumer_class)
+        expect { MyErrorConsumer.new.decode_key('123') }.
+          to raise_error('No key config given - if you are not decoding keys, please use `key_config plain: true`')
+      end
+
+    end
+
+    describe 'timestamps' do
+      before(:each) do
+        # :nodoc:
+        consumer_class = Class.new(described_class) do
+          schema 'MySchemaWithDateTimes'
+          namespace 'com.my-namespace'
+          key_config plain: true
+
+          # :nodoc:
+          def consume(_payload, _metadata)
+            raise 'This should not be called unless call_original is set'
+          end
+        end
+        stub_const('ConsumerTest::MyConsumer', consumer_class)
+      end
+
+      it 'should consume a message' do
+        expect(Deimos.config.metrics).to receive(:histogram).twice
+        test_consume_message('my_consume_topic',
+                             'test_id' => 'foo',
+                             'some_int' => 123,
+                             'updated_at' => Time.now.to_i,
+                             'timestamp' => 2.minutes.ago.to_s) do |payload, _metadata|
+                               expect(payload['test_id']).to eq('foo')
+                             end
+      end
+
+      it 'should fail nicely when timestamp wrong format' do
+        expect(Deimos.config.metrics).to receive(:histogram).twice
+        test_consume_message('my_consume_topic',
+                             'test_id' => 'foo',
+                             'some_int' => 123,
+                             'updated_at' => Time.now.to_i,
+                             'timestamp' => 'dffdf') do |payload, _metadata|
+                               expect(payload['test_id']).to eq('foo')
+                             end
+        test_consume_message('my_consume_topic',
+                             'test_id' => 'foo',
+                             'some_int' => 123,
+                             'updated_at' => Time.now.to_i,
+                             'timestamp' => '') do |payload, _metadata|
+                               expect(payload['test_id']).to eq('foo')
+                             end
+      end
+
+    end
+  end
+end

data/spec/deimos_spec.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+describe Deimos do
+
+  let(:phobos_configuration) do
+    { 'logger' =>
+        { 'file' => 'log/phobos.log',
+          'stdout_json' => false,
+          'level' => 'debug',
+          'ruby_kafka' =>
+              { 'level' => 'debug' } },
+      'kafka' =>
+        { 'client_id' => 'phobos',
+          'connect_timeout' => 15,
+          'socket_timeout' => 15,
+          'seed_brokers' => 'my_seed_broker.com',
+          'ssl_ca_cert' => 'my_ssl_ca_cert',
+          'ssl_client_cert' => 'my_ssl_client_cert',
+          'ssl_client_cert_key' => 'my_ssl_client_cert_key' },
+      'producer' =>
+        { 'ack_timeout' => 5,
+          'required_acks' => :all,
+          'max_retries' => 2,
+          'retry_backoff' => 1,
+          'max_buffer_size' => 10_000,
+          'max_buffer_bytesize' => 10_000_000,
+          'compression_codec' => nil,
+          'compression_threshold' => 1,
+          'max_queue_size' => 10_000,
+          'delivery_threshold' => 0,
+          'delivery_interval' => 0 },
+      'consumer' =>
+        { 'session_timeout' => 300,
+          'offset_commit_interval' => 10,
+          'offset_commit_threshold' => 0,
+          'heartbeat_interval' => 10 },
+      'backoff' =>
+        { 'min_ms' => 1000,
+          'max_ms' => 60_000 },
+      'listeners' => [
+        { 'handler' => 'ConsumerTest::MyConsumer',
+          'topic' => 'my_consume_topic',
+          'group_id' => 'my_group_id',
+          'max_bytes_per_partition' => 524_288 },
+        { 'handler' => 'ConsumerTest::MyBatchConsumer',
+          'topic' => 'my_batch_consume_topic',
+          'group_id' => 'my_batch_group_id',
+          'delivery' => 'inline_batch' }
+      ],
+      'custom_logger' => nil,
+      'custom_kafka_logger' => nil }
+  end
+
+  let(:config_path) { File.join(File.dirname(__FILE__), 'phobos.yml') }
+
+  it 'should have a version number' do
+    expect(Deimos::VERSION).not_to be_nil
+  end
+
+  it 'should error if required_acks is not all' do
+    expect {
+      described_class.configure do |config|
+        config.producers.backend = :db
+        config.phobos_config_file = File.join(File.dirname(__FILE__), 'phobos.bad_db.yml')
+      end
+    }.to raise_error('Cannot set producers.backend to :db unless producers.required_acks is set to ":all"!')
+  end
+
+  describe '#start_db_backend!' do
+    it 'should start if backend is db and thread_count is > 0' do
+      signal_handler = instance_double(Sigurd::SignalHandler)
+      allow(signal_handler).to receive(:run!)
+      expect(Sigurd::Executor).to receive(:new).
+        with(anything, sleep_seconds: 5, logger: anything).and_call_original
+      expect(Sigurd::SignalHandler).to receive(:new) do |executor|
+        expect(executor.runners.size).to eq(2)
+        signal_handler
+      end
+      described_class.configure do |config|
+        config.producers.backend = :db
+      end
+      described_class.start_db_backend!(thread_count: 2)
+    end
+
+    it 'should not start if backend is not db' do
+      expect(Sigurd::SignalHandler).not_to receive(:new)
+      described_class.configure do |config|
+        config.producers.backend = :kafka
+      end
+      expect { described_class.start_db_backend!(thread_count: 2) }.
+        to raise_error('Publish backend is not set to :db, exiting')
+    end
+
+    it 'should not start if thread_count is nil' do
+      expect(Sigurd::SignalHandler).not_to receive(:new)
+      described_class.configure do |config|
+        config.producers.backend = :db
+      end
+      expect { described_class.start_db_backend!(thread_count: nil) }.
+        to raise_error('Thread count is not given or set to zero, exiting')
+    end
+
+    it 'should not start if thread_count is 0' do
+      expect(Sigurd::SignalHandler).not_to receive(:new)
+      described_class.configure do |config|
+        config.producers.backend = :db
+      end
+      expect { described_class.start_db_backend!(thread_count: 0) }.
+        to raise_error('Thread count is not given or set to zero, exiting')
+    end
+  end
+
+  describe 'delivery configuration' do
+    before(:each) do
+      allow(YAML).to receive(:load).and_return(phobos_configuration)
+    end
+
+    it 'should not raise an error with properly configured handlers' do
+      expect {
+        described_class.configure do
+          consumer do
+            class_name 'ConsumerTest::MyConsumer'
+            delivery :message
+          end
+          consumer do
+            class_name 'ConsumerTest::MyConsumer'
+            delivery :batch
+          end
+          consumer do
+            class_name 'ConsumerTest::MyBatchConsumer'
+            delivery :inline_batch
+          end
+        end
+      }.not_to raise_error
+    end
+
+    it 'should raise an error if inline_batch listeners do not implement consume_batch' do
+      expect {
+        described_class.configure do
+          consumer do
+            class_name 'ConsumerTest::MyConsumer'
+            delivery :inline_batch
+          end
+        end
+      }.to raise_error('BatchConsumer ConsumerTest::MyConsumer does not implement `consume_batch`')
+    end
+
+    it 'should raise an error if Consumers do not have message or batch delivery' do
+      expect {
+        described_class.configure do
+          consumer do
+            class_name 'ConsumerTest::MyBatchConsumer'
+            delivery :message
+          end
+        end
+      }.to raise_error('Non-batch Consumer ConsumerTest::MyBatchConsumer does not implement `consume`')
+    end
+
+    it 'should treat nil as `batch`' do
+      expect {
+        described_class.configure do
+          consumer do
+            class_name 'ConsumerTest::MyConsumer'
+          end
+        end
+      }.not_to raise_error
+    end
+  end
+end

data/spec/generators/active_record_generator_spec.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require 'generators/deimos/active_record_generator'
+
+RSpec.describe Deimos::Generators::ActiveRecordGenerator do
+
+  after(:each) do
+    FileUtils.rm_rf('db') if File.exist?('db')
+    FileUtils.rm_rf('app') if File.exist?('app')
+  end
+
+  it 'should generate a migration' do
+    expect(Dir['db/migrate/*.rb']).to be_empty
+    expect(Dir['app/models/*.rb']).to be_empty
+    described_class.start(['generated_table', 'com.my-namespace.Generated'])
+    files = Dir['db/migrate/*.rb']
+    expect(files.length).to eq(1)
+    results = <<~MIGRATION
+      class CreateGeneratedTable < ActiveRecord::Migration[6.1]
+        def up
+          if table_exists?(:generated_table)
+            warn "generated_table already exists, exiting"
+            return
+          end
+          create_table :generated_table do |t|
+            t.string :a_string
+            t.integer :a_int
+            t.bigint :a_long
+            t.float :a_float
+            t.float :a_double
+            t.string :an_enum
+            t.json :an_array
+            t.json :a_map
+            t.json :a_record
+          end
+      
+          # TODO add indexes as necessary
+        end
+      
+        def down
+          return unless table_exists?(:generated_table)
+          drop_table :generated_table
+        end
+      
+      end
+    MIGRATION
+    expect(File.read(files[0])).to eq(results)
+    model = <<~MODEL
+      class GeneratedTable < ApplicationRecord
+        enum an_enum: {sym1: 'sym1', sym2: 'sym2'}
+      end
+    MODEL
+    expect(File.read('app/models/generated_table.rb')).to eq(model)
+  end
+
+end

data/spec/handlers/my_batch_consumer.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module ConsumerTest
+  # Mock consumer
+  class MyBatchConsumer < Deimos::Consumer
+    # :no-doc:
+    def consume_batch
+    end
+  end
+end

data/spec/handlers/my_consumer.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module ConsumerTest
+  # Mock consumer
+  class MyConsumer < Deimos::Consumer
+    # :no-doc:
+    def consume
+    end
+  end
+end