deimos-ruby 1.24.2 → 2.0.0.pre.alpha1

data/spec/deimos_spec.rb

@@ -2,72 +2,12 @@
 
 describe Deimos do
 
-  let(:phobos_configuration) do
-    { 'logger' =>
-        { 'file' => 'log/phobos.log',
-          'stdout_json' => false,
-          'level' => 'debug',
-          'ruby_kafka' =>
-              { 'level' => 'debug' } },
-      'kafka' =>
-        { 'client_id' => 'phobos',
-          'connect_timeout' => 15,
-          'socket_timeout' => 15,
-          'seed_brokers' => 'my_seed_broker.com',
-          'ssl_ca_cert' => 'my_ssl_ca_cert',
-          'ssl_client_cert' => 'my_ssl_client_cert',
-          'ssl_client_cert_key' => 'my_ssl_client_cert_key' },
-      'producer' =>
-        { 'ack_timeout' => 5,
-          'required_acks' => :all,
-          'max_retries' => 2,
-          'retry_backoff' => 1,
-          'max_buffer_size' => 10_000,
-          'max_buffer_bytesize' => 10_000_000,
-          'compression_codec' => nil,
-          'compression_threshold' => 1,
-          'max_queue_size' => 10_000,
-          'delivery_threshold' => 0,
-          'delivery_interval' => 0 },
-      'consumer' =>
-        { 'session_timeout' => 300,
-          'offset_commit_interval' => 10,
-          'offset_commit_threshold' => 0,
-          'heartbeat_interval' => 10 },
-      'backoff' =>
-        { 'min_ms' => 1000,
-          'max_ms' => 60_000 },
-      'listeners' => [
-        { 'handler' => 'ConsumerTest::MyConsumer',
-          'topic' => 'my_consume_topic',
-          'group_id' => 'my_group_id',
-          'max_bytes_per_partition' => 524_288 },
-        { 'handler' => 'ConsumerTest::MyBatchConsumer',
-          'topic' => 'my_batch_consume_topic',
-          'group_id' => 'my_batch_group_id',
-          'delivery' => 'inline_batch' }
-      ],
-      'custom_logger' => nil,
-      'custom_kafka_logger' => nil }
-  end
-
-  let(:config_path) { File.join(File.dirname(__FILE__), 'phobos.yml') }
-
   it 'should have a version number' do
     expect(Deimos::VERSION).not_to be_nil
   end
 
-  it 'should error if required_acks is not all' do
-    expect {
-      described_class.configure do |config|
-        config.producers.backend = :db
-        config.phobos_config_file = File.join(File.dirname(__FILE__), 'phobos.bad_db.yml')
-      end
-    }.to raise_error('Cannot set producers.backend to :db unless producers.required_acks is set to ":all"!')
-  end
-
-  describe '#start_db_backend!' do
-    it 'should start if backend is db and thread_count is > 0' do
+  describe '#start_outbox_backend!' do
+    it 'should start if backend is outbox and thread_count is > 0' do
       signal_handler = instance_double(Sigurd::SignalHandler)
       allow(signal_handler).to receive(:run!)
       expect(Sigurd::Executor).to receive(:new).
@@ -77,9 +17,9 @@ describe Deimos do
         signal_handler
       end
       described_class.configure do |config|
-        config.producers.backend = :db
+        config.producers.backend = :outbox
       end
-      described_class.start_db_backend!(thread_count: 2)
+      described_class.start_outbox_backend!(thread_count: 2)
     end
 
     it 'should not start if backend is not db' do
@@ -87,83 +27,27 @@ describe Deimos do
       described_class.configure do |config|
         config.producers.backend = :kafka
       end
-      expect { described_class.start_db_backend!(thread_count: 2) }.
-        to raise_error('Publish backend is not set to :db, exiting')
+      expect { described_class.start_outbox_backend!(thread_count: 2) }.
+        to raise_error('Publish backend is not set to :outbox, exiting')
     end
 
     it 'should not start if thread_count is nil' do
       expect(Sigurd::SignalHandler).not_to receive(:new)
       described_class.configure do |config|
-        config.producers.backend = :db
+        config.producers.backend = :outbox
       end
-      expect { described_class.start_db_backend!(thread_count: nil) }.
+      expect { described_class.start_outbox_backend!(thread_count: nil) }.
         to raise_error('Thread count is not given or set to zero, exiting')
     end
 
     it 'should not start if thread_count is 0' do
       expect(Sigurd::SignalHandler).not_to receive(:new)
       described_class.configure do |config|
-        config.producers.backend = :db
+        config.producers.backend = :outbox
       end
-      expect { described_class.start_db_backend!(thread_count: 0) }.
+      expect { described_class.start_outbox_backend!(thread_count: 0) }.
         to raise_error('Thread count is not given or set to zero, exiting')
     end
   end
 
-  describe 'delivery configuration' do
-    before(:each) do
-      allow(YAML).to receive(:load).and_return(phobos_configuration)
-    end
-
-    it 'should not raise an error with properly configured handlers' do
-      expect {
-        described_class.configure do
-          consumer do
-            class_name 'ConsumerTest::MyConsumer'
-            delivery :message
-          end
-          consumer do
-            class_name 'ConsumerTest::MyConsumer'
-            delivery :batch
-          end
-          consumer do
-            class_name 'ConsumerTest::MyBatchConsumer'
-            delivery :inline_batch
-          end
-        end
-      }.not_to raise_error
-    end
-
-    it 'should raise an error if inline_batch listeners do not implement consume_batch' do
-      expect {
-        described_class.configure do
-          consumer do
-            class_name 'ConsumerTest::MyConsumer'
-            delivery :inline_batch
-          end
-        end
-      }.to raise_error('BatchConsumer ConsumerTest::MyConsumer does not implement `consume_batch`')
-    end
-
-    it 'should raise an error if Consumers do not have message or batch delivery' do
-      expect {
-        described_class.configure do
-          consumer do
-            class_name 'ConsumerTest::MyBatchConsumer'
-            delivery :message
-          end
-        end
-      }.to raise_error('Non-batch Consumer ConsumerTest::MyBatchConsumer does not implement `consume`')
-    end
-
-    it 'should treat nil as `batch`' do
-      expect {
-        described_class.configure do
-          consumer do
-            class_name 'ConsumerTest::MyConsumer'
-          end
-        end
-      }.not_to raise_error
-    end
-  end
 end

data/spec/kafka_source_spec.rb

@@ -17,18 +17,10 @@ module KafkaSourceSpec
 
       # Dummy producer which mimicks the behavior of a real producer
       class WidgetProducer < Deimos::ActiveRecordProducer
-        topic 'my-topic'
-        namespace 'com.my-namespace'
-        schema 'Widget'
-        key_config field: :id
       end
 
       # Dummy producer which mimicks the behavior of a real producer
       class WidgetProducerTheSecond < Deimos::ActiveRecordProducer
-        topic 'my-topic-the-second'
-        namespace 'com.my-namespace'
-        schema 'WidgetTheSecond'
-        key_config field: :id
       end
 
       # Dummy class we can include the mixin in. Has a backing table created
@@ -51,6 +43,22 @@ module KafkaSourceSpec
 
     before(:each) do
       Widget.delete_all
+      Karafka::App.routes.redraw do
+        topic 'my-topic' do
+          namespace 'com.my-namespace'
+          schema 'Widget'
+          key_config field: :id
+          producer_class WidgetProducer
+        end
+
+        topic 'my-topic-the-second' do
+          namespace 'com.my-namespace'
+          schema 'WidgetTheSecond'
+          key_config field: :id
+          producer_class WidgetProducerTheSecond
+        end
+
+      end
     end
 
     it 'should send events on creation, update, and deletion' do
@@ -206,10 +214,9 @@ module KafkaSourceSpec
     context 'with DB backend' do
       before(:each) do
         Deimos.configure do |config|
-          config.producers.backend = :db
+          config.producers.backend = :outbox
         end
         setup_db(DB_OPTIONS.last) # sqlite
-        allow(Deimos::Producer).to receive(:produce_batch).and_call_original
       end
 
       it 'should save to the DB' do
@@ -309,46 +316,6 @@ module KafkaSourceSpec
       end
     end
 
-    context 'with AR models that implement the kafka_producer interface' do
-      before(:each) do
-        # Dummy class we can include the mixin in. Has a backing table created
-        # earlier and has the import hook disabled
-        deprecated_class = Class.new(ActiveRecord::Base) do
-          include Deimos::KafkaSource
-          self.table_name = 'widgets'
-
-          # :nodoc:
-          def self.kafka_config
-            {
-              update: true,
-              delete: true,
-              import: false,
-              create: true
-            }
-          end
-
-          # :nodoc:
-          def self.kafka_producer
-            WidgetProducer
-          end
-        end
-        stub_const('WidgetDeprecated', deprecated_class)
-        WidgetDeprecated.reset_column_information
-      end
-
-      it 'logs a warning and sends the message as usual' do
-        expect(Deimos.config.logger).to receive(:warn).with({ message: WidgetDeprecated::DEPRECATION_WARNING })
-        widget = WidgetDeprecated.create(widget_id: 1, name: 'Widget 1')
-        expect('my-topic').to have_sent({
-                                          widget_id: 1,
-                                          name: 'Widget 1',
-                                          id: widget.id,
-                                          created_at: anything,
-                                          updated_at: anything
-                                        }, widget.id)
-      end
-    end
-
     context 'with AR models that do not implement any producer interface' do
       before(:each) do
         # Dummy class we can include the mixin in. Has a backing table created
@@ -371,10 +338,10 @@ module KafkaSourceSpec
         WidgetBuggy.reset_column_information
       end
 
-      it 'raises a NotImplementedError exception' do
+      it 'raises a MissingImplementationError exception' do
         expect {
           WidgetBuggy.create(widget_id: 1, name: 'Widget 1')
-        }.to raise_error(NotImplementedError)
+        }.to raise_error(Deimos::MissingImplementationError)
       end
     end
   end

data/spec/karafka/karafka.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+class KarafkaApp < Karafka::App
+  setup do |config|
+    config.kafka = { 'bootstrap.servers': '127.0.0.1:9092' }
+    config.client_id = 'example_app'
+    # Recreate consumers with each batch. This will allow Rails code reload to work in the
+    # development mode. Otherwise Karafka process would not be aware of code changes
+    config.consumer_persistence = !Rails.env.development?
+  end
+
+  # Comment out this part if you are not using instrumentation and/or you are not
+  # interested in logging events for certain environments. Since instrumentation
+  # notifications add extra boilerplate, if you want to achieve max performance,
+  # listen to only what you really need for given environment.
+  Karafka.monitor.subscribe(Karafka::Instrumentation::LoggerListener.new)
+  # Karafka.monitor.subscribe(Karafka::Instrumentation::ProctitleListener.new)
+
+  # This logger prints the producer development info using the Karafka logger.
+  # It is similar to the consumer logger listener but producer oriented.
+  Karafka.producer.monitor.subscribe(
+    WaterDrop::Instrumentation::LoggerListener.new(
+      # Log producer operations using the Karafka logger
+      Karafka.logger,
+      # If you set this to true, logs will contain each message details
+      # Please note, that this can be extensive
+      log_messages: false
+    )
+  )
+
+  # You can subscribe to all consumer related errors and record/track then that way
+  #
+  # Karafka.monitor.subscribe 'error.occurred' do |event|
+  #   type = event[:type]
+  #   error = event[:error]
+  #   details = (error.backtrace || []).join("\n")
+  #   ErrorTracker.send_error(error, type, details)
+  # end
+
+  # You can subscribe to all producer related errors and record/track then that way
+  # Please note, that producer and consumer have their own notifications pipeline so you need to
+  # setup error tracking independently for each of them
+  #
+  # Karafka.producer.monitor.subscribe('error.occurred') do |event|
+  #   type = event[:type]
+  #   error = event[:error]
+  #   details = (error.backtrace || []).join("\n")
+  #   ErrorTracker.send_error(error, type, details)
+  # end
+
+  routes.draw do
+    # Uncomment this if you use Karafka with ActiveJob
+    # You need to define the topic per each queue name you use
+    # active_job_topic :default
+    #    topic :example do
+      # Uncomment this if you want Karafka to manage your topics configuration
+      # Managing topics configuration via routing will allow you to ensure config consistency
+      # across multiple environments
+      #
+      # config(partitions: 2, 'cleanup.policy': 'compact')
+    # consumer ExampleConsumer
+    # end
+  end
+end
+
+# Karafka now features a Web UI!
+# Visit the setup documentation to get started and enhance your experience.
+#
+# https://karafka.io/docs/Web-UI-Getting-Started
+Deimos.setup_karafka

data/spec/karafka_config/karafka_spec.rb

@@ -0,0 +1,97 @@
+RSpec.describe 'Karafka configs' do
+  before(:each) do
+    KarafkaApp.routes.clear
+    $found_stuff = nil
+  end
+
+  let(:consumer_class) do
+    Class.new(Deimos::Consumer) do
+      def consume_message(message)
+        $found_stuff = message.payload
+      end
+    end
+  end
+
+  let(:producer_class) do
+    Class.new(Deimos::Producer) do
+    end
+  end
+
+  describe 'producers' do
+    before(:each) do
+      stub_const('MyProducer', producer_class)
+    end
+
+    it 'should work with key none' do
+      KarafkaApp.routes.draw do
+        topic 'MyTopic' do
+          producer_class MyProducer
+          schema 'MySchema'
+          namespace 'com.my-namespace'
+          key_config(none: true)
+        end
+      end
+      producer_class.publish({test_id: "id1", some_int: 5})
+      expect('MyTopic').to have_sent({test_id: "id1", some_int: 5})
+    end
+
+    it 'should work with key plain' do
+      KarafkaApp.routes.draw do
+        topic 'MyTopic' do
+          producer_class MyProducer
+          schema 'MySchema'
+          namespace 'com.my-namespace'
+          key_config({plain: true})
+        end
+      end
+      producer_class.publish({test_id: "id1", some_int: 5, payload_key: 'key'})
+      expect('MyTopic').to have_sent({test_id: "id1", some_int: 5}, 'key')
+    end
+
+    it 'should work with key field' do
+      KarafkaApp.routes.draw do
+        topic 'MyTopic' do
+          producer_class MyProducer
+          schema 'MySchema'
+          namespace 'com.my-namespace'
+          key_config({field: :test_id})
+        end
+      end
+      producer_class.publish({test_id: "id1", some_int: 5})
+      expect('MyTopic').to have_sent({test_id: "id1", some_int: 5}, 'id1')
+    end
+
+    it 'should work with key schema' do
+      KarafkaApp.routes.draw do
+        topic 'MyTopic' do
+          producer_class MyProducer
+          schema 'MySchema'
+          namespace 'com.my-namespace'
+          key_config({schema: 'MySchema_key'})
+        end
+      end
+      producer_class.publish({test_id: "id1", some_int: 5, payload_key: {test_id: 'id3'}})
+      expect('MyTopic').to have_sent({test_id: "id1", some_int: 5}, { test_id: 'id3'})
+    end
+
+  end
+
+  it 'should be able to pick up a consumer' do
+    stub_const('MyConsumer', consumer_class)
+    KarafkaApp.routes.draw do
+      topic 'MyTopic' do
+        consumer MyConsumer
+        schema 'MySchema'
+        namespace 'com.my-namespace'
+        key_config({field: :test_id})
+      end
+    end
+
+    test_consume_message('MyTopic', {test_id: "id1", some_int: 5}, key: "id1")
+    expect($found_stuff).to eq({'test_id' => "id1", 'some_int' => 5})
+    $found_stuff = nil
+    test_consume_message(MyConsumer, {test_id: "id1", some_int: 5}, key: "id1")
+    expect($found_stuff).to eq({'test_id' => "id1", 'some_int' => 5})
+  end
+
+end

data/spec/logging_spec.rb

@@ -0,0 +1,25 @@
+RSpec.describe Deimos::Logging do
+  include_context 'with publish_backend'
+  describe '#messages_log_text' do
+    it 'should return whole payload (default behavior)' do
+      log_message = described_class.messages_log_text(:payloads, messages)
+      expect(log_message[:payloads].count).to eq(3)
+      expect(log_message[:payloads].first[:payload]).to eq({ some_int: 1, test_id: 'foo1' })
+      expect(log_message[:payloads].first[:key]).to eq('foo1')
+    end
+
+    it 'should return only keys of messages' do
+      log_message = described_class.messages_log_text(:keys, messages)
+      expect(log_message[:payload_keys].count).to eq(3)
+      expect(log_message[:payload_keys]).to be_a(Array)
+      expect(log_message[:payload_keys].first).to eq('foo1')
+    end
+
+    it 'should return only messages count' do
+      log_message = described_class.messages_log_text(:count, messages)
+      expect(log_message[:payloads_count]).to be_a(Integer)
+      expect(log_message[:payloads_count]).to eq(3)
+    end
+  end
+
+end

data/spec/message_spec.rb

@@ -2,37 +2,37 @@
 
 RSpec.describe(Deimos::Message) do
   it 'should detect tombstones' do
-    expect(described_class.new(nil, nil, key: 'key1')).
+    expect(described_class.new(nil, key: 'key1')).
       to be_tombstone
-    expect(described_class.new({ v: 'val1' }, nil, key: 'key1')).
+    expect(described_class.new({ v: 'val1' }, key: 'key1')).
       not_to be_tombstone
-    expect(described_class.new({ v: '' }, nil, key: 'key1')).
+    expect(described_class.new({ v: '' }, key: 'key1')).
       not_to be_tombstone
-    expect(described_class.new({ v: 'val1' }, nil, key: nil)).
+    expect(described_class.new({ v: 'val1' }, key: nil)).
       not_to be_tombstone
   end
 
   it 'can support complex keys/values' do
-    expect { described_class.new({ a: 1, b: 2 }, nil, key: { c: 3, d: 4 }) }.
+    expect { described_class.new({ a: 1, b: 2 }, key: { c: 3, d: 4 }) }.
       not_to raise_exception
   end
 
   describe 'headers' do
     it 'returns nil when not set' do
-      expect(described_class.new({ v: 'val1' }, nil, key: 'key1')).
+      expect(described_class.new({ v: 'val1' }, key: 'key1')).
         to have_attributes(headers: nil)
     end
 
     it 'can set and get headers' do
-      expect(described_class.new({ v: 'val1' }, nil, key: 'key1', headers: { a: 1 })).
+      expect(described_class.new({ v: 'val1' }, key: 'key1', headers: { a: 1 })).
         to have_attributes(headers: { a: 1 })
     end
 
     it 'includes headers when converting to Hash' do
-      expect(described_class.new({ v: 'val1' }, nil, key: 'key1', headers: { a: 1 }).to_h).
+      expect(described_class.new({ v: 'val1' }, key: 'key1', headers: { a: 1 }).to_h).
         to include(headers: { a: 1 })
 
-      expect(described_class.new({ v: 'val1' }, nil, key: 'key1', headers: { a: 1 }).encoded_hash).
+      expect(described_class.new({ v: 'val1' }, key: 'key1', headers: { a: 1 }).encoded_hash).
         to include(headers: { a: 1 })
     end
   end