deimos-temp-fork 0.0.1

data/spec/active_record_consume/batch_slicer_spec.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+RSpec.describe Deimos::ActiveRecordConsume::BatchSlicer do
+  describe '#slice' do
+    let(:batch) do
+      [
+        Deimos::Message.new({ v: 1 }, nil, key: 'C'),
+        Deimos::Message.new({ v: 123 }, nil, key: 'A'),
+        Deimos::Message.new({ v: 999 }, nil, key: 'B'),
+        Deimos::Message.new({ v: 456 }, nil, key: 'A'),
+        Deimos::Message.new({ v: 2 }, nil, key: 'C'),
+        Deimos::Message.new({ v: 3 }, nil, key: 'C')
+      ]
+    end
+
+    it 'should slice a batch by key' do
+      slices = described_class.slice(batch)
+
+      expect(slices).
+        to match([
+                   match_array([
+                                 Deimos::Message.new({ v: 1 }, nil, key: 'C'),
+                                 Deimos::Message.new({ v: 123 }, nil, key: 'A'),
+                                 Deimos::Message.new({ v: 999 }, nil, key: 'B')
+                               ]),
+                   match_array([
+                                 Deimos::Message.new({ v: 456 }, nil, key: 'A'),
+                                 Deimos::Message.new({ v: 2 }, nil, key: 'C')
+                               ]),
+                   match_array([
+                                 Deimos::Message.new({ v: 3 }, nil, key: 'C')
+                               ])
+                 ])
+    end
+
+    it 'should handle empty batches' do
+      slices = described_class.slice([])
+
+      expect(slices).to be_empty
+    end
+  end
+end

data/spec/active_record_consume/schema_model_converter_spec.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require 'deimos/active_record_consume/schema_model_converter'
+require 'deimos/schema_backends/avro_local'
+
+# Wrapped in a module to prevent class leakage
+module SchemaModelConverterTest
+  describe Deimos::ActiveRecordConsume::SchemaModelConverter do
+    # Create ActiveRecord table and model
+    before(:all) do
+      ActiveRecord::Base.connection.create_table(:wibbles, force: true) do |t|
+        t.integer(:wibble_id)
+        t.string(:name)
+        t.integer(:bar)
+        t.datetime(:birthday_int)
+        t.datetime(:birthday_long)
+        t.datetime(:birthday_optional)
+        t.timestamps
+      end
+
+      # :nodoc:
+      class Wibble < ActiveRecord::Base
+      end
+      Wibble.reset_column_information
+    end
+
+    after(:all) do
+      ActiveRecord::Base.connection.drop_table(:wibbles)
+    end
+
+    let(:schema) { Deimos::SchemaBackends::AvroLocal.new(schema: 'Wibble', namespace: 'com.my-namespace') }
+    let(:inst) { described_class.new(schema, Wibble) }
+
+    describe '#convert' do
+      it 'should extract attributes from the payload' do
+        payload = { 'id' => 123, 'wibble_id' => 456, 'name' => 'wibble' }
+
+        expect(inst.convert(payload)).to include('id' => 123, 'wibble_id' => 456, 'name' => 'wibble')
+      end
+
+      it 'should ignore payload fields that don\'t exist' do
+        payload = { 'foo' => 'abc' }
+
+        expect(inst.convert(payload)).not_to include('foo')
+      end
+
+      it 'should ignore fields in the schema but not in the model' do
+        payload = { 'floop' => 'def' }
+
+        expect(inst.convert(payload)).not_to include('floop')
+      end
+
+      it 'should ignore model fields not in the schema' do
+        payload = { 'bar' => 'xyz' }
+
+        expect(inst.convert(payload)).not_to include('bar')
+      end
+
+      it 'should handle nils' do
+        payload = { 'name' => nil }
+
+        expect(inst.convert(payload)['name']).to be_nil
+      end
+
+      describe 'timestamps' do
+        it 'should ignore AR timestamp fields' do
+          payload = { 'updated_at' => '1234567890', 'created_at' => '2345678901' }
+
+          expect(inst.convert(payload)).not_to include('updated_at', 'created_at')
+        end
+
+        it 'should parse timestamps' do
+          first = Time.zone.local(2019, 1, 1, 11, 12, 13)
+          second = Time.zone.local(2019, 2, 2, 12, 13, 14)
+
+          payload = { 'birthday_int' => first.to_i, 'birthday_long' => second.to_i }
+
+          expect(inst.convert(payload)).to include('birthday_int' => first, 'birthday_long' => second)
+        end
+
+        it 'should parse integer-string timestamps' do
+          date = Time.zone.local(2019, 1, 1, 11, 12, 13)
+
+          payload = { 'birthday_int' => '1546359133' }
+
+          expect(inst.convert(payload)).to include('birthday_int' => date)
+        end
+
+        it 'should ignore other strings for timestamps' do
+          payload = { 'birthday_int' => 'some-other-val' }
+
+          expect(inst.convert(payload)).to include('birthday_int' => 'some-other-val')
+        end
+      end
+
+      it 'should coerce nullable unions' do
+        date = Time.zone.local(2019, 1, 1, 11, 12, 13)
+
+        payload = { 'birthday_optional' => date.to_i }
+
+        expect(inst.convert(payload)).to include('birthday_optional' => date)
+      end
+    end
+  end
+end

data/spec/active_record_consumer_spec.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+require 'date'
+
+# Wrapped in a module to prevent class leakage
+module ActiveRecordConsumerTest
+  describe Deimos::ActiveRecordConsumer, 'Message Consumer' do
+
+    before(:all) do
+      ActiveRecord::Base.connection.create_table(:widgets, force: true) do |t|
+        t.string(:test_id)
+        t.integer(:some_int)
+        t.boolean(:some_bool)
+        t.datetime(:some_datetime_int)
+        t.timestamps
+      end
+
+      # :nodoc:
+      class Widget < ActiveRecord::Base
+        default_scope -> { where(some_bool: false) }
+      end
+      Widget.reset_column_information
+    end
+
+    before(:each) do
+      Widget.delete_all
+    end
+
+    after(:all) do
+      ActiveRecord::Base.connection.drop_table(:widgets)
+    end
+
+    prepend_before(:each) do
+
+      consumer_class = Class.new(Deimos::ActiveRecordConsumer) do
+        schema 'MySchemaWithDateTimes'
+        namespace 'com.my-namespace'
+        key_config plain: true
+        record_class Widget
+      end
+      stub_const('MyConsumer', consumer_class)
+
+      consumer_class = Class.new(Deimos::ActiveRecordConsumer) do
+        schema 'MySchema'
+        namespace 'com.my-namespace'
+        key_config none: true
+        record_class Widget
+
+        # :nodoc:
+        def assign_key(_record, _payload, _key)
+          # do nothing since we're not using primary keys
+        end
+
+        # :nodoc:
+        def fetch_record(klass, payload, _key)
+          klass.unscoped.where('test_id' => payload[:test_id]).first
+        end
+      end
+      stub_const('MyCustomFetchConsumer', consumer_class)
+
+      Time.zone = 'Eastern Time (US & Canada)'
+    end
+
+    it 'should receive events correctly' do
+      travel 1.day do
+        expect(Widget.count).to eq(0)
+        test_consume_message(MyConsumer, {
+                               test_id: 'abc',
+                               some_int: 3,
+                               updated_at: 1.day.ago.to_i,
+                               some_datetime_int: Time.zone.now.to_i,
+                               timestamp: 2.minutes.ago.to_s
+                             }, { call_original: true, key: 5 })
+
+        expect(Widget.count).to eq(1)
+        widget = Widget.last
+        expect(widget.id).to eq(5)
+        expect(widget.test_id).to eq('abc')
+        expect(widget.some_int).to eq(3)
+        expect(widget.some_datetime_int).to eq(Time.zone.now)
+        expect(widget.some_bool).to eq(false)
+        expect(widget.updated_at).to eq(Time.zone.now)
+
+        # test unscoped
+        widget.update_attribute(:some_bool, true)
+
+        # test update
+        test_consume_message(MyConsumer, {
+                               test_id: 'abcd',
+                               some_int: 3,
+                               some_datetime_int: Time.zone.now.to_i,
+                               timestamp: 2.minutes.ago.to_s
+                             }, { call_original: true, key: 5 })
+        expect(Widget.unscoped.count).to eq(1)
+        widget = Widget.unscoped.last
+        expect(widget.id).to eq(5)
+        expect(widget.test_id).to eq('abcd')
+        expect(widget.some_int).to eq(3)
+
+        # test delete
+        test_consume_message(MyConsumer, nil, call_original: true, key: 5)
+        expect(Widget.count).to eq(0)
+
+      end
+
+    end
+
+    it 'should update only updated_at' do
+      travel_to Time.local(2020, 5, 5, 5, 5, 5)
+      widget1 = Widget.create!(test_id: 'id1', some_int: 3)
+      expect(widget1.updated_at.in_time_zone).to eq(Time.local(2020, 5, 5, 5, 5, 5))
+
+      travel 1.day
+      test_consume_message(MyCustomFetchConsumer, {
+                             test_id: 'id1',
+                             some_int: 3
+                           }, { call_original: true })
+      expect(widget1.reload.updated_at.in_time_zone).
+        to eq(Time.local(2020, 5, 6, 5, 5, 5))
+      travel_back
+    end
+
+    it 'should find widgets by custom logic' do
+      widget1 = Widget.create!(test_id: 'id1')
+      expect(widget1.some_int).to be_nil
+      test_consume_message(MyCustomFetchConsumer, {
+                             test_id: 'id1',
+                             some_int: 3
+                           }, { call_original: true })
+      expect(widget1.reload.some_int).to eq(3)
+      expect(Widget.count).to eq(1)
+      test_consume_message(MyCustomFetchConsumer, {
+                             test_id: 'id2',
+                             some_int: 4
+                           }, { call_original: true })
+      expect(Widget.count).to eq(2)
+      expect(Widget.find_by_test_id('id1').some_int).to eq(3)
+      expect(Widget.find_by_test_id('id2').some_int).to eq(4)
+    end
+
+    it 'should not create record of process_message returns false' do
+      MyConsumer.any_instance.stub(:process_message?).and_return(false)
+      expect(Widget.count).to eq(0)
+      test_consume_message(MyConsumer, {
+                             test_id: 'abc',
+                             some_int: 3,
+                             updated_at: 1.day.ago.to_i,
+                             some_datetime_int: Time.zone.now.to_i,
+                             timestamp: 2.minutes.ago.to_s
+                           }, { call_original: true, key: 5 })
+      expect(Widget.count).to eq(0)
+    end
+  end
+end

data/spec/active_record_producer_spec.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+describe Deimos::ActiveRecordProducer do
+
+  include_context 'with widgets'
+
+  prepend_before(:each) do
+
+    producer_class = Class.new(Deimos::ActiveRecordProducer) do
+      schema 'MySchema'
+      namespace 'com.my-namespace'
+      topic 'my-topic'
+      key_config none: true
+    end
+    stub_const('MyProducer', producer_class)
+
+    producer_class = Class.new(Deimos::ActiveRecordProducer) do
+      schema 'MySchemaWithBooleans'
+      namespace 'com.my-namespace'
+      topic 'my-topic-with-boolean'
+      key_config none: true
+    end
+    stub_const('MyBooleanProducer', producer_class)
+
+    producer_class = Class.new(Deimos::ActiveRecordProducer) do
+      schema 'MySchemaWithId'
+      namespace 'com.my-namespace'
+      topic 'my-topic-with-id'
+      key_config none: true
+      record_class Widget
+
+      # :nodoc:
+      def self.generate_payload(attrs, widget)
+        super.merge(message_id: widget.generated_id)
+      end
+
+    end
+    stub_const('MyProducerWithID', producer_class)
+
+    producer_class = Class.new(Deimos::ActiveRecordProducer) do
+      schema 'MySchemaWithUniqueId'
+      namespace 'com.my-namespace'
+      topic 'my-topic-with-unique-id'
+      key_config field: :id
+      record_class Widget
+    end
+    stub_const('MyProducerWithUniqueID', producer_class)
+  end
+
+  it 'should send events correctly' do
+    MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 3))
+    expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
+  end
+
+  it 'should coerce values' do
+    MyProducer.send_event(Widget.new(test_id: 'abc', some_int: '3'))
+    MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 4.5))
+    expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
+    expect('my-topic').to have_sent(test_id: 'abc', some_int: 4)
+    expect {
+      MyProducer.send_event(Widget.new(test_id: 'abc', some_int: nil))
+    }.to raise_error(Avro::SchemaValidator::ValidationError)
+
+    MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: nil))
+    MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: true))
+    expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: false)
+    expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: true)
+  end
+
+  it 'should be able to call the record' do
+    widget = Widget.create!(test_id: 'abc2', some_int: 3)
+    MyProducerWithID.send_event(id: widget.id, test_id: 'abc2', some_int: 3)
+    expect('my-topic-with-id').to have_sent(
+      test_id: 'abc2',
+      some_int: 3,
+      message_id: 'generated_id',
+      timestamp: anything
+    )
+  end
+
+  specify '#watched_attributes' do
+    expect(MyProducer.watched_attributes).to eq(%w(test_id some_int))
+  end
+
+end

data/spec/backends/base_spec.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+RSpec.describe Deimos::Backends::Base do
+  include_context 'with publish_backend'
+  it 'should call execute' do
+    expect(described_class).to receive(:execute).
+      with(messages: messages, producer_class: MyProducer)
+    described_class.publish(producer_class: MyProducer, messages: messages)
+  end
+end

data/spec/backends/db_spec.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+each_db_config(Deimos::Backends::Db) do
+  include_context 'with publish_backend'
+
+  it 'should save to the database' do
+    expect(Deimos.config.metrics).to receive(:increment).with(
+      'db_producer.insert',
+      tags: %w(topic:my-topic),
+      by: 3
+    )
+    described_class.publish(producer_class: MyProducer, messages: messages)
+    records = Deimos::KafkaMessage.all
+    expect(records.size).to eq(3)
+    expect(records[0].attributes.to_h).to include(
+      'message' => '{"foo"=>1}',
+      'topic' => 'my-topic',
+      'key' => 'foo1'
+    )
+    expect(records[1].attributes.to_h).to include(
+      'message' => '{"foo"=>2}',
+      'topic' => 'my-topic',
+      'key' => 'foo2'
+    )
+    expect(records[2].attributes.to_h).to include(
+      'message' => '{"foo"=>3}',
+      'topic' => 'my-topic',
+      'key' => 'foo3'
+    )
+  end
+
+  it 'should add nil messages' do
+    described_class.publish(producer_class: MyProducer,
+                            messages: [build_message(nil, 'my-topic', 'foo1')])
+    expect(Deimos::KafkaMessage.count).to eq(1)
+    expect(Deimos::KafkaMessage.last.message).to eq(nil)
+  end
+
+  it 'should add to non-keyed messages' do
+    described_class.publish(producer_class: MyNoKeyProducer,
+                            messages: messages)
+    expect(Deimos::KafkaMessage.count).to eq(3)
+    described_class.publish(producer_class: MyNoKeyProducer,
+                            messages: [messages.first])
+    expect(Deimos::KafkaMessage.count).to eq(4)
+  end
+
+  it 'should add messages with Hash keys with JSON encoding' do
+    described_class.publish(producer_class: MyProducer,
+                            messages: [build_message({ foo: 0 }, 'my-topic', { 'test_id' => 0 })])
+    expect(Deimos::KafkaMessage.count).to eq(1)
+    expect(Deimos::KafkaMessage.last.partition_key).to eq(%(---\ntest_id: 0\n))
+  end
+end