deimos-ruby 1.24.2 → 2.0.0.pre.alpha1

data/spec/active_record_consume/batch_slicer_spec.rb

@@ -4,12 +4,12 @@ RSpec.describe Deimos::ActiveRecordConsume::BatchSlicer do
   describe '#slice' do
     let(:batch) do
       [
-        Deimos::Message.new({ v: 1 }, nil, key: 'C'),
-        Deimos::Message.new({ v: 123 }, nil, key: 'A'),
-        Deimos::Message.new({ v: 999 }, nil, key: 'B'),
-        Deimos::Message.new({ v: 456 }, nil, key: 'A'),
-        Deimos::Message.new({ v: 2 }, nil, key: 'C'),
-        Deimos::Message.new({ v: 3 }, nil, key: 'C')
+        Deimos::Message.new({ v: 1 }, key: 'C'),
+        Deimos::Message.new({ v: 123 }, key: 'A'),
+        Deimos::Message.new({ v: 999 }, key: 'B'),
+        Deimos::Message.new({ v: 456 }, key: 'A'),
+        Deimos::Message.new({ v: 2 }, key: 'C'),
+        Deimos::Message.new({ v: 3 }, key: 'C')
       ]
     end
 
@@ -19,16 +19,16 @@ RSpec.describe Deimos::ActiveRecordConsume::BatchSlicer do
       expect(slices).
         to match([
                    match_array([
-                                 Deimos::Message.new({ v: 1 }, nil, key: 'C'),
-                                 Deimos::Message.new({ v: 123 }, nil, key: 'A'),
-                                 Deimos::Message.new({ v: 999 }, nil, key: 'B')
+                                 Deimos::Message.new({ v: 1 }, key: 'C'),
+                                 Deimos::Message.new({ v: 123 }, key: 'A'),
+                                 Deimos::Message.new({ v: 999 }, key: 'B')
                                ]),
                    match_array([
-                                 Deimos::Message.new({ v: 456 }, nil, key: 'A'),
-                                 Deimos::Message.new({ v: 2 }, nil, key: 'C')
+                                 Deimos::Message.new({ v: 456 }, key: 'A'),
+                                 Deimos::Message.new({ v: 2 }, key: 'C')
                                ]),
                    match_array([
-                                 Deimos::Message.new({ v: 3 }, nil, key: 'C')
+                                 Deimos::Message.new({ v: 3 }, key: 'C')
                                ])
                  ])
     end

data/spec/active_record_consume/mass_updater_spec.rb

@@ -114,5 +114,142 @@ RSpec.describe Deimos::ActiveRecordConsume::MassUpdater do
 
       end
 
+      context 'with save_associations_first' do
+        before(:all) do
+          ActiveRecord::Base.connection.create_table(:fidgets, force: true) do |t|
+            t.string(:test_id)
+            t.integer(:some_int)
+            t.string(:bulk_import_id)
+            t.timestamps
+          end
+
+          ActiveRecord::Base.connection.create_table(:fidget_details, force: true) do |t|
+            t.string(:title)
+            t.string(:bulk_import_id)
+            t.belongs_to(:fidget)
+
+            t.index(%i(title), unique: true)
+          end
+
+          ActiveRecord::Base.connection.create_table(:widget_fidgets, force: true, id: false) do |t|
+            t.belongs_to(:fidget)
+            t.belongs_to(:widget)
+            t.string(:bulk_import_id)
+            t.string(:note)
+            t.index(%i(widget_id fidget_id), unique: true)
+          end
+        end
+
+        after(:all) do
+          ActiveRecord::Base.connection.drop_table(:fidgets)
+          ActiveRecord::Base.connection.drop_table(:fidget_details)
+          ActiveRecord::Base.connection.drop_table(:widget_fidgets)
+        end
+
+        let(:fidget_detail_class) do
+          Class.new(ActiveRecord::Base) do
+            self.table_name = 'fidget_details'
+            belongs_to :fidget
+          end
+        end
+
+        let(:fidget_class) do
+          Class.new(ActiveRecord::Base) do
+            self.table_name = 'fidgets'
+            has_one :fidget_detail
+          end
+        end
+
+        let(:widget_fidget_class) do
+          Class.new(ActiveRecord::Base) do
+            self.table_name = 'widget_fidgets'
+            belongs_to :fidget
+            belongs_to :widget
+          end
+        end
+
+        let(:bulk_id_generator) { proc { SecureRandom.uuid } }
+
+        let(:key_proc) do
+          lambda do |klass|
+            case klass.to_s
+            when 'Widget', 'Fidget'
+              %w(id)
+            when 'WidgetFidget'
+              %w(widget_id fidget_id)
+            when 'FidgetDetail', 'Detail'
+              %w(title)
+            else
+              raise "Key Columns for #{klass} not defined"
+            end
+
+          end
+        end
+
+        before(:each) do
+          stub_const('Fidget', fidget_class)
+          stub_const('FidgetDetail', fidget_detail_class)
+          stub_const('WidgetFidget', widget_fidget_class)
+          Widget.reset_column_information
+          Fidget.reset_column_information
+          WidgetFidget.reset_column_information
+        end
+
+        # rubocop:disable RSpec/MultipleExpectations, RSpec/ExampleLength
+        it 'should backfill the associations when upserting primary records' do
+          batch = Deimos::ActiveRecordConsume::BatchRecordList.new(
+            [
+              Deimos::ActiveRecordConsume::BatchRecord.new(
+                klass: WidgetFidget,
+                attributes: {
+                  widget: { test_id: 'id1', some_int: 10, detail: { title: 'Widget Title 1' } },
+                  fidget: { test_id: 'id1', some_int: 10, fidget_detail: { title: 'Fidget Title 1' } },
+                  note: 'Stuff 1'
+                },
+                bulk_import_column: 'bulk_import_id',
+                bulk_import_id_generator: bulk_id_generator
+              ),
+              Deimos::ActiveRecordConsume::BatchRecord.new(
+                klass: WidgetFidget,
+                attributes: {
+                  widget: { test_id: 'id2', some_int: 20, detail: { title: 'Widget Title 2' } },
+                  fidget: { test_id: 'id2', some_int: 20, fidget_detail: { title: 'Fidget Title 2' } },
+                  note: 'Stuff 2'
+                },
+                bulk_import_column: 'bulk_import_id',
+                bulk_import_id_generator: bulk_id_generator
+              )
+            ]
+          )
+
+          results = described_class.new(WidgetFidget,
+                                        bulk_import_id_generator: bulk_id_generator,
+                                        bulk_import_id_column: 'bulk_import_id',
+                                        key_col_proc: key_proc,
+                                        save_associations_first: true).mass_update(batch)
+          expect(results.count).to eq(2)
+          expect(Widget.count).to eq(2)
+          expect(Detail.count).to eq(2)
+          expect(Fidget.count).to eq(2)
+          expect(FidgetDetail.count).to eq(2)
+
+          WidgetFidget.all.each_with_index do |widget_fidget, ind|
+            widget = Widget.find_by(id: widget_fidget.widget_id)
+            expect(widget.test_id).to eq("id#{ind + 1}")
+            expect(widget.some_int).to eq((ind + 1) * 10)
+            detail = Detail.find_by(widget_id: widget_fidget.widget_id)
+            expect(detail.title).to eq("Widget Title #{ind + 1}")
+            fidget = Fidget.find_by(id: widget_fidget.fidget_id)
+            expect(fidget.test_id).to eq("id#{ind + 1}")
+            expect(fidget.some_int).to eq((ind + 1) * 10)
+            fidget_detail = FidgetDetail.find_by(fidget_id: widget_fidget.fidget_id)
+            expect(fidget_detail.title).to eq("Fidget Title #{ind + 1}")
+            expect(widget_fidget.note).to eq("Stuff #{ind + 1}")
+          end
+        end
+        # rubocop:enable RSpec/MultipleExpectations, RSpec/ExampleLength
+
+      end
+
     end
 end

data/spec/active_record_consumer_spec.rb

@@ -33,25 +33,16 @@ module ActiveRecordConsumerTest
     prepend_before(:each) do
 
       consumer_class = Class.new(Deimos::ActiveRecordConsumer) do
-        schema 'MySchemaWithDateTimes'
-        namespace 'com.my-namespace'
-        key_config plain: true
         record_class Widget
       end
       stub_const('MyConsumer', consumer_class)
 
       consumer_class = Class.new(Deimos::ActiveRecordConsumer) do
-        schema 'MySchemaWithDateTimes'
-        namespace 'com.my-namespace'
-        key_config schema: 'MySchemaId_key'
         record_class Widget
       end
       stub_const('MyConsumerWithKey', consumer_class)
 
       consumer_class = Class.new(Deimos::ActiveRecordConsumer) do
-        schema 'MySchema'
-        namespace 'com.my-namespace'
-        key_config none: true
         record_class Widget
 
         # :nodoc:
@@ -135,14 +126,39 @@ module ActiveRecordConsumerTest
         end
       end
       stub_const('Schemas::MySchemaWithDateTimes', schema_datetime_class)
+
+      Karafka::App.routes.redraw do
+        topic "my-topic" do
+          consumer MyConsumer
+          schema 'MySchemaWithDateTimes'
+          namespace 'com.my-namespace'
+          key_config plain: true
+        end
+        topic "my-topic2" do
+          consumer MyConsumerWithKey
+          schema 'MySchemaWithDateTimes'
+          namespace 'com.my-namespace'
+          key_config schema: 'MySchemaId_key'
+        end
+        topic "my-topic3" do
+          consumer MyCustomFetchConsumer
+          schema 'MySchema'
+          namespace 'com.my-namespace'
+          key_config none: true
+        end
+      end
     end
 
     describe 'consume' do
       SCHEMA_CLASS_SETTINGS.each do |setting, use_schema_classes|
         context "with Schema Class consumption #{setting}" do
           before(:each) do
+            Karafka::App.routes.draw do
+              defaults do
+                use_schema_classes use_schema_classes
+              end
+            end
             Deimos.configure do |config|
-              config.schema.use_schema_classes = use_schema_classes
               config.schema.use_full_namespace = true
             end
           end
@@ -200,7 +216,7 @@ module ActiveRecordConsumerTest
             test_consume_message(MyCustomFetchConsumer, {
                                    test_id: 'id1',
                                    some_int: 3
-                                 }, call_original: true)
+                                 })
             expect(widget1.reload.updated_at.in_time_zone).
               to eq(Time.local(2020, 5, 6, 5, 5, 5))
             travel_back
@@ -225,13 +241,13 @@ module ActiveRecordConsumerTest
             test_consume_message(MyCustomFetchConsumer, {
                                    test_id: 'id1',
                                    some_int: 3
-                                 }, call_original: true)
+                                 })
             expect(widget1.reload.some_int).to eq(3)
             expect(Widget.count).to eq(1)
             test_consume_message(MyCustomFetchConsumer, {
                                    test_id: 'id2',
                                    some_int: 4
-                                 }, call_original: true)
+                                 })
             expect(Widget.count).to eq(2)
             expect(Widget.find_by_test_id('id1').some_int).to eq(3)
             expect(Widget.find_by_test_id('id2').some_int).to eq(4)

data/spec/active_record_producer_spec.rb

@@ -5,28 +5,13 @@ describe Deimos::ActiveRecordProducer do
   include_context 'with widgets'
 
   prepend_before(:each) do
-
-    producer_class = Class.new(Deimos::ActiveRecordProducer) do
-      schema 'MySchema'
-      namespace 'com.my-namespace'
-      topic 'my-topic'
-      key_config none: true
-    end
+    producer_class = Class.new(Deimos::ActiveRecordProducer)
     stub_const('MyProducer', producer_class)
 
-    producer_class = Class.new(Deimos::ActiveRecordProducer) do
-      schema 'MySchemaWithBooleans'
-      namespace 'com.my-namespace'
-      topic 'my-topic-with-boolean'
-      key_config none: true
-    end
+    producer_class = Class.new(Deimos::ActiveRecordProducer)
     stub_const('MyBooleanProducer', producer_class)
 
     producer_class = Class.new(Deimos::ActiveRecordProducer) do
-      schema 'MySchemaWithId'
-      namespace 'com.my-namespace'
-      topic 'my-topic-with-id'
-      key_config none: true
       record_class Widget
 
       # :nodoc:
@@ -38,19 +23,11 @@ describe Deimos::ActiveRecordProducer do
     stub_const('MyProducerWithID', producer_class)
 
     producer_class = Class.new(Deimos::ActiveRecordProducer) do
-      schema 'MySchemaWithUniqueId'
-      namespace 'com.my-namespace'
-      topic 'my-topic-with-unique-id'
-      key_config field: :id
       record_class Widget
     end
     stub_const('MyProducerWithUniqueID', producer_class)
 
     producer_class = Class.new(Deimos::ActiveRecordProducer) do
-      schema 'MySchemaWithUniqueId'
-      namespace 'com.my-namespace'
-      topic 'my-topic-with-unique-id'
-      key_config field: :id
       record_class Widget
 
       # :nodoc:
@@ -63,6 +40,39 @@ describe Deimos::ActiveRecordProducer do
     end
 
     stub_const('MyProducerWithPostProcess', producer_class)
+    Karafka::App.routes.redraw do
+      topic 'my-topic' do
+        schema 'MySchema'
+        namespace 'com.my-namespace'
+        key_config none: true
+        producer_class MyProducer
+      end
+      topic 'my-topic-with-boolean' do
+        producer_class MyBooleanProducer
+        schema 'MySchemaWithBooleans'
+        namespace 'com.my-namespace'
+        key_config none: true
+      end
+      topic 'my-topic-with-id' do
+        schema 'MySchemaWithId'
+        namespace 'com.my-namespace'
+        key_config none: true
+        producer_class MyProducerWithID
+      end
+      topic 'my-topic-with-unique-id' do
+        schema 'MySchemaWithUniqueId'
+        namespace 'com.my-namespace'
+        key_config field: :id
+        producer_class MyProducerWithUniqueID
+      end
+      topic 'my-topic-with-post-process' do
+        schema 'MySchemaWithUniqueId'
+        namespace 'com.my-namespace'
+        key_config none: true
+        producer_class MyProducerWithPostProcess
+      end
+    end
+
   end
 
   describe 'produce' do
@@ -117,7 +127,7 @@ describe Deimos::ActiveRecordProducer do
   end
 
   specify '#watched_attributes' do
-    expect(MyProducer.watched_attributes).to eq(%w(test_id some_int))
+    expect(MyProducer.watched_attributes(nil)).to eq(%w(test_id some_int))
   end
 
 end

data/spec/backends/base_spec.rb

@@ -8,27 +8,4 @@ RSpec.describe Deimos::Backends::Base do
     described_class.publish(producer_class: MyProducer, messages: messages)
   end
 
-  describe 'payload_log method' do
-    it 'should return whole payload (default behavior)' do
-      log_message = described_class.send(:log_message, messages)
-      expect(log_message[:payloads].count).to eq(3)
-      expect(log_message[:payloads].first[:payload]).to eq({ 'foo' => 1 })
-      expect(log_message[:payloads].first[:key]).to eq('foo1')
-    end
-
-    it 'should return only keys of messages' do
-      Deimos.config.payload_log = :keys
-      log_message = described_class.send(:log_message, messages)
-      expect(log_message[:payload_keys].count).to eq(3)
-      expect(log_message[:payload_keys]).to be_a(Array)
-      expect(log_message[:payload_keys].first).to eq('foo1')
-    end
-
-    it 'should return only messages count' do
-      Deimos.config.payload_log = :count
-      log_message = described_class.send(:log_message, messages)
-      expect(log_message[:payloads_count]).to be_a(Integer)
-      expect(log_message[:payloads_count]).to eq(3)
-    end
-  end
 end

data/spec/backends/kafka_async_spec.rb

@@ -3,9 +3,7 @@
 RSpec.describe Deimos::Backends::KafkaAsync do
   include_context 'with publish_backend'
   it 'should publish to Kafka asynchronously' do
-    producer = instance_double(Phobos::Producer::ClassMethods::PublicAPI)
-    expect(producer).to receive(:async_publish_list).with(messages.map(&:encoded_hash))
-    expect(described_class).to receive(:producer).and_return(producer)
+    expect(Karafka.producer).to receive(:produce_many_async).with(messages)
     described_class.publish(producer_class: MyProducer, messages: messages)
   end
 end

data/spec/backends/kafka_spec.rb

@@ -3,9 +3,7 @@
 RSpec.describe Deimos::Backends::Kafka do
   include_context 'with publish_backend'
   it 'should publish to Kafka synchronously' do
-    producer = instance_double(Phobos::Producer::ClassMethods::PublicAPI)
-    expect(producer).to receive(:publish_list).with(messages.map(&:encoded_hash))
-    expect(described_class).to receive(:producer).and_return(producer)
+    expect(Karafka.producer).to receive(:produce_many_sync).with(messages)
     described_class.publish(producer_class: MyProducer, messages: messages)
   end
 end

data/spec/backends/{db_spec.rb → outbox_spec.rb}

@@ -1,31 +1,31 @@
 # frozen_string_literal: true
 
-each_db_config(Deimos::Backends::Db) do
+each_db_config(Deimos::Backends::Outbox) do
   include_context 'with publish_backend'
 
   it 'should save to the database' do
-    expect(Deimos.config.metrics).to receive(:increment).with(
-      'db_producer.insert',
-      tags: %w(topic:my-topic),
-      by: 3
-    )
+    # expect(Deimos.config.metrics).to receive(:increment).with(
+    #   'outbox.insert',
+    #   tags: %w(topic:my-topic),
+    #   by: 3
+    # )
     described_class.publish(producer_class: MyProducer, messages: messages)
     records = Deimos::KafkaMessage.all
     expect(records.size).to eq(3)
     expect(records[0].attributes.to_h).to include(
-      'message' => '{"foo"=>1}',
+      'message' => '{"test_id":"foo1","some_int":1}',
       'topic' => 'my-topic',
-      'key' => 'foo1'
+      'key' => '{"test_id":"foo1"}'
     )
     expect(records[1].attributes.to_h).to include(
-      'message' => '{"foo"=>2}',
+      'message' => '{"test_id":"foo2","some_int":2}',
       'topic' => 'my-topic',
-      'key' => 'foo2'
+      'key' => '{"test_id":"foo2"}'
     )
     expect(records[2].attributes.to_h).to include(
-      'message' => '{"foo"=>3}',
+      'message' => '{"test_id":"foo3","some_int":3}',
       'topic' => 'my-topic',
-      'key' => 'foo3'
+      'key' => '{"test_id":"foo3"}'
     )
   end
 
@@ -37,18 +37,12 @@ each_db_config(Deimos::Backends::Db) do
   end
 
   it 'should add to non-keyed messages' do
+    orig_messages = messages.deep_dup
     described_class.publish(producer_class: MyNoKeyProducer,
                             messages: messages)
     expect(Deimos::KafkaMessage.count).to eq(3)
     described_class.publish(producer_class: MyNoKeyProducer,
-                            messages: [messages.first])
+                            messages: [orig_messages.first])
     expect(Deimos::KafkaMessage.count).to eq(4)
   end
-
-  it 'should add messages with Hash keys with JSON encoding' do
-    described_class.publish(producer_class: MyProducer,
-                            messages: [build_message({ foo: 0 }, 'my-topic', { 'test_id' => 0 })])
-    expect(Deimos::KafkaMessage.count).to eq(1)
-    expect(Deimos::KafkaMessage.last.partition_key).to eq(%(---\ntest_id: 0\n))
-  end
 end