deimos-ruby 1.8.2.pre.beta1 → 1.8.5

data/spec/backends/db_spec.rb

@@ -43,6 +43,12 @@ each_db_config(Deimos::Backends::Db) do
     described_class.publish(producer_class: MyNoKeyProducer,
                             messages: [messages.first])
     expect(Deimos::KafkaMessage.count).to eq(4)
+  end
 
+  it 'should add messages with Hash keys with JSON encoding' do
+    described_class.publish(producer_class: MyProducer,
+                            messages: [build_message({ foo: 0 }, 'my-topic', { 'test_id' => 0 })])
+    expect(Deimos::KafkaMessage.count).to eq(1)
+    expect(Deimos::KafkaMessage.last.partition_key).to eq(%(---\ntest_id: 0\n))
   end
 end

data/spec/config/configuration_spec.rb

@@ -6,6 +6,14 @@ class MyConfigConsumer < Deimos::Consumer
   def consume
   end
 end
+
+# Mock consumer 2
+class MyConfigConsumer2 < Deimos::Consumer
+  # :no-doc:
+  def consume
+  end
+end
+
 describe Deimos, 'configuration' do
   it 'should configure with deprecated fields' do
     logger = Logger.new(nil)
@@ -171,6 +179,13 @@ describe Deimos, 'configuration' do
         offset_retention_time 13
         heartbeat_interval 13
       end
+      consumer do
+        disabled true
+        class_name 'MyConfigConsumer2'
+        schema 'blah2'
+        topic 'blah2'
+        group_id 'myconsumerid2'
+      end
     end
 
     expect(described_class.config.phobos_config).

data/spec/generators/active_record_generator_spec.rb

@@ -16,7 +16,7 @@ RSpec.describe Deimos::Generators::ActiveRecordGenerator do
     files = Dir['db/migrate/*.rb']
     expect(files.length).to eq(1)
     results = <<~MIGRATION
-      class CreateGeneratedTable < ActiveRecord::Migration[6.0]
+      class CreateGeneratedTable < ActiveRecord::Migration[6.1]
         def up
           if table_exists?(:generated_table)
             warn "generated_table already exists, exiting"

data/spec/kafka_source_spec.rb

@@ -225,5 +225,88 @@ module KafkaSourceSpec
         expect(Deimos::KafkaMessage.count).to eq(0)
       end
     end
+
+    context 'with import hooks disabled' do
+      before(:each) do
+        # Dummy class we can include the mixin in. Has a backing table created
+        # earlier and has the import hook disabled
+        class WidgetNoImportHook < ActiveRecord::Base
+          include Deimos::KafkaSource
+          self.table_name = 'widgets'
+
+          # :nodoc:
+          def self.kafka_config
+            {
+              update: true,
+              delete: true,
+              import: false,
+              create: true
+            }
+          end
+
+          # :nodoc:
+          def self.kafka_producers
+            [WidgetProducer]
+          end
+        end
+        WidgetNoImportHook.reset_column_information
+      end
+
+      it 'should not fail when bulk-importing with existing records' do
+        widget1 = WidgetNoImportHook.create(widget_id: 1, name: 'Widget 1')
+        widget2 = WidgetNoImportHook.create(widget_id: 2, name: 'Widget 2')
+        widget1.name = 'New Widget No Import Hook 1'
+        widget2.name = 'New Widget No Import Hook 2'
+
+        expect {
+          WidgetNoImportHook.import([widget1, widget2], on_duplicate_key_update: %i(widget_id name))
+        }.not_to raise_error
+
+        expect('my-topic').not_to have_sent({
+                                              widget_id: 1,
+                                              name: 'New Widget No Import Hook 1',
+                                              id: widget1.id,
+                                              created_at: anything,
+                                              updated_at: anything
+                                            }, widget1.id)
+        expect('my-topic').not_to have_sent({
+                                              widget_id: 2,
+                                              name: 'New Widget No Import Hook 2',
+                                              id: widget2.id,
+                                              created_at: anything,
+                                              updated_at: anything
+                                            }, widget2.id)
+      end
+
+      it 'should not fail when mixing existing and new records' do
+        widget1 = WidgetNoImportHook.create(widget_id: 1, name: 'Widget 1')
+        expect('my-topic').to have_sent({
+                                          widget_id: 1,
+                                          name: 'Widget 1',
+                                          id: widget1.id,
+                                          created_at: anything,
+                                          updated_at: anything
+                                        }, widget1.id)
+
+        widget2 = WidgetNoImportHook.new(widget_id: 2, name: 'Widget 2')
+        widget1.name = 'New Widget 1'
+        WidgetNoImportHook.import([widget1, widget2], on_duplicate_key_update: %i(widget_id))
+        widgets = WidgetNoImportHook.all
+        expect('my-topic').not_to have_sent({
+                                              widget_id: 1,
+                                              name: 'New Widget 1',
+                                              id: widgets[0].id,
+                                              created_at: anything,
+                                              updated_at: anything
+                                            }, widgets[0].id)
+        expect('my-topic').not_to have_sent({
+                                              widget_id: 2,
+                                              name: 'Widget 2',
+                                              id: widgets[1].id,
+                                              created_at: anything,
+                                              updated_at: anything
+                                            }, widgets[1].id)
+      end
+    end
   end
 end

data/spec/kafka_topic_info_spec.rb

@@ -51,13 +51,13 @@ each_db_config(Deimos::KafkaTopicInfo) do
       expect(record.locked_at).to eq(nil)
       expect(record.error).to eq(false)
       expect(record.retries).to eq(0)
-      expect(record.last_processed_at.to_s).to eq(Time.zone.now.to_s)
+      expect(record.last_processed_at.in_time_zone.to_s).to eq(Time.zone.now.to_s)
       record = Deimos::KafkaTopicInfo.last
       expect(record.locked_by).not_to eq(nil)
       expect(record.locked_at).not_to eq(nil)
       expect(record.error).not_to eq(false)
       expect(record.retries).not_to eq(0)
-      expect(record.last_processed_at.to_s).to eq(20.seconds.ago.to_s)
+      expect(record.last_processed_at.in_time_zone.to_s).to eq(20.seconds.ago.to_s)
     end
   end
 
@@ -70,11 +70,11 @@ each_db_config(Deimos::KafkaTopicInfo) do
                                           locked_by: 'me', locked_at: 1.minute.ago)
 
       expect(Deimos::KafkaTopicInfo.count).to eq(3)
-      Deimos::KafkaTopicInfo.all.each { |t| expect(t.last_processed_at.to_s).to eq(old_time) }
+      Deimos::KafkaTopicInfo.all.each { |t| expect(t.last_processed_at.in_time_zone.to_s).to eq(old_time) }
       Deimos::KafkaTopicInfo.ping_empty_topics(%w(topic1))
-      expect(t1.reload.last_processed_at.to_s).to eq(old_time) # was passed as an exception
-      expect(t2.reload.last_processed_at.to_s).to eq(Time.zone.now.to_s)
-      expect(t3.reload.last_processed_at.to_s).to eq(old_time) # is locked
+      expect(t1.reload.last_processed_at.in_time_zone.to_s).to eq(old_time) # was passed as an exception
+      expect(t2.reload.last_processed_at.in_time_zone.to_s).to eq(Time.zone.now.to_s)
+      expect(t3.reload.last_processed_at.in_time_zone.to_s).to eq(old_time) # is locked
     end
   end
 

data/spec/producer_spec.rb

@@ -64,6 +64,14 @@ module ProducerTest
       end
       stub_const('MyErrorProducer', producer_class)
 
+      producer_class = Class.new(Deimos::Producer) do
+        schema 'MySchema'
+        namespace 'com.my-namespace'
+        topic nil
+        key_config none: true
+      end
+      stub_const('MyNoTopicProducer', producer_class)
+
     end
 
     it 'should fail on invalid message with error handler' do
@@ -228,6 +236,7 @@ module ProducerTest
     end
 
     it 'should encode the key' do
+      Deimos.configure { |c| c.producers.topic_prefix = nil }
       expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'my-topic-key')
       expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'bar', topic: 'my-topic-key')
       expect(MyProducer.encoder).to receive(:encode).with({
@@ -245,6 +254,21 @@ module ProducerTest
       )
     end
 
+    it 'should encode the key with topic prefix' do
+      Deimos.configure { |c| c.producers.topic_prefix = 'prefix.' }
+      expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'prefix.my-topic-key')
+      expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'bar', topic: 'prefix.my-topic-key')
+      expect(MyProducer.encoder).to receive(:encode).with({  'test_id' => 'foo',
+                                                             'some_int' => 123 },
+                                                          { topic: 'prefix.my-topic-value' })
+      expect(MyProducer.encoder).to receive(:encode).with({  'test_id' => 'bar',
+                                                             'some_int' => 124 },
+                                                          { topic: 'prefix.my-topic-value' })
+
+      MyProducer.publish_list([{ 'test_id' => 'foo', 'some_int' => 123 },
+                               { 'test_id' => 'bar', 'some_int' => 124 }])
+    end
+
     it 'should not encode with plaintext key' do
       expect(MyNonEncodedProducer.key_encoder).not_to receive(:encode_key)
 
@@ -296,6 +320,31 @@ module ProducerTest
       )
     end
 
+    it 'should raise error if blank topic is passed in explicitly' do
+      expect {
+        MyProducer.publish_list(
+          [{  'test_id' => 'foo',
+              'some_int' => 123 },
+           {  'test_id' => 'bar',
+              'some_int' => 124 }],
+          topic: ''
+        )
+      }.to raise_error(RuntimeError,
+                       'Topic not specified. Please specify the topic.')
+    end
+
+    it 'should raise error if the producer has not been initialized with a topic' do
+      expect {
+        MyNoTopicProducer.publish_list(
+          [{  'test_id' => 'foo',
+              'some_int' => 123 },
+           {  'test_id' => 'bar',
+              'some_int' => 124 }]
+        )
+      }.to raise_error(RuntimeError,
+                       'Topic not specified. Please specify the topic.')
+    end
+
     it 'should error with nothing set' do
       expect {
         MyErrorProducer.publish_list(

data/spec/schema_backends/avro_base_shared.rb

@@ -42,6 +42,20 @@ RSpec.shared_examples_for('an Avro backend') do
         {
           'name' => 'union-int-field',
           'type' => %w(null int)
+        },
+        {
+          'name' => 'timestamp-millis-field',
+          'type' => {
+            'type' => 'long',
+            'logicalType' => 'timestamp-millis'
+          }
+        },
+        {
+          'name' => 'timestamp-micros-field',
+          'type' => {
+            'type' => 'long',
+            'logicalType' => 'timestamp-micros'
+          }
         }
       ]
     }
@@ -95,7 +109,9 @@ RSpec.shared_examples_for('an Avro backend') do
         'string-field' => 'hi mom',
         'boolean-field' => true,
         'union-field' => nil,
-        'union-int-field' => nil
+        'union-int-field' => nil,
+        'timestamp-millis-field' => Time.utc(2020, 11, 12, 13, 14, 15, 909_090),
+        'timestamp-micros-field' => Time.utc(2020, 11, 12, 13, 14, 15, 909_090)
       }
     end
 
@@ -169,6 +185,15 @@ RSpec.shared_examples_for('an Avro backend') do
       expect(result['union-field']).to eq('itsme')
     end
 
+    it 'should not convert timestamp-millis' do
+      result = backend.coerce(payload)
+      expect(result['timestamp-millis-field']).to eq(Time.utc(2020, 11, 12, 13, 14, 15, 909_090))
+    end
+
+    it 'should not convert timestamp-micros' do
+      result = backend.coerce(payload)
+      expect(result['timestamp-micros-field']).to eq(Time.utc(2020, 11, 12, 13, 14, 15, 909_090))
+    end
   end
 
 end

data/spec/schemas/com/my-namespace/request/CreateTopic.avsc

@@ -0,0 +1,11 @@
+{
+  "namespace": "com.my-namespace.request",
+  "name": "CreateTopic",
+  "type": "record",
+  "fields": [
+    {
+      "name": "request_id",
+      "type": "string"
+    }
+  ]
+}

data/spec/schemas/com/my-namespace/response/CreateTopic.avsc

@@ -0,0 +1,11 @@
+{
+  "namespace": "com.my-namespace.response",
+  "name": "CreateTopic",
+  "type": "record",
+  "fields": [
+    {
+      "name": "response_id",
+      "type": "string"
+    }
+  ]
+}

data/spec/spec_helper.rb

@@ -87,7 +87,7 @@ module DbConfigs
       port: 3306,
       username: 'root',
       database: 'test',
-      host: ENV['MYSQL_HOST'] || 'localhost'
+      host: ENV['MYSQL_HOST'] || '127.0.0.1'
     },
     {
       adapter: 'sqlite3',

data/spec/utils/db_producer_spec.rb

@@ -96,7 +96,32 @@ each_db_config(Deimos::Utils::DbProducer) do
       expect(phobos_producer).to have_received(:publish_list).with(['A'] * 100).once
       expect(phobos_producer).to have_received(:publish_list).with(['A'] * 10).once
       expect(phobos_producer).to have_received(:publish_list).with(['A']).once
+    end
+
+    it 'should not resend batches of sent messages' do
+      allow(phobos_producer).to receive(:publish_list) do |group|
+        raise Kafka::BufferOverflow if group.any?('A') && group.size >= 1000
+        raise Kafka::BufferOverflow if group.any?('BIG') && group.size >= 10
+      end
+      allow(Deimos.config.metrics).to receive(:increment)
+      batch = ['A'] * 450 + ['BIG'] * 550
+      producer.produce_messages(batch)
+
+      expect(phobos_producer).to have_received(:publish_list).with(batch)
+      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 100).exactly(4).times
+      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 50 + ['BIG'] * 50)
+      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 10).exactly(5).times
+      expect(phobos_producer).to have_received(:publish_list).with(['BIG'] * 1).exactly(550).times
 
+      expect(Deimos.config.metrics).to have_received(:increment).with('publish',
+                                                                      tags: %w(status:success topic:),
+                                                                      by: 100).exactly(4).times
+      expect(Deimos.config.metrics).to have_received(:increment).with('publish',
+                                                                      tags: %w(status:success topic:),
+                                                                      by: 10).exactly(5).times
+      expect(Deimos.config.metrics).to have_received(:increment).with('publish',
+                                                                      tags: %w(status:success topic:),
+                                                                      by: 1).exactly(550).times
     end
 
     describe '#compact_messages' do
@@ -289,6 +314,8 @@ each_db_config(Deimos::Utils::DbProducer) do
           message: "mess#{i}",
           partition_key: "key#{i}"
         )
+      end
+      (5..8).each do |i|
         Deimos::KafkaMessage.create!(
           id: i,
           topic: 'my-topic2',

data/spec/utils/inline_consumer_spec.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+describe Deimos::Utils::SeekListener do
+
+  describe '#start_listener' do
+    let(:consumer) { instance_double(Kafka::Consumer) }
+    let(:handler) { class_double(Deimos::Utils::MessageBankHandler) }
+
+    before(:each) do
+      allow(handler).to receive(:start)
+      allow(consumer).to receive(:subscribe)
+      allow_any_instance_of(Phobos::Listener).to receive(:create_kafka_consumer).and_return(consumer)
+      allow_any_instance_of(Kafka::Client).to receive(:last_offset_for).and_return(100)
+      stub_const('Deimos::Utils::SeekListener::MAX_SEEK_RETRIES', 2)
+    end
+
+    it 'should seek offset' do
+      allow(consumer).to receive(:seek)
+      expect(consumer).to receive(:seek).once
+      seek_listener = described_class.new({ handler: handler, group_id: 999, topic: 'test_topic' })
+      seek_listener.start_listener
+    end
+
+    it 'should retry on errors when seeking offset' do
+      allow(consumer).to receive(:seek).and_raise(StandardError)
+      expect(consumer).to receive(:seek).twice
+      seek_listener = described_class.new({ handler: handler, group_id: 999, topic: 'test_topic' })
+      seek_listener.start_listener
+    end
+  end
+end

data/spec/utils/schema_controller_mixin_spec.rb

@@ -17,6 +17,7 @@ RSpec.describe Deimos::Utils::SchemaControllerMixin, type: :controller do
     request_namespace 'com.my-namespace.request'
     response_namespace 'com.my-namespace.response'
     schemas :index, :show
+    schemas create: 'CreateTopic'
     schemas :update, request: 'UpdateRequest', response: 'UpdateResponse'
 
     # :nodoc:
@@ -29,6 +30,11 @@ RSpec.describe Deimos::Utils::SchemaControllerMixin, type: :controller do
       render_schema({ 'response_id' => payload[:request_id] + ' dad' })
     end
 
+    # :nodoc:
+    def create
+      render_schema({ 'response_id' => payload[:request_id] + ' bro' })
+    end
+
     # :nodoc:
     def update
       render_schema({ 'update_response_id' => payload[:update_request_id] + ' sis' })
@@ -65,4 +71,14 @@ RSpec.describe Deimos::Utils::SchemaControllerMixin, type: :controller do
     expect(response_backend.decode(response.body)).to eq({ 'update_response_id' => 'hi sis' })
   end
 
+  it 'should render the correct response for create' do
+    request_backend = Deimos.schema_backend(schema: 'CreateTopic',
+                                            namespace: 'com.my-namespace.request')
+    response_backend = Deimos.schema_backend(schema: 'CreateTopic',
+                                             namespace: 'com.my-namespace.response')
+    request.content_type = 'avro/binary'
+    post :create, params: { id: 1 }, body: request_backend.encode({ 'request_id' => 'hi' })
+    expect(response_backend.decode(response.body)).to eq({ 'response_id' => 'hi bro' })
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.8.2.pre.beta1
+  version: 1.8.5
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-09-09 00:00:00.000000000 Z
+date: 2021-01-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf
@@ -280,30 +280,30 @@ dependencies:
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0.72'
+        version: 0.88.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0.72'
+        version: 0.88.0
 - !ruby/object:Gem::Dependency
   name: rubocop-rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '1.27'
+        version: 1.42.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '1.27'
+        version: 1.42.0
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -348,7 +348,9 @@ files:
 - docs/ARCHITECTURE.md
 - docs/CONFIGURATION.md
 - docs/DATABASE_BACKEND.md
+- docs/INTEGRATION_TESTS.md
 - docs/PULL_REQUEST_TEMPLATE.md
+- docs/UPGRADING.md
 - lib/deimos.rb
 - lib/deimos/active_record_consume/batch_consumption.rb
 - lib/deimos/active_record_consume/batch_slicer.rb
@@ -453,14 +455,17 @@ files:
 - spec/schemas/com/my-namespace/Wibble.avsc
 - spec/schemas/com/my-namespace/Widget.avsc
 - spec/schemas/com/my-namespace/WidgetTheSecond.avsc
+- spec/schemas/com/my-namespace/request/CreateTopic.avsc
 - spec/schemas/com/my-namespace/request/Index.avsc
 - spec/schemas/com/my-namespace/request/UpdateRequest.avsc
+- spec/schemas/com/my-namespace/response/CreateTopic.avsc
 - spec/schemas/com/my-namespace/response/Index.avsc
 - spec/schemas/com/my-namespace/response/UpdateResponse.avsc
 - spec/spec_helper.rb
 - spec/utils/db_poller_spec.rb
 - spec/utils/db_producer_spec.rb
 - spec/utils/deadlock_retry_spec.rb
+- spec/utils/inline_consumer_spec.rb
 - spec/utils/lag_reporter_spec.rb
 - spec/utils/platform_schema_validation_spec.rb
 - spec/utils/schema_controller_mixin_spec.rb
@@ -483,11 +488,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubygems_version: 3.1.3
+rubygems_version: 3.0.9
 signing_key:
 specification_version: 4
 summary: Kafka libraries for Ruby.
@@ -534,14 +539,17 @@ test_files:
 - spec/schemas/com/my-namespace/Wibble.avsc
 - spec/schemas/com/my-namespace/Widget.avsc
 - spec/schemas/com/my-namespace/WidgetTheSecond.avsc
+- spec/schemas/com/my-namespace/request/CreateTopic.avsc
 - spec/schemas/com/my-namespace/request/Index.avsc
 - spec/schemas/com/my-namespace/request/UpdateRequest.avsc
+- spec/schemas/com/my-namespace/response/CreateTopic.avsc
 - spec/schemas/com/my-namespace/response/Index.avsc
 - spec/schemas/com/my-namespace/response/UpdateResponse.avsc
 - spec/spec_helper.rb
 - spec/utils/db_poller_spec.rb
 - spec/utils/db_producer_spec.rb
 - spec/utils/deadlock_retry_spec.rb
+- spec/utils/inline_consumer_spec.rb
 - spec/utils/lag_reporter_spec.rb
 - spec/utils/platform_schema_validation_spec.rb
 - spec/utils/schema_controller_mixin_spec.rb