deimos-ruby 1.8.2 → 1.8.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 237cb6cd22b6a057003fedae1caeec81c999a3d9b90d837bb8b0a4add43ca4e2
-  data.tar.gz: fab51933cdcd5c5fd10b8c92bbc67854fe56f23cdbec41660579f18b9075e09c
+  metadata.gz: 2badd671866a8fbde743e03acbc2da623279ac08abd5ed983ae9a15a0bef415d
+  data.tar.gz: e4b3cd2e80b13fc00ff54ebc85b59ab2e1daad052ab170dafd9a75f57eb80ce3
 SHA512:
-  metadata.gz: 193b9b593b4f92edecc6c239eb6106fbe40e0acd1453694508f3fb02525166c39e4f1eba71230362493cd1647ab19e98a63ecf55f5371b844e9d0c38ffa5b2ea
-  data.tar.gz: e7d42ec7ec2bc864735c9230a925685935678d2dd0f3918426926dc9f358d5eb810dd5b324bcacaf9201c78bdfbbfa58037a4cac4cf4ca66257737440b3d2c97
+  metadata.gz: 98b41fa7354e624a4538ac5006a091e1a144a2caaa57e2ef77c214e350cd8dcbf2b126c01a3663365d28bbed19baa7525e3c52390cc60f6378537ad8fe6bee12
+  data.tar.gz: 208ba417a35c7042155383d5c0eb9d9e35be95c0459cf2e58cdc24168c8fdf30af16434fcc66c612d197aab38279d790907f146e19b6a36157cde008c5aa60a3

data/CHANGELOG.md

@@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+## 1.8.3 - 2020-11-18
+
+### Fixes :wrench:
+- Do not resend already sent messages when splitting up batches
+  (fixes [#24](https://github.com/flipp-oss/deimos/issues/24))
+- KafkaSource crashing on bulk-imports if import hooks are disabled
+  (fixes [#73](https://github.com/flipp-oss/deimos/issues/73))
+- #96 Use string-safe encoding for partition keys
+
 ## 1.8.2 - 2020-09-25
 
 ### Features :star:

data/README.md

@@ -42,6 +42,7 @@ Please see the following for further information not covered by this readme:
 * [Configuration Reference](docs/CONFIGURATION.md)
 * [Database Backend Feature](docs/DATABASE_BACKEND.md)
 * [Upgrading Deimos](docs/UPGRADING.md)
+* [Contributing to Integration Tests](docs/INTEGRATION_TESTS.md)
 
 # Installation
 

data/docs/INTEGRATION_TESTS.md

@@ -0,0 +1,52 @@
+# Running Integration Tests
+
+This repo includes integration tests in the [spec/utils](spec/utils) directory.
+Here, there are tests for more deimos features that include a database integration like
+* [Database Poller](README.md#Database Poller)
+* [Database Backend](docs/DATABASE_BACKEND.md)
+* [Deadlock Retrying](lib/deimos/utils/deadlock_retry.rb)
+
+You will need to set up the following databases to develop and create unit tests in these test suites.
+* [SQLite](#SQLite)
+* [MySQL](#MySQL)
+* [PostgreSQL](#PostgreSQL)
+
+### SQLite
+This database is covered through the `sqlite3` gem. 
+
+## MySQL
+### Setting up a local MySQL server (Mac)
+```bash
+# Download MySQL (Optionally, choose a version you are comfortable with)
+brew install mysql
+# Start automatically after rebooting your machine
+brew services start mysql
+
+# Cleanup once you are done with MySQL
+brew services stop mysql
+```
+
+## PostgreSQL
+### Setting up a local PostgreSQL server (Mac)
+```bash
+# Install postgres if it's not already installed
+brew install postgres
+
+# Initialize and Start up postgres db
+brew services start postgres
+initdb /usr/local/var/postgres
+# Create the default database and user
+# Use the password "root"
+createuser -s --password postgres
+
+# Cleanup once done with Postgres
+killall postgres
+brew services stop postgres
+```
+
+## Running Integration Tests
+You must specify the tag "integration" when running these these test suites.
+This can be done through the CLI with the `--tag integration` argument.
+```bash
+rspec spec/utils/ --tag integration
+```

data/lib/deimos/backends/db.rb

@@ -14,7 +14,7 @@ module Deimos
             message = Deimos::KafkaMessage.new(
               message: m.encoded_payload ? m.encoded_payload.to_s.b : nil,
               topic: m.topic,
-              partition_key: m.partition_key || m.key
+              partition_key: partition_key_for(m)
             )
             message.key = m.encoded_key.to_s.b unless producer_class.config[:no_keys]
             message
@@ -26,6 +26,15 @@ module Deimos
             by: records.size
           )
         end
+
+        # @param message [Deimos::Message]
+        # @return [String] the partition key to use for this message
+        def partition_key_for(message)
+          return message.partition_key if message.partition_key.present?
+          return message.key unless message.key.is_a?(Hash)
+
+          message.key.to_yaml
+        end
       end
     end
   end

data/lib/deimos/config/configurable.rb

@@ -15,6 +15,11 @@ module Deimos
   #       enabled true
   #       ca_cert_file 'my_file'
   #     end
+  #     config.kafka do
+  #       ssl do
+  #         enabled true
+  #       end
+  #     end
   #   end
   # - Allows for arrays of configurations:
   #   Deimos.configure do |config|

data/lib/deimos/kafka_source.rb

@@ -88,8 +88,9 @@ module Deimos
                                                   array_of_attributes,
                                                   options={})
         results = super
-        return unless self.kafka_config[:import]
-        return if array_of_attributes.empty?
+        if !self.kafka_config[:import] || array_of_attributes.empty?
+          return results
+        end
 
         # This will contain an array of hashes, where each hash is the actual
         # attribute hash that created the object.

data/lib/deimos/utils/db_producer.rb

@@ -190,11 +190,14 @@ module Deimos
         end
       end
 
+      # Produce messages in batches, reducing the size 1/10 if the batch is too
+      # large. Does not retry batches of messages that have already been sent.
       # @param batch [Array<Hash>]
       def produce_messages(batch)
         batch_size = batch.size
+        current_index = 0
         begin
-          batch.in_groups_of(batch_size, false).each do |group|
+          batch[current_index..-1].in_groups_of(batch_size, false).each do |group|
             @logger.debug("Publishing #{group.size} messages to #{@current_topic}")
             producer.publish_list(group)
             Deimos.config.metrics&.increment(
@@ -202,6 +205,7 @@ module Deimos
               tags: %W(status:success topic:#{@current_topic}),
               by: group.size
             )
+            current_index += group.size
             @logger.info("Sent #{group.size} messages to #{@current_topic}")
           end
         rescue Kafka::BufferOverflow, Kafka::MessageSizeTooLarge,

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.8.2'
+  VERSION = '1.8.3'
 end

data/spec/backends/db_spec.rb

@@ -43,6 +43,12 @@ each_db_config(Deimos::Backends::Db) do
     described_class.publish(producer_class: MyNoKeyProducer,
                             messages: [messages.first])
     expect(Deimos::KafkaMessage.count).to eq(4)
+  end
 
+  it 'should add messages with Hash keys with JSON encoding' do
+    described_class.publish(producer_class: MyProducer,
+                            messages: [build_message({ foo: 0 }, 'my-topic', { 'test_id' => 0 })])
+    expect(Deimos::KafkaMessage.count).to eq(1)
+    expect(Deimos::KafkaMessage.last.partition_key).to eq(%(---\ntest_id: 0\n))
   end
 end

data/spec/kafka_source_spec.rb

@@ -225,5 +225,88 @@ module KafkaSourceSpec
         expect(Deimos::KafkaMessage.count).to eq(0)
       end
     end
+
+    context 'with import hooks disabled' do
+      before(:each) do
+        # Dummy class we can include the mixin in. Has a backing table created
+        # earlier and has the import hook disabled
+        class WidgetNoImportHook < ActiveRecord::Base
+          include Deimos::KafkaSource
+          self.table_name = 'widgets'
+
+          # :nodoc:
+          def self.kafka_config
+            {
+              update: true,
+              delete: true,
+              import: false,
+              create: true
+            }
+          end
+
+          # :nodoc:
+          def self.kafka_producers
+            [WidgetProducer]
+          end
+        end
+        WidgetNoImportHook.reset_column_information
+      end
+
+      it 'should not fail when bulk-importing with existing records' do
+        widget1 = WidgetNoImportHook.create(widget_id: 1, name: 'Widget 1')
+        widget2 = WidgetNoImportHook.create(widget_id: 2, name: 'Widget 2')
+        widget1.name = 'New Widget No Import Hook 1'
+        widget2.name = 'New Widget No Import Hook 2'
+
+        expect {
+          WidgetNoImportHook.import([widget1, widget2], on_duplicate_key_update: %i(widget_id name))
+        }.not_to raise_error
+
+        expect('my-topic').not_to have_sent({
+                                              widget_id: 1,
+                                              name: 'New Widget No Import Hook 1',
+                                              id: widget1.id,
+                                              created_at: anything,
+                                              updated_at: anything
+                                            }, widget1.id)
+        expect('my-topic').not_to have_sent({
+                                              widget_id: 2,
+                                              name: 'New Widget No Import Hook 2',
+                                              id: widget2.id,
+                                              created_at: anything,
+                                              updated_at: anything
+                                            }, widget2.id)
+      end
+
+      it 'should not fail when mixing existing and new records' do
+        widget1 = WidgetNoImportHook.create(widget_id: 1, name: 'Widget 1')
+        expect('my-topic').to have_sent({
+                                          widget_id: 1,
+                                          name: 'Widget 1',
+                                          id: widget1.id,
+                                          created_at: anything,
+                                          updated_at: anything
+                                        }, widget1.id)
+
+        widget2 = WidgetNoImportHook.new(widget_id: 2, name: 'Widget 2')
+        widget1.name = 'New Widget 1'
+        WidgetNoImportHook.import([widget1, widget2], on_duplicate_key_update: %i(widget_id))
+        widgets = WidgetNoImportHook.all
+        expect('my-topic').not_to have_sent({
+                                              widget_id: 1,
+                                              name: 'New Widget 1',
+                                              id: widgets[0].id,
+                                              created_at: anything,
+                                              updated_at: anything
+                                            }, widgets[0].id)
+        expect('my-topic').not_to have_sent({
+                                              widget_id: 2,
+                                              name: 'Widget 2',
+                                              id: widgets[1].id,
+                                              created_at: anything,
+                                              updated_at: anything
+                                            }, widgets[1].id)
+      end
+    end
   end
 end

data/spec/utils/db_producer_spec.rb

@@ -96,7 +96,32 @@ each_db_config(Deimos::Utils::DbProducer) do
       expect(phobos_producer).to have_received(:publish_list).with(['A'] * 100).once
       expect(phobos_producer).to have_received(:publish_list).with(['A'] * 10).once
       expect(phobos_producer).to have_received(:publish_list).with(['A']).once
+    end
+
+    it 'should not resend batches of sent messages' do
+      allow(phobos_producer).to receive(:publish_list) do |group|
+        raise Kafka::BufferOverflow if group.any?('A') && group.size >= 1000
+        raise Kafka::BufferOverflow if group.any?('BIG') && group.size >= 10
+      end
+      allow(Deimos.config.metrics).to receive(:increment)
+      batch = ['A'] * 450 + ['BIG'] * 550
+      producer.produce_messages(batch)
+
+      expect(phobos_producer).to have_received(:publish_list).with(batch)
+      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 100).exactly(4).times
+      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 50 + ['BIG'] * 50)
+      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 10).exactly(5).times
+      expect(phobos_producer).to have_received(:publish_list).with(['BIG'] * 1).exactly(550).times
 
+      expect(Deimos.config.metrics).to have_received(:increment).with('publish',
+                                                                      tags: %w(status:success topic:),
+                                                                      by: 100).exactly(4).times
+      expect(Deimos.config.metrics).to have_received(:increment).with('publish',
+                                                                      tags: %w(status:success topic:),
+                                                                      by: 10).exactly(5).times
+      expect(Deimos.config.metrics).to have_received(:increment).with('publish',
+                                                                      tags: %w(status:success topic:),
+                                                                      by: 1).exactly(550).times
     end
 
     describe '#compact_messages' do
@@ -289,6 +314,8 @@ each_db_config(Deimos::Utils::DbProducer) do
           message: "mess#{i}",
           partition_key: "key#{i}"
         )
+      end
+      (5..8).each do |i|
         Deimos::KafkaMessage.create!(
           id: i,
           topic: 'my-topic2',

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.8.2
+  version: 1.8.3
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-09-25 00:00:00.000000000 Z
+date: 2020-11-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf
@@ -348,6 +348,7 @@ files:
 - docs/ARCHITECTURE.md
 - docs/CONFIGURATION.md
 - docs/DATABASE_BACKEND.md
+- docs/INTEGRATION_TESTS.md
 - docs/PULL_REQUEST_TEMPLATE.md
 - docs/UPGRADING.md
 - lib/deimos.rb