deimos-ruby 1.8.2 → 1.8.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 237cb6cd22b6a057003fedae1caeec81c999a3d9b90d837bb8b0a4add43ca4e2
-  data.tar.gz: fab51933cdcd5c5fd10b8c92bbc67854fe56f23cdbec41660579f18b9075e09c
+  metadata.gz: 3c12e06b13f4b1910a93d955ddca456bdfaa0d8c5ee1a925cc1aea5c2d0f4df3
+  data.tar.gz: b4ef92834678153a746b585a0c6c5ff36cb32fc65cd3fc94accc8a35e7fd6feb
 SHA512:
-  metadata.gz: 193b9b593b4f92edecc6c239eb6106fbe40e0acd1453694508f3fb02525166c39e4f1eba71230362493cd1647ab19e98a63ecf55f5371b844e9d0c38ffa5b2ea
-  data.tar.gz: e7d42ec7ec2bc864735c9230a925685935678d2dd0f3918426926dc9f358d5eb810dd5b324bcacaf9201c78bdfbbfa58037a4cac4cf4ca66257737440b3d2c97
+  metadata.gz: 4c3eaf327c4483302847733e1a5134b2f559f4e0ed2e7b090d893292d0cde80ad7f9677c54b0f3b415d3933e377013c286fc133399705803419f0df6cb446f75
+  data.tar.gz: a3bb3aa32da8ed98ee4bb763cd5c123f1289042219e85b219629c9e6b4a3ddd75870e0c7d9ea74b0e7d24a6edc707c5d0af39b3e6847de3da3d271679113040f

data/CHANGELOG.md

@@ -7,6 +7,41 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+## 1.8.7 - 2021-01-14
+
+- ### Roadmap :car:
+- Update Phobos version to allow version 1.9 or 2.x.
+
+## 1.8.6 - 2021-01-14
+
+- ### Fixes :wrench:
+- Fix for configuration bug with Ruby 3.0 (** instead of passing hash)
+
+## 1.8.5 - 2021-01-13
+
+- ### Fixes :wrench:
+- Fixes for Rails 6.1 (remove usage of `update_attributes!`)
+
+## 1.8.4 - 2020-12-02
+
+### Features :star:
+- Add overridable "process_message?" method to ActiveRecordConsumer to allow for skipping of saving/updating records
+
+### Fixes :wrench:
+
+- Do not apply type coercion to `timestamp-millis` and `timestamp-micros` logical types (fixes [#97](https://github.com/flipp-oss/deimos/issues/97))
+
+## 1.8.3 - 2020-11-18
+
+### Fixes :wrench:
+- Do not resend already sent messages when splitting up batches
+  (fixes [#24](https://github.com/flipp-oss/deimos/issues/24))
+- KafkaSource crashing on bulk-imports if import hooks are disabled
+  (fixes [#73](https://github.com/flipp-oss/deimos/issues/73))
+- #96 Use string-safe encoding for partition keys
+- Retry on offset seek failures in inline consumer
+  (fixes [#5](Inline consumer should use retries when seeking))
+
 ## 1.8.2 - 2020-09-25
 
 ### Features :star:

data/Gemfile.lock

@@ -1,72 +1,76 @@
 PATH
   remote: .
   specs:
-    deimos-ruby (1.8.2)
+    deimos-ruby (1.8.7)
       avro_turf (~> 0.11)
-      phobos (~> 1.9)
+      phobos (>= 1.9, < 3.0)
       ruby-kafka (~> 0.7)
       sigurd (= 0.0.1)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (6.0.3.2)
-      actionpack (= 6.0.3.2)
+    actioncable (6.1.1)
+      actionpack (= 6.1.1)
+      activesupport (= 6.1.1)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailbox (6.0.3.2)
-      actionpack (= 6.0.3.2)
-      activejob (= 6.0.3.2)
-      activerecord (= 6.0.3.2)
-      activestorage (= 6.0.3.2)
-      activesupport (= 6.0.3.2)
+    actionmailbox (6.1.1)
+      actionpack (= 6.1.1)
+      activejob (= 6.1.1)
+      activerecord (= 6.1.1)
+      activestorage (= 6.1.1)
+      activesupport (= 6.1.1)
       mail (>= 2.7.1)
-    actionmailer (6.0.3.2)
-      actionpack (= 6.0.3.2)
-      actionview (= 6.0.3.2)
-      activejob (= 6.0.3.2)
+    actionmailer (6.1.1)
+      actionpack (= 6.1.1)
+      actionview (= 6.1.1)
+      activejob (= 6.1.1)
+      activesupport (= 6.1.1)
       mail (~> 2.5, >= 2.5.4)
       rails-dom-testing (~> 2.0)
-    actionpack (6.0.3.2)
-      actionview (= 6.0.3.2)
-      activesupport (= 6.0.3.2)
-      rack (~> 2.0, >= 2.0.8)
+    actionpack (6.1.1)
+      actionview (= 6.1.1)
+      activesupport (= 6.1.1)
+      rack (~> 2.0, >= 2.0.9)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actiontext (6.0.3.2)
-      actionpack (= 6.0.3.2)
-      activerecord (= 6.0.3.2)
-      activestorage (= 6.0.3.2)
-      activesupport (= 6.0.3.2)
+    actiontext (6.1.1)
+      actionpack (= 6.1.1)
+      activerecord (= 6.1.1)
+      activestorage (= 6.1.1)
+      activesupport (= 6.1.1)
       nokogiri (>= 1.8.5)
-    actionview (6.0.3.2)
-      activesupport (= 6.0.3.2)
+    actionview (6.1.1)
+      activesupport (= 6.1.1)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.1, >= 1.2.0)
-    activejob (6.0.3.2)
-      activesupport (= 6.0.3.2)
+    activejob (6.1.1)
+      activesupport (= 6.1.1)
       globalid (>= 0.3.6)
-    activemodel (6.0.3.2)
-      activesupport (= 6.0.3.2)
-    activerecord (6.0.3.2)
-      activemodel (= 6.0.3.2)
-      activesupport (= 6.0.3.2)
-    activerecord-import (1.0.5)
+    activemodel (6.1.1)
+      activesupport (= 6.1.1)
+    activerecord (6.1.1)
+      activemodel (= 6.1.1)
+      activesupport (= 6.1.1)
+    activerecord-import (1.0.7)
       activerecord (>= 3.2)
-    activestorage (6.0.3.2)
-      actionpack (= 6.0.3.2)
-      activejob (= 6.0.3.2)
-      activerecord (= 6.0.3.2)
+    activestorage (6.1.1)
+      actionpack (= 6.1.1)
+      activejob (= 6.1.1)
+      activerecord (= 6.1.1)
+      activesupport (= 6.1.1)
       marcel (~> 0.3.1)
-    activesupport (6.0.3.2)
+      mimemagic (~> 0.3.2)
+    activesupport (6.1.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (>= 0.7, < 2)
-      minitest (~> 5.1)
-      tzinfo (~> 1.1)
-      zeitwerk (~> 2.2, >= 2.2.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
     ast (2.4.1)
     avro (1.9.2)
       multi_json
@@ -75,19 +79,19 @@ GEM
       excon (~> 0.45)
     builder (3.2.4)
     coderay (1.1.3)
-    concurrent-ruby (1.1.6)
-    concurrent-ruby-ext (1.1.6)
-      concurrent-ruby (= 1.1.6)
+    concurrent-ruby (1.1.7)
+    concurrent-ruby-ext (1.1.7)
+      concurrent-ruby (= 1.1.7)
     crass (1.0.6)
     database_cleaner (1.8.5)
     ddtrace (0.37.0)
       msgpack
     diff-lcs (1.4.4)
-    digest-crc (0.6.1)
-      rake (~> 13.0)
+    digest-crc (0.6.3)
+      rake (>= 12.0.0, < 14.0.0)
     dogstatsd-ruby (4.8.1)
-    erubi (1.9.0)
-    excon (0.76.0)
+    erubi (1.10.0)
+    excon (0.78.1)
     exponential-backoff (0.0.4)
     ffi (1.13.1)
     formatador (0.2.5)
@@ -110,7 +114,7 @@ GEM
     guard-rubocop (1.3.0)
       guard (~> 2.0)
       rubocop (~> 0.20)
-    i18n (1.8.4)
+    i18n (1.8.7)
       concurrent-ruby (~> 1.0)
     listen (3.2.1)
       rb-fsevent (~> 0.10, >= 0.10.3)
@@ -119,7 +123,7 @@ GEM
     logging (2.3.0)
       little-plugger (~> 1.1)
       multi_json (~> 1.14)
-    loofah (2.6.0)
+    loofah (2.8.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     lumberjack (1.2.6)
@@ -130,15 +134,16 @@ GEM
     method_source (1.0.0)
     mimemagic (0.3.5)
     mini_mime (1.0.2)
-    mini_portile2 (2.4.0)
-    minitest (5.14.1)
+    mini_portile2 (2.5.0)
+    minitest (5.14.3)
     msgpack (1.3.3)
     multi_json (1.15.0)
     mysql2 (0.5.3)
     nenv (0.3.0)
-    nio4r (2.5.2)
-    nokogiri (1.10.10)
-      mini_portile2 (~> 2.4.0)
+    nio4r (2.5.4)
+    nokogiri (1.11.1)
+      mini_portile2 (~> 2.5.0)
+      racc (~> 1.4)
     notiffany (0.1.3)
       nenv (~> 0.1)
       shellany (~> 0.0)
@@ -146,7 +151,7 @@ GEM
     parser (2.7.1.4)
       ast (~> 2.4.1)
     pg (1.2.3)
-    phobos (1.9.0)
+    phobos (2.0.1)
       activesupport (>= 3.0.0)
       concurrent-ruby (>= 1.0.2)
       concurrent-ruby-ext (>= 1.0.2)
@@ -157,37 +162,38 @@ GEM
     pry (0.13.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
+    racc (1.5.2)
     rack (2.2.3)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
-    rails (6.0.3.2)
-      actioncable (= 6.0.3.2)
-      actionmailbox (= 6.0.3.2)
-      actionmailer (= 6.0.3.2)
-      actionpack (= 6.0.3.2)
-      actiontext (= 6.0.3.2)
-      actionview (= 6.0.3.2)
-      activejob (= 6.0.3.2)
-      activemodel (= 6.0.3.2)
-      activerecord (= 6.0.3.2)
-      activestorage (= 6.0.3.2)
-      activesupport (= 6.0.3.2)
-      bundler (>= 1.3.0)
-      railties (= 6.0.3.2)
+    rails (6.1.1)
+      actioncable (= 6.1.1)
+      actionmailbox (= 6.1.1)
+      actionmailer (= 6.1.1)
+      actionpack (= 6.1.1)
+      actiontext (= 6.1.1)
+      actionview (= 6.1.1)
+      activejob (= 6.1.1)
+      activemodel (= 6.1.1)
+      activerecord (= 6.1.1)
+      activestorage (= 6.1.1)
+      activesupport (= 6.1.1)
+      bundler (>= 1.15.0)
+      railties (= 6.1.1)
       sprockets-rails (>= 2.0.0)
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
     rails-html-sanitizer (1.3.0)
       loofah (~> 2.3)
-    railties (6.0.3.2)
-      actionpack (= 6.0.3.2)
-      activesupport (= 6.0.3.2)
+    railties (6.1.1)
+      actionpack (= 6.1.1)
+      activesupport (= 6.1.1)
       method_source
       rake (>= 0.8.7)
-      thor (>= 0.20.3, < 2.0)
+      thor (~> 1.0)
     rainbow (3.0.0)
-    rake (13.0.1)
+    rake (13.0.3)
     rb-fsevent (0.10.4)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
@@ -239,20 +245,19 @@ GEM
     sprockets (4.0.2)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
-    sprockets-rails (3.2.1)
+    sprockets-rails (3.2.2)
       actionpack (>= 4.0)
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)
     sqlite3 (1.4.2)
     thor (1.0.1)
-    thread_safe (0.3.6)
-    tzinfo (1.2.7)
-      thread_safe (~> 0.1)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
     unicode-display_width (1.7.0)
     websocket-driver (0.7.3)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
-    zeitwerk (2.4.0)
+    zeitwerk (2.4.2)
 
 PLATFORMS
   ruby
@@ -274,9 +279,9 @@ DEPENDENCIES
   rspec (~> 3)
   rspec-rails (~> 4)
   rspec_junit_formatter (~> 0.3)
-  rubocop (~> 0.72)
-  rubocop-rspec (~> 1.27)
+  rubocop (= 0.88.0)
+  rubocop-rspec (= 1.42.0)
   sqlite3 (~> 1.3)
 
 BUNDLED WITH
-   2.1.4
+   2.2.5

data/README.md

@@ -42,6 +42,7 @@ Please see the following for further information not covered by this readme:
 * [Configuration Reference](docs/CONFIGURATION.md)
 * [Database Backend Feature](docs/DATABASE_BACKEND.md)
 * [Upgrading Deimos](docs/UPGRADING.md)
+* [Contributing to Integration Tests](docs/INTEGRATION_TESTS.md)
 
 # Installation
 
@@ -625,6 +626,15 @@ class MyConsumer < Deimos::ActiveRecordConsumer
   def record_key(payload)
     super
   end
+
+  # Optional override, returns true by default.
+  # When this method returns true, a record corresponding to the message
+  # is created/updated.
+  # When this method returns false, message processing is skipped and a
+  # corresponding record will NOT be created/updated.
+  def process_message?(payload)
+    super
+  end
 end
 ```
 

data/deimos-ruby.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_runtime_dependency('avro_turf', '~> 0.11')
-  spec.add_runtime_dependency('phobos', '~> 1.9')
+  spec.add_runtime_dependency('phobos', '>= 1.9', '< 3.0')
   spec.add_runtime_dependency('ruby-kafka', '~> 0.7')
   spec.add_runtime_dependency('sigurd', '0.0.1')
 
@@ -38,7 +38,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency('rspec', '~> 3')
   spec.add_development_dependency('rspec_junit_formatter', '~>0.3')
   spec.add_development_dependency('rspec-rails', '~> 4')
-  spec.add_development_dependency('rubocop', '~> 0.72')
-  spec.add_development_dependency('rubocop-rspec', '~> 1.27')
+  spec.add_development_dependency('rubocop', '0.88.0')
+  spec.add_development_dependency('rubocop-rspec', '1.42.0')
   spec.add_development_dependency('sqlite3', '~> 1.3')
 end

data/docs/INTEGRATION_TESTS.md

@@ -0,0 +1,52 @@
+# Running Integration Tests
+
+This repo includes integration tests in the [spec/utils](spec/utils) directory.
+Here, there are tests for more deimos features that include a database integration like
+* [Database Poller](README.md#Database Poller)
+* [Database Backend](docs/DATABASE_BACKEND.md)
+* [Deadlock Retrying](lib/deimos/utils/deadlock_retry.rb)
+
+You will need to set up the following databases to develop and create unit tests in these test suites.
+* [SQLite](#SQLite)
+* [MySQL](#MySQL)
+* [PostgreSQL](#PostgreSQL)
+
+### SQLite
+This database is covered through the `sqlite3` gem. 
+
+## MySQL
+### Setting up a local MySQL server (Mac)
+```bash
+# Download MySQL (Optionally, choose a version you are comfortable with)
+brew install mysql
+# Start automatically after rebooting your machine
+brew services start mysql
+
+# Cleanup once you are done with MySQL
+brew services stop mysql
+```
+
+## PostgreSQL
+### Setting up a local PostgreSQL server (Mac)
+```bash
+# Install postgres if it's not already installed
+brew install postgres
+
+# Initialize and Start up postgres db
+brew services start postgres
+initdb /usr/local/var/postgres
+# Create the default database and user
+# Use the password "root"
+createuser -s --password postgres
+
+# Cleanup once done with Postgres
+killall postgres
+brew services stop postgres
+```
+
+## Running Integration Tests
+You must specify the tag "integration" when running these these test suites.
+This can be done through the CLI with the `--tag integration` argument.
+```bash
+rspec spec/utils/ --tag integration
+```

data/docs/PULL_REQUEST_TEMPLATE.md

@@ -28,6 +28,7 @@ Please describe the tests that you ran to verify your changes. Provide instructi
 - [ ] I have performed a self-review of my own code
 - [ ] I have commented my code, particularly in hard-to-understand areas
 - [ ] I have made corresponding changes to the documentation
+- [ ] I have added a line in the CHANGELOG describing this change, under the UNRELEASED heading
 - [ ] My changes generate no new warnings
 - [ ] I have added tests that prove my fix is effective or that my feature works
 - [ ] New and existing unit tests pass locally with my changes

data/lib/deimos/active_record_consume/message_consumption.rb

@@ -26,6 +26,15 @@ module Deimos
 
       # :nodoc:
       def consume(payload, metadata)
+        unless self.process_message?(payload)
+          Deimos.config.logger.debug(
+            message: 'Skipping processing of message',
+            payload: payload,
+            metadata: metadata
+          )
+          return
+        end
+
         key = metadata.with_indifferent_access[:key]
         klass = self.class.config[:record_class]
         record = fetch_record(klass, (payload || {}).with_indifferent_access, key)

data/lib/deimos/active_record_consumer.rb

@@ -55,5 +55,13 @@ module Deimos
     def record_attributes(payload, _key=nil)
       @converter.convert(payload)
     end
+
+    # Override this message to conditionally save records
+    # @param payload [Hash] The kafka message as a hash
+    # @return [Boolean] if true, record is created/update.
+    #   If false, record processing is skipped but message offset is still committed.
+    def process_message?(_payload)
+      true
+    end
   end
 end

data/lib/deimos/backends/db.rb

@@ -14,7 +14,7 @@ module Deimos
             message = Deimos::KafkaMessage.new(
               message: m.encoded_payload ? m.encoded_payload.to_s.b : nil,
               topic: m.topic,
-              partition_key: m.partition_key || m.key
+              partition_key: partition_key_for(m)
             )
             message.key = m.encoded_key.to_s.b unless producer_class.config[:no_keys]
             message
@@ -26,6 +26,15 @@ module Deimos
             by: records.size
           )
         end
+
+        # @param message [Deimos::Message]
+        # @return [String] the partition key to use for this message
+        def partition_key_for(message)
+          return message.partition_key if message.partition_key.present?
+          return message.key unless message.key.is_a?(Hash)
+
+          message.key.to_yaml
+        end
       end
     end
   end

data/lib/deimos/config/configurable.rb

@@ -15,6 +15,11 @@ module Deimos
   #       enabled true
   #       ca_cert_file 'my_file'
   #     end
+  #     config.kafka do
+  #       ssl do
+  #         enabled true
+  #       end
+  #     end
   #   end
   # - Allows for arrays of configurations:
   #   Deimos.configure do |config|

data/lib/deimos/config/configuration.rb

@@ -67,7 +67,7 @@ module Deimos
       topic(kafka_config.topic) if kafka_config.topic.present? && klass.respond_to?(:topic)
       schema(kafka_config.schema) if kafka_config.schema.present?
       namespace(kafka_config.namespace) if kafka_config.namespace.present?
-      key_config(kafka_config.key_config) if kafka_config.key_config.present?
+      key_config(**kafka_config.key_config) if kafka_config.key_config.present?
     end
   end
 

data/lib/deimos/kafka_source.rb

@@ -88,8 +88,9 @@ module Deimos
                                                   array_of_attributes,
                                                   options={})
         results = super
-        return unless self.kafka_config[:import]
-        return if array_of_attributes.empty?
+        if !self.kafka_config[:import] || array_of_attributes.empty?
+          return results
+        end
 
         # This will contain an array of hashes, where each hash is the actual
         # attribute hash that created the object.

data/lib/deimos/kafka_topic_info.rb

@@ -85,11 +85,8 @@ module Deimos
                       locked_at: Time.zone.now,
                       error: true,
                       retries: record.retries + 1 }
-        if ActiveRecord::VERSION::MAJOR >= 4
-          record.update!(attr_hash)
-        else
-          record.update_attributes!(attr_hash)
-        end
+        record.attributes = attr_hash
+        record.save!
       end
 
       # Update the locked_at timestamp to indicate that the producer is still

data/lib/deimos/schema_backends/avro_schema_coercer.rb

@@ -44,9 +44,11 @@ module Deimos
 
       case field_type
       when :int, :long
-        if val.is_a?(Integer) ||
-           _is_integer_string?(val) ||
-           int_classes.any? { |klass| val.is_a?(klass) }
+        if %w(timestamp-millis timestamp-micros).include?(type.logical_type)
+          val
+        elsif val.is_a?(Integer) ||
+              _is_integer_string?(val) ||
+              int_classes.any? { |klass| val.is_a?(klass) }
           val.to_i
         else
           val # this will fail

data/lib/deimos/utils/db_poller.rb

@@ -142,7 +142,8 @@ module Deimos
         last_id = record.public_send(id_method)
         last_updated_at = last_updated(record)
         @producer.send_events(batch)
-        @info.update_attributes!(last_sent: last_updated_at, last_sent_id: last_id)
+        @info.attributes = { last_sent: last_updated_at, last_sent_id: last_id }
+        @info.save!
       end
     end
   end

data/lib/deimos/utils/db_producer.rb

@@ -190,11 +190,14 @@ module Deimos
         end
       end
 
+      # Produce messages in batches, reducing the size 1/10 if the batch is too
+      # large. Does not retry batches of messages that have already been sent.
       # @param batch [Array<Hash>]
       def produce_messages(batch)
         batch_size = batch.size
+        current_index = 0
         begin
-          batch.in_groups_of(batch_size, false).each do |group|
+          batch[current_index..-1].in_groups_of(batch_size, false).each do |group|
             @logger.debug("Publishing #{group.size} messages to #{@current_topic}")
             producer.publish_list(group)
             Deimos.config.metrics&.increment(
@@ -202,6 +205,7 @@ module Deimos
               tags: %W(status:success topic:#{@current_topic}),
               by: group.size
             )
+            current_index += group.size
             @logger.info("Sent #{group.size} messages to #{@current_topic}")
           end
         rescue Kafka::BufferOverflow, Kafka::MessageSizeTooLarge,

data/lib/deimos/utils/inline_consumer.rb

@@ -6,6 +6,7 @@ module Deimos
   module Utils
     # Listener that can seek to get the last X messages in a topic.
     class SeekListener < Phobos::Listener
+      MAX_SEEK_RETRIES = 3
       attr_accessor :num_messages
 
       # :nodoc:
@@ -13,8 +14,10 @@ module Deimos
         @num_messages ||= 10
         @consumer = create_kafka_consumer
         @consumer.subscribe(topic, @subscribe_opts)
+        attempt = 0
 
         begin
+          attempt += 1
           last_offset = @kafka_client.last_offset_for(topic, 0)
           offset = last_offset - num_messages
           if offset.positive?
@@ -22,7 +25,11 @@ module Deimos
             @consumer.seek(topic, 0, offset)
           end
         rescue StandardError => e
-          "Could not seek to offset: #{e.message}"
+          if attempt < MAX_SEEK_RETRIES
+            sleep(1.seconds * attempt)
+            retry
+          end
+          log_error("Could not seek to offset: #{e.message} after #{MAX_SEEK_RETRIES} retries", listener_metadata)
         end
 
         instrument('listener.start_handler', listener_metadata) do
@@ -50,7 +57,6 @@ module Deimos
 
       # :nodoc:
       def consume(payload, metadata)
-        puts "Got #{payload}"
         self.class.total_messages << {
           key: metadata[:key],
           payload: payload

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.8.2'
+  VERSION = '1.8.7'
 end

data/spec/active_record_consumer_spec.rb

@@ -137,5 +137,18 @@ module ActiveRecordConsumerTest
       expect(Widget.find_by_test_id('id1').some_int).to eq(3)
       expect(Widget.find_by_test_id('id2').some_int).to eq(4)
     end
+
+    it 'should not create record of process_message returns false' do
+      MyConsumer.any_instance.stub(:process_message?).and_return(false)
+      expect(Widget.count).to eq(0)
+      test_consume_message(MyConsumer, {
+                             test_id: 'abc',
+                             some_int: 3,
+                             updated_at: 1.day.ago.to_i,
+                             some_datetime_int: Time.zone.now.to_i,
+                             timestamp: 2.minutes.ago.to_s
+                           }, { call_original: true, key: 5 })
+      expect(Widget.count).to eq(0)
+    end
   end
 end

data/spec/backends/db_spec.rb

@@ -43,6 +43,12 @@ each_db_config(Deimos::Backends::Db) do
     described_class.publish(producer_class: MyNoKeyProducer,
                             messages: [messages.first])
     expect(Deimos::KafkaMessage.count).to eq(4)
+  end
 
+  it 'should add messages with Hash keys with JSON encoding' do
+    described_class.publish(producer_class: MyProducer,
+                            messages: [build_message({ foo: 0 }, 'my-topic', { 'test_id' => 0 })])
+    expect(Deimos::KafkaMessage.count).to eq(1)
+    expect(Deimos::KafkaMessage.last.partition_key).to eq(%(---\ntest_id: 0\n))
   end
 end

data/spec/generators/active_record_generator_spec.rb

@@ -16,7 +16,7 @@ RSpec.describe Deimos::Generators::ActiveRecordGenerator do
     files = Dir['db/migrate/*.rb']
     expect(files.length).to eq(1)
     results = <<~MIGRATION
-      class CreateGeneratedTable < ActiveRecord::Migration[6.0]
+      class CreateGeneratedTable < ActiveRecord::Migration[6.1]
         def up
           if table_exists?(:generated_table)
             warn "generated_table already exists, exiting"

data/spec/kafka_source_spec.rb

@@ -225,5 +225,88 @@ module KafkaSourceSpec
         expect(Deimos::KafkaMessage.count).to eq(0)
       end
     end
+
+    context 'with import hooks disabled' do
+      before(:each) do
+        # Dummy class we can include the mixin in. Has a backing table created
+        # earlier and has the import hook disabled
+        class WidgetNoImportHook < ActiveRecord::Base
+          include Deimos::KafkaSource
+          self.table_name = 'widgets'
+
+          # :nodoc:
+          def self.kafka_config
+            {
+              update: true,
+              delete: true,
+              import: false,
+              create: true
+            }
+          end
+
+          # :nodoc:
+          def self.kafka_producers
+            [WidgetProducer]
+          end
+        end
+        WidgetNoImportHook.reset_column_information
+      end
+
+      it 'should not fail when bulk-importing with existing records' do
+        widget1 = WidgetNoImportHook.create(widget_id: 1, name: 'Widget 1')
+        widget2 = WidgetNoImportHook.create(widget_id: 2, name: 'Widget 2')
+        widget1.name = 'New Widget No Import Hook 1'
+        widget2.name = 'New Widget No Import Hook 2'
+
+        expect {
+          WidgetNoImportHook.import([widget1, widget2], on_duplicate_key_update: %i(widget_id name))
+        }.not_to raise_error
+
+        expect('my-topic').not_to have_sent({
+                                              widget_id: 1,
+                                              name: 'New Widget No Import Hook 1',
+                                              id: widget1.id,
+                                              created_at: anything,
+                                              updated_at: anything
+                                            }, widget1.id)
+        expect('my-topic').not_to have_sent({
+                                              widget_id: 2,
+                                              name: 'New Widget No Import Hook 2',
+                                              id: widget2.id,
+                                              created_at: anything,
+                                              updated_at: anything
+                                            }, widget2.id)
+      end
+
+      it 'should not fail when mixing existing and new records' do
+        widget1 = WidgetNoImportHook.create(widget_id: 1, name: 'Widget 1')
+        expect('my-topic').to have_sent({
+                                          widget_id: 1,
+                                          name: 'Widget 1',
+                                          id: widget1.id,
+                                          created_at: anything,
+                                          updated_at: anything
+                                        }, widget1.id)
+
+        widget2 = WidgetNoImportHook.new(widget_id: 2, name: 'Widget 2')
+        widget1.name = 'New Widget 1'
+        WidgetNoImportHook.import([widget1, widget2], on_duplicate_key_update: %i(widget_id))
+        widgets = WidgetNoImportHook.all
+        expect('my-topic').not_to have_sent({
+                                              widget_id: 1,
+                                              name: 'New Widget 1',
+                                              id: widgets[0].id,
+                                              created_at: anything,
+                                              updated_at: anything
+                                            }, widgets[0].id)
+        expect('my-topic').not_to have_sent({
+                                              widget_id: 2,
+                                              name: 'Widget 2',
+                                              id: widgets[1].id,
+                                              created_at: anything,
+                                              updated_at: anything
+                                            }, widgets[1].id)
+      end
+    end
   end
 end

data/spec/kafka_topic_info_spec.rb

@@ -51,13 +51,13 @@ each_db_config(Deimos::KafkaTopicInfo) do
       expect(record.locked_at).to eq(nil)
       expect(record.error).to eq(false)
       expect(record.retries).to eq(0)
-      expect(record.last_processed_at.to_s).to eq(Time.zone.now.to_s)
+      expect(record.last_processed_at.in_time_zone.to_s).to eq(Time.zone.now.to_s)
       record = Deimos::KafkaTopicInfo.last
       expect(record.locked_by).not_to eq(nil)
       expect(record.locked_at).not_to eq(nil)
       expect(record.error).not_to eq(false)
       expect(record.retries).not_to eq(0)
-      expect(record.last_processed_at.to_s).to eq(20.seconds.ago.to_s)
+      expect(record.last_processed_at.in_time_zone.to_s).to eq(20.seconds.ago.to_s)
     end
   end
 
@@ -70,11 +70,11 @@ each_db_config(Deimos::KafkaTopicInfo) do
                                           locked_by: 'me', locked_at: 1.minute.ago)
 
       expect(Deimos::KafkaTopicInfo.count).to eq(3)
-      Deimos::KafkaTopicInfo.all.each { |t| expect(t.last_processed_at.to_s).to eq(old_time) }
+      Deimos::KafkaTopicInfo.all.each { |t| expect(t.last_processed_at.in_time_zone.to_s).to eq(old_time) }
       Deimos::KafkaTopicInfo.ping_empty_topics(%w(topic1))
-      expect(t1.reload.last_processed_at.to_s).to eq(old_time) # was passed as an exception
-      expect(t2.reload.last_processed_at.to_s).to eq(Time.zone.now.to_s)
-      expect(t3.reload.last_processed_at.to_s).to eq(old_time) # is locked
+      expect(t1.reload.last_processed_at.in_time_zone.to_s).to eq(old_time) # was passed as an exception
+      expect(t2.reload.last_processed_at.in_time_zone.to_s).to eq(Time.zone.now.to_s)
+      expect(t3.reload.last_processed_at.in_time_zone.to_s).to eq(old_time) # is locked
     end
   end
 

data/spec/schema_backends/avro_base_shared.rb

@@ -42,6 +42,20 @@ RSpec.shared_examples_for('an Avro backend') do
         {
           'name' => 'union-int-field',
           'type' => %w(null int)
+        },
+        {
+          'name' => 'timestamp-millis-field',
+          'type' => {
+            'type' => 'long',
+            'logicalType' => 'timestamp-millis'
+          }
+        },
+        {
+          'name' => 'timestamp-micros-field',
+          'type' => {
+            'type' => 'long',
+            'logicalType' => 'timestamp-micros'
+          }
         }
       ]
     }
@@ -95,7 +109,9 @@ RSpec.shared_examples_for('an Avro backend') do
         'string-field' => 'hi mom',
         'boolean-field' => true,
         'union-field' => nil,
-        'union-int-field' => nil
+        'union-int-field' => nil,
+        'timestamp-millis-field' => Time.utc(2020, 11, 12, 13, 14, 15, 909_090),
+        'timestamp-micros-field' => Time.utc(2020, 11, 12, 13, 14, 15, 909_090)
       }
     end
 
@@ -169,6 +185,15 @@ RSpec.shared_examples_for('an Avro backend') do
       expect(result['union-field']).to eq('itsme')
     end
 
+    it 'should not convert timestamp-millis' do
+      result = backend.coerce(payload)
+      expect(result['timestamp-millis-field']).to eq(Time.utc(2020, 11, 12, 13, 14, 15, 909_090))
+    end
+
+    it 'should not convert timestamp-micros' do
+      result = backend.coerce(payload)
+      expect(result['timestamp-micros-field']).to eq(Time.utc(2020, 11, 12, 13, 14, 15, 909_090))
+    end
   end
 
 end

data/spec/spec_helper.rb

@@ -87,7 +87,7 @@ module DbConfigs
       port: 3306,
       username: 'root',
       database: 'test',
-      host: ENV['MYSQL_HOST'] || 'localhost'
+      host: ENV['MYSQL_HOST'] || '127.0.0.1'
     },
     {
       adapter: 'sqlite3',

data/spec/utils/db_producer_spec.rb

@@ -96,7 +96,32 @@ each_db_config(Deimos::Utils::DbProducer) do
       expect(phobos_producer).to have_received(:publish_list).with(['A'] * 100).once
       expect(phobos_producer).to have_received(:publish_list).with(['A'] * 10).once
       expect(phobos_producer).to have_received(:publish_list).with(['A']).once
+    end
+
+    it 'should not resend batches of sent messages' do
+      allow(phobos_producer).to receive(:publish_list) do |group|
+        raise Kafka::BufferOverflow if group.any?('A') && group.size >= 1000
+        raise Kafka::BufferOverflow if group.any?('BIG') && group.size >= 10
+      end
+      allow(Deimos.config.metrics).to receive(:increment)
+      batch = ['A'] * 450 + ['BIG'] * 550
+      producer.produce_messages(batch)
+
+      expect(phobos_producer).to have_received(:publish_list).with(batch)
+      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 100).exactly(4).times
+      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 50 + ['BIG'] * 50)
+      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 10).exactly(5).times
+      expect(phobos_producer).to have_received(:publish_list).with(['BIG'] * 1).exactly(550).times
 
+      expect(Deimos.config.metrics).to have_received(:increment).with('publish',
+                                                                      tags: %w(status:success topic:),
+                                                                      by: 100).exactly(4).times
+      expect(Deimos.config.metrics).to have_received(:increment).with('publish',
+                                                                      tags: %w(status:success topic:),
+                                                                      by: 10).exactly(5).times
+      expect(Deimos.config.metrics).to have_received(:increment).with('publish',
+                                                                      tags: %w(status:success topic:),
+                                                                      by: 1).exactly(550).times
     end
 
     describe '#compact_messages' do
@@ -289,6 +314,8 @@ each_db_config(Deimos::Utils::DbProducer) do
           message: "mess#{i}",
           partition_key: "key#{i}"
         )
+      end
+      (5..8).each do |i|
         Deimos::KafkaMessage.create!(
           id: i,
           topic: 'my-topic2',

data/spec/utils/inline_consumer_spec.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+describe Deimos::Utils::SeekListener do
+
+  describe '#start_listener' do
+    let(:consumer) { instance_double(Kafka::Consumer) }
+    let(:handler) { class_double(Deimos::Utils::MessageBankHandler) }
+
+    before(:each) do
+      allow(handler).to receive(:start)
+      allow(consumer).to receive(:subscribe)
+      allow_any_instance_of(Phobos::Listener).to receive(:create_kafka_consumer).and_return(consumer)
+      allow_any_instance_of(Kafka::Client).to receive(:last_offset_for).and_return(100)
+      stub_const('Deimos::Utils::SeekListener::MAX_SEEK_RETRIES', 2)
+    end
+
+    it 'should seek offset' do
+      allow(consumer).to receive(:seek)
+      expect(consumer).to receive(:seek).once
+      seek_listener = described_class.new({ handler: handler, group_id: 999, topic: 'test_topic' })
+      seek_listener.start_listener
+    end
+
+    it 'should retry on errors when seeking offset' do
+      allow(consumer).to receive(:seek).and_raise(StandardError)
+      expect(consumer).to receive(:seek).twice
+      seek_listener = described_class.new({ handler: handler, group_id: 999, topic: 'test_topic' })
+      seek_listener.start_listener
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.8.2
+  version: 1.8.7
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-09-25 00:00:00.000000000 Z
+date: 2021-01-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf
@@ -28,16 +28,22 @@ dependencies:
   name: phobos
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.9'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.9'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: ruby-kafka
   requirement: !ruby/object:Gem::Requirement
@@ -280,30 +286,30 @@ dependencies:
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0.72'
+        version: 0.88.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0.72'
+        version: 0.88.0
 - !ruby/object:Gem::Dependency
   name: rubocop-rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '1.27'
+        version: 1.42.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '1.27'
+        version: 1.42.0
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -348,6 +354,7 @@ files:
 - docs/ARCHITECTURE.md
 - docs/CONFIGURATION.md
 - docs/DATABASE_BACKEND.md
+- docs/INTEGRATION_TESTS.md
 - docs/PULL_REQUEST_TEMPLATE.md
 - docs/UPGRADING.md
 - lib/deimos.rb
@@ -464,6 +471,7 @@ files:
 - spec/utils/db_poller_spec.rb
 - spec/utils/db_producer_spec.rb
 - spec/utils/deadlock_retry_spec.rb
+- spec/utils/inline_consumer_spec.rb
 - spec/utils/lag_reporter_spec.rb
 - spec/utils/platform_schema_validation_spec.rb
 - spec/utils/schema_controller_mixin_spec.rb
@@ -490,7 +498,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.3
+rubygems_version: 3.0.9
 signing_key:
 specification_version: 4
 summary: Kafka libraries for Ruby.
@@ -547,6 +555,7 @@ test_files:
 - spec/utils/db_poller_spec.rb
 - spec/utils/db_producer_spec.rb
 - spec/utils/deadlock_retry_spec.rb
+- spec/utils/inline_consumer_spec.rb
 - spec/utils/lag_reporter_spec.rb
 - spec/utils/platform_schema_validation_spec.rb
 - spec/utils/schema_controller_mixin_spec.rb