deimos-ruby 1.8.3 → 1.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2badd671866a8fbde743e03acbc2da623279ac08abd5ed983ae9a15a0bef415d
-  data.tar.gz: e4b3cd2e80b13fc00ff54ebc85b59ab2e1daad052ab170dafd9a75f57eb80ce3
+  metadata.gz: 754a19c6aef123b6505a28e4f2f6209a237c02a28b3d7c797b531ead05732c35
+  data.tar.gz: bef461f0a501cf3e838135c34c33b746f424ecbfacc99c277e5b16e3d5df74af
 SHA512:
-  metadata.gz: 98b41fa7354e624a4538ac5006a091e1a144a2caaa57e2ef77c214e350cd8dcbf2b126c01a3663365d28bbed19baa7525e3c52390cc60f6378537ad8fe6bee12
-  data.tar.gz: 208ba417a35c7042155383d5c0eb9d9e35be95c0459cf2e58cdc24168c8fdf30af16434fcc66c612d197aab38279d790907f146e19b6a36157cde008c5aa60a3
+  metadata.gz: e2496c1de094594c1d1a634f0cfbe14ac755000b748a4010f00705d47f0750411ee0b49e84b87b107419f8df788e5a3a77d09b6467a6d710c1f73538514e6025
+  data.tar.gz: 41e6b26586b82b649e5fd8a5b49d52efde0127838e6cda70e81664d9630a16e790e3d5971db4a31273d94030aab5630e5d485af082c08168deb36c659543a81b

data/CHANGELOG.md

@@ -7,6 +7,42 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+## 1.9.0 - 2021-01-28
+
+- ### Roadmap :car:
+
+- Bumped the version of ruby-kafka to latest
+
+- ### Fixes :wrench:
+
+- Prevents DB Poller from reconnecting to DB if there is an open transaction
+- Replaces `before` by `prepend_before` for more consistent test setups.
+- Adds validation in the `kafka_producers` method (fixes [#90](https://github.com/flipp-oss/deimos/issues/90))
+
+## 1.8.7 - 2021-01-14
+
+- ### Roadmap :car:
+- Update Phobos version to allow version 1.9 or 2.x.
+
+## 1.8.6 - 2021-01-14
+
+- ### Fixes :wrench:
+- Fix for configuration bug with Ruby 3.0 (** instead of passing hash)
+
+## 1.8.5 - 2021-01-13
+
+- ### Fixes :wrench:
+- Fixes for Rails 6.1 (remove usage of `update_attributes!`)
+
+## 1.8.4 - 2020-12-02
+
+### Features :star:
+- Add overridable "process_message?" method to ActiveRecordConsumer to allow for skipping of saving/updating records
+
+### Fixes :wrench:
+
+- Do not apply type coercion to `timestamp-millis` and `timestamp-micros` logical types (fixes [#97](https://github.com/flipp-oss/deimos/issues/97))
+
 ## 1.8.3 - 2020-11-18
 
 ### Fixes :wrench:
@@ -15,6 +51,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - KafkaSource crashing on bulk-imports if import hooks are disabled
   (fixes [#73](https://github.com/flipp-oss/deimos/issues/73))
 - #96 Use string-safe encoding for partition keys
+- Retry on offset seek failures in inline consumer
+  (fixes [#5](Inline consumer should use retries when seeking))
 
 ## 1.8.2 - 2020-09-25
 

data/Gemfile.lock

@@ -1,73 +1,77 @@
 PATH
   remote: .
   specs:
-    deimos-ruby (1.8.2)
+    deimos-ruby (1.8.7)
       avro_turf (~> 0.11)
-      phobos (~> 1.9)
-      ruby-kafka (~> 0.7)
+      phobos (>= 1.9, < 3.0)
+      ruby-kafka (< 2)
       sigurd (= 0.0.1)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (6.0.3.2)
-      actionpack (= 6.0.3.2)
+    actioncable (6.1.1)
+      actionpack (= 6.1.1)
+      activesupport (= 6.1.1)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailbox (6.0.3.2)
-      actionpack (= 6.0.3.2)
-      activejob (= 6.0.3.2)
-      activerecord (= 6.0.3.2)
-      activestorage (= 6.0.3.2)
-      activesupport (= 6.0.3.2)
+    actionmailbox (6.1.1)
+      actionpack (= 6.1.1)
+      activejob (= 6.1.1)
+      activerecord (= 6.1.1)
+      activestorage (= 6.1.1)
+      activesupport (= 6.1.1)
       mail (>= 2.7.1)
-    actionmailer (6.0.3.2)
-      actionpack (= 6.0.3.2)
-      actionview (= 6.0.3.2)
-      activejob (= 6.0.3.2)
+    actionmailer (6.1.1)
+      actionpack (= 6.1.1)
+      actionview (= 6.1.1)
+      activejob (= 6.1.1)
+      activesupport (= 6.1.1)
       mail (~> 2.5, >= 2.5.4)
       rails-dom-testing (~> 2.0)
-    actionpack (6.0.3.2)
-      actionview (= 6.0.3.2)
-      activesupport (= 6.0.3.2)
-      rack (~> 2.0, >= 2.0.8)
+    actionpack (6.1.1)
+      actionview (= 6.1.1)
+      activesupport (= 6.1.1)
+      rack (~> 2.0, >= 2.0.9)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actiontext (6.0.3.2)
-      actionpack (= 6.0.3.2)
-      activerecord (= 6.0.3.2)
-      activestorage (= 6.0.3.2)
-      activesupport (= 6.0.3.2)
+    actiontext (6.1.1)
+      actionpack (= 6.1.1)
+      activerecord (= 6.1.1)
+      activestorage (= 6.1.1)
+      activesupport (= 6.1.1)
       nokogiri (>= 1.8.5)
-    actionview (6.0.3.2)
-      activesupport (= 6.0.3.2)
+    actionview (6.1.1)
+      activesupport (= 6.1.1)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.1, >= 1.2.0)
-    activejob (6.0.3.2)
-      activesupport (= 6.0.3.2)
+    activejob (6.1.1)
+      activesupport (= 6.1.1)
       globalid (>= 0.3.6)
-    activemodel (6.0.3.2)
-      activesupport (= 6.0.3.2)
-    activerecord (6.0.3.2)
-      activemodel (= 6.0.3.2)
-      activesupport (= 6.0.3.2)
-    activerecord-import (1.0.5)
+    activemodel (6.1.1)
+      activesupport (= 6.1.1)
+    activerecord (6.1.1)
+      activemodel (= 6.1.1)
+      activesupport (= 6.1.1)
+    activerecord-import (1.0.7)
       activerecord (>= 3.2)
-    activestorage (6.0.3.2)
-      actionpack (= 6.0.3.2)
-      activejob (= 6.0.3.2)
-      activerecord (= 6.0.3.2)
+    activestorage (6.1.1)
+      actionpack (= 6.1.1)
+      activejob (= 6.1.1)
+      activerecord (= 6.1.1)
+      activesupport (= 6.1.1)
       marcel (~> 0.3.1)
-    activesupport (6.0.3.2)
+      mimemagic (~> 0.3.2)
+    activesupport (6.1.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (>= 0.7, < 2)
-      minitest (~> 5.1)
-      tzinfo (~> 1.1)
-      zeitwerk (~> 2.2, >= 2.2.2)
-    ast (2.4.1)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
+    ast (2.4.2)
     avro (1.9.2)
       multi_json
     avro_turf (0.11.0)
@@ -75,21 +79,21 @@ GEM
       excon (~> 0.45)
     builder (3.2.4)
     coderay (1.1.3)
-    concurrent-ruby (1.1.6)
-    concurrent-ruby-ext (1.1.6)
-      concurrent-ruby (= 1.1.6)
+    concurrent-ruby (1.1.8)
+    concurrent-ruby-ext (1.1.8)
+      concurrent-ruby (= 1.1.8)
     crass (1.0.6)
     database_cleaner (1.8.5)
-    ddtrace (0.37.0)
+    ddtrace (0.45.0)
       msgpack
     diff-lcs (1.4.4)
-    digest-crc (0.6.1)
-      rake (~> 13.0)
-    dogstatsd-ruby (4.8.1)
-    erubi (1.9.0)
-    excon (0.76.0)
+    digest-crc (0.6.3)
+      rake (>= 12.0.0, < 14.0.0)
+    dogstatsd-ruby (4.8.2)
+    erubi (1.10.0)
+    excon (0.78.1)
     exponential-backoff (0.0.4)
-    ffi (1.13.1)
+    ffi (1.14.2)
     formatador (0.2.5)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
@@ -107,22 +111,22 @@ GEM
       guard (~> 2.1)
       guard-compat (~> 1.1)
       rspec (>= 2.99.0, < 4.0)
-    guard-rubocop (1.3.0)
+    guard-rubocop (1.4.0)
       guard (~> 2.0)
-      rubocop (~> 0.20)
-    i18n (1.8.4)
+      rubocop (< 2.0)
+    i18n (1.8.7)
       concurrent-ruby (~> 1.0)
-    listen (3.2.1)
+    listen (3.4.1)
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
     little-plugger (1.1.4)
     logging (2.3.0)
       little-plugger (~> 1.1)
       multi_json (~> 1.14)
-    loofah (2.6.0)
+    loofah (2.9.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
-    lumberjack (1.2.6)
+    lumberjack (1.2.8)
     mail (2.7.1)
       mini_mime (>= 0.1.1)
     marcel (0.3.3)
@@ -130,23 +134,24 @@ GEM
     method_source (1.0.0)
     mimemagic (0.3.5)
     mini_mime (1.0.2)
-    mini_portile2 (2.4.0)
-    minitest (5.14.1)
-    msgpack (1.3.3)
+    mini_portile2 (2.5.0)
+    minitest (5.14.3)
+    msgpack (1.4.1)
     multi_json (1.15.0)
     mysql2 (0.5.3)
     nenv (0.3.0)
-    nio4r (2.5.2)
-    nokogiri (1.10.10)
-      mini_portile2 (~> 2.4.0)
+    nio4r (2.5.4)
+    nokogiri (1.11.1)
+      mini_portile2 (~> 2.5.0)
+      racc (~> 1.4)
     notiffany (0.1.3)
       nenv (~> 0.1)
       shellany (~> 0.0)
-    parallel (1.19.2)
-    parser (2.7.1.4)
+    parallel (1.20.1)
+    parser (3.0.0.0)
       ast (~> 2.4.1)
     pg (1.2.3)
-    phobos (1.9.0)
+    phobos (2.0.2)
       activesupport (>= 3.0.0)
       concurrent-ruby (>= 1.0.2)
       concurrent-ruby-ext (>= 1.0.2)
@@ -157,63 +162,64 @@ GEM
     pry (0.13.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
+    racc (1.5.2)
     rack (2.2.3)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
-    rails (6.0.3.2)
-      actioncable (= 6.0.3.2)
-      actionmailbox (= 6.0.3.2)
-      actionmailer (= 6.0.3.2)
-      actionpack (= 6.0.3.2)
-      actiontext (= 6.0.3.2)
-      actionview (= 6.0.3.2)
-      activejob (= 6.0.3.2)
-      activemodel (= 6.0.3.2)
-      activerecord (= 6.0.3.2)
-      activestorage (= 6.0.3.2)
-      activesupport (= 6.0.3.2)
-      bundler (>= 1.3.0)
-      railties (= 6.0.3.2)
+    rails (6.1.1)
+      actioncable (= 6.1.1)
+      actionmailbox (= 6.1.1)
+      actionmailer (= 6.1.1)
+      actionpack (= 6.1.1)
+      actiontext (= 6.1.1)
+      actionview (= 6.1.1)
+      activejob (= 6.1.1)
+      activemodel (= 6.1.1)
+      activerecord (= 6.1.1)
+      activestorage (= 6.1.1)
+      activesupport (= 6.1.1)
+      bundler (>= 1.15.0)
+      railties (= 6.1.1)
       sprockets-rails (>= 2.0.0)
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
     rails-html-sanitizer (1.3.0)
       loofah (~> 2.3)
-    railties (6.0.3.2)
-      actionpack (= 6.0.3.2)
-      activesupport (= 6.0.3.2)
+    railties (6.1.1)
+      actionpack (= 6.1.1)
+      activesupport (= 6.1.1)
       method_source
       rake (>= 0.8.7)
-      thor (>= 0.20.3, < 2.0)
+      thor (~> 1.0)
     rainbow (3.0.0)
-    rake (13.0.1)
+    rake (13.0.3)
     rb-fsevent (0.10.4)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
-    regexp_parser (1.7.1)
+    regexp_parser (2.0.3)
     rexml (3.2.4)
-    rspec (3.9.0)
-      rspec-core (~> 3.9.0)
-      rspec-expectations (~> 3.9.0)
-      rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.2)
-      rspec-support (~> 3.9.3)
-    rspec-expectations (3.9.2)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.1)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-rails (4.0.1)
+      rspec-support (~> 3.10.0)
+    rspec-rails (4.0.2)
       actionpack (>= 4.2)
       activesupport (>= 4.2)
       railties (>= 4.2)
-      rspec-core (~> 3.9)
-      rspec-expectations (~> 3.9)
-      rspec-mocks (~> 3.9)
-      rspec-support (~> 3.9)
-    rspec-support (3.9.3)
+      rspec-core (~> 3.10)
+      rspec-expectations (~> 3.10)
+      rspec-mocks (~> 3.10)
+      rspec-support (~> 3.10)
+    rspec-support (3.10.1)
     rspec_junit_formatter (0.4.1)
       rspec-core (>= 2, < 4, != 2.12.0)
     rubocop (0.88.0)
@@ -225,13 +231,13 @@ GEM
       rubocop-ast (>= 0.1.0, < 1.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 2.0)
-    rubocop-ast (0.2.0)
-      parser (>= 2.7.0.1)
+    rubocop-ast (0.8.0)
+      parser (>= 2.7.1.5)
     rubocop-rspec (1.42.0)
       rubocop (>= 0.87.0)
-    ruby-kafka (0.7.10)
+    ruby-kafka (1.3.0)
       digest-crc
-    ruby-progressbar (1.10.1)
+    ruby-progressbar (1.11.0)
     shellany (0.0.1)
     sigurd (0.0.1)
       concurrent-ruby (~> 1)
@@ -239,20 +245,19 @@ GEM
     sprockets (4.0.2)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
-    sprockets-rails (3.2.1)
+    sprockets-rails (3.2.2)
       actionpack (>= 4.0)
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)
     sqlite3 (1.4.2)
-    thor (1.0.1)
-    thread_safe (0.3.6)
-    tzinfo (1.2.7)
-      thread_safe (~> 0.1)
+    thor (1.1.0)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
     unicode-display_width (1.7.0)
     websocket-driver (0.7.3)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
-    zeitwerk (2.4.0)
+    zeitwerk (2.4.2)
 
 PLATFORMS
   ruby
@@ -274,9 +279,9 @@ DEPENDENCIES
   rspec (~> 3)
   rspec-rails (~> 4)
   rspec_junit_formatter (~> 0.3)
-  rubocop (~> 0.72)
-  rubocop-rspec (~> 1.27)
+  rubocop (= 0.88.0)
+  rubocop-rspec (= 1.42.0)
   sqlite3 (~> 1.3)
 
 BUNDLED WITH
-   2.1.4
+   2.2.5

data/README.md

@@ -626,6 +626,15 @@ class MyConsumer < Deimos::ActiveRecordConsumer
   def record_key(payload)
     super
   end
+
+  # Optional override, returns true by default.
+  # When this method returns true, a record corresponding to the message
+  # is created/updated.
+  # When this method returns false, message processing is skipped and a
+  # corresponding record will NOT be created/updated.
+  def process_message?(payload)
+    super
+  end
 end
 ```
 

data/deimos-ruby.gemspec

@@ -19,8 +19,8 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_runtime_dependency('avro_turf', '~> 0.11')
-  spec.add_runtime_dependency('phobos', '~> 1.9')
-  spec.add_runtime_dependency('ruby-kafka', '~> 0.7')
+  spec.add_runtime_dependency('phobos', '>= 1.9', '< 3.0')
+  spec.add_runtime_dependency('ruby-kafka', '< 2')
   spec.add_runtime_dependency('sigurd', '0.0.1')
 
   spec.add_development_dependency('activerecord-import')
@@ -38,7 +38,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency('rspec', '~> 3')
   spec.add_development_dependency('rspec_junit_formatter', '~>0.3')
   spec.add_development_dependency('rspec-rails', '~> 4')
-  spec.add_development_dependency('rubocop', '~> 0.72')
-  spec.add_development_dependency('rubocop-rspec', '~> 1.27')
+  spec.add_development_dependency('rubocop', '0.88.0')
+  spec.add_development_dependency('rubocop-rspec', '1.42.0')
   spec.add_development_dependency('sqlite3', '~> 1.3')
 end

data/docs/PULL_REQUEST_TEMPLATE.md

@@ -28,6 +28,7 @@ Please describe the tests that you ran to verify your changes. Provide instructi
 - [ ] I have performed a self-review of my own code
 - [ ] I have commented my code, particularly in hard-to-understand areas
 - [ ] I have made corresponding changes to the documentation
+- [ ] I have added a line in the CHANGELOG describing this change, under the UNRELEASED heading
 - [ ] My changes generate no new warnings
 - [ ] I have added tests that prove my fix is effective or that my feature works
 - [ ] New and existing unit tests pass locally with my changes

data/lib/deimos/active_record_consume/message_consumption.rb

@@ -26,6 +26,15 @@ module Deimos
 
       # :nodoc:
       def consume(payload, metadata)
+        unless self.process_message?(payload)
+          Deimos.config.logger.debug(
+            message: 'Skipping processing of message',
+            payload: payload,
+            metadata: metadata
+          )
+          return
+        end
+
         key = metadata.with_indifferent_access[:key]
         klass = self.class.config[:record_class]
         record = fetch_record(klass, (payload || {}).with_indifferent_access, key)

data/lib/deimos/active_record_consumer.rb

@@ -55,5 +55,13 @@ module Deimos
     def record_attributes(payload, _key=nil)
       @converter.convert(payload)
     end
+
+    # Override this message to conditionally save records
+    # @param payload [Hash] The kafka message as a hash
+    # @return [Boolean] if true, record is created/update.
+    #   If false, record processing is skipped but message offset is still committed.
+    def process_message?(_payload)
+      true
+    end
   end
 end

data/lib/deimos/config/configuration.rb

@@ -67,7 +67,7 @@ module Deimos
       topic(kafka_config.topic) if kafka_config.topic.present? && klass.respond_to?(:topic)
       schema(kafka_config.schema) if kafka_config.schema.present?
       namespace(kafka_config.namespace) if kafka_config.namespace.present?
-      key_config(kafka_config.key_config) if kafka_config.key_config.present?
+      key_config(**kafka_config.key_config) if kafka_config.key_config.present?
     end
   end
 

data/lib/deimos/kafka_source.rb

@@ -6,6 +6,9 @@ module Deimos
   module KafkaSource
     extend ActiveSupport::Concern
 
+    DEPRECATION_WARNING = 'The kafka_producer interface will be deprecated ' \
+        'in future releases. Please use kafka_producers instead.'
+
     included do
       after_create(:send_kafka_event_on_create)
       after_update(:send_kafka_event_on_update)
@@ -64,19 +67,12 @@ module Deimos
 
       # @return [Array<Deimos::ActiveRecordProducer>] the producers to run.
       def kafka_producers
-        raise NotImplementedError if self.method(:kafka_producer).
-          owner == Deimos::KafkaSource
-
-        [self.kafka_producer]
-      end
-
-      # Deprecated - use #kafka_producers instead.
-      # @return [Deimos::ActiveRecordProducer] the producer to use.
-      def kafka_producer
-        raise NotImplementedError if self.method(:kafka_producers).
-          owner == Deimos::KafkaSource
+        if self.respond_to?(:kafka_producer)
+          Deimos.config.logger.warn(message: DEPRECATION_WARNING)
+          return [self.kafka_producer]
+        end
 
-        self.kafka_producers.first
+        raise NotImplementedError
       end
 
       # This is an internal method, part of the activerecord_import gem. It's

data/lib/deimos/kafka_topic_info.rb

@@ -85,11 +85,8 @@ module Deimos
                       locked_at: Time.zone.now,
                       error: true,
                       retries: record.retries + 1 }
-        if ActiveRecord::VERSION::MAJOR >= 4
-          record.update!(attr_hash)
-        else
-          record.update_attributes!(attr_hash)
-        end
+        record.attributes = attr_hash
+        record.save!
       end
 
       # Update the locked_at timestamp to indicate that the producer is still

data/lib/deimos/schema_backends/avro_schema_coercer.rb

@@ -44,9 +44,11 @@ module Deimos
 
       case field_type
       when :int, :long
-        if val.is_a?(Integer) ||
-           _is_integer_string?(val) ||
-           int_classes.any? { |klass| val.is_a?(klass) }
+        if %w(timestamp-millis timestamp-micros).include?(type.logical_type)
+          val
+        elsif val.is_a?(Integer) ||
+              _is_integer_string?(val) ||
+              int_classes.any? { |klass| val.is_a?(klass) }
           val.to_i
         else
           val # this will fail

data/lib/deimos/test_helpers.rb

@@ -34,7 +34,7 @@ module Deimos
           end
         end
 
-        config.before(:each) do
+        config.prepend_before(:each) do
           client = double('client').as_null_object
           allow(client).to receive(:time) do |*_args, &block|
             block.call

data/lib/deimos/utils/db_poller.rb

@@ -68,7 +68,7 @@ module Deimos
 
       # Grab the PollInfo or create if it doesn't exist.
       def retrieve_poll_info
-        ActiveRecord::Base.connection.reconnect!
+        ActiveRecord::Base.connection.reconnect! unless ActiveRecord::Base.connection.open_transactions.positive?
         new_time = @config.start_from_beginning ? Time.new(0) : Time.zone.now
         @info = Deimos::PollInfo.find_by_producer(@config.producer_class) ||
                 Deimos::PollInfo.create!(producer: @config.producer_class,
@@ -142,7 +142,8 @@ module Deimos
         last_id = record.public_send(id_method)
         last_updated_at = last_updated(record)
         @producer.send_events(batch)
-        @info.update_attributes!(last_sent: last_updated_at, last_sent_id: last_id)
+        @info.attributes = { last_sent: last_updated_at, last_sent_id: last_id }
+        @info.save!
       end
     end
   end

data/lib/deimos/utils/inline_consumer.rb

@@ -6,6 +6,7 @@ module Deimos
   module Utils
     # Listener that can seek to get the last X messages in a topic.
     class SeekListener < Phobos::Listener
+      MAX_SEEK_RETRIES = 3
       attr_accessor :num_messages
 
       # :nodoc:
@@ -13,8 +14,10 @@ module Deimos
         @num_messages ||= 10
         @consumer = create_kafka_consumer
         @consumer.subscribe(topic, @subscribe_opts)
+        attempt = 0
 
         begin
+          attempt += 1
           last_offset = @kafka_client.last_offset_for(topic, 0)
           offset = last_offset - num_messages
           if offset.positive?
@@ -22,7 +25,11 @@ module Deimos
             @consumer.seek(topic, 0, offset)
           end
         rescue StandardError => e
-          "Could not seek to offset: #{e.message}"
+          if attempt < MAX_SEEK_RETRIES
+            sleep(1.seconds * attempt)
+            retry
+          end
+          log_error("Could not seek to offset: #{e.message} after #{MAX_SEEK_RETRIES} retries", listener_metadata)
         end
 
         instrument('listener.start_handler', listener_metadata) do
@@ -50,7 +57,6 @@ module Deimos
 
       # :nodoc:
       def consume(payload, metadata)
-        puts "Got #{payload}"
         self.class.total_messages << {
           key: metadata[:key],
           payload: payload

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.8.3'
+  VERSION = '1.9.0'
 end

data/spec/active_record_consumer_spec.rb

@@ -137,5 +137,18 @@ module ActiveRecordConsumerTest
       expect(Widget.find_by_test_id('id1').some_int).to eq(3)
       expect(Widget.find_by_test_id('id2').some_int).to eq(4)
     end
+
+    it 'should not create record of process_message returns false' do
+      MyConsumer.any_instance.stub(:process_message?).and_return(false)
+      expect(Widget.count).to eq(0)
+      test_consume_message(MyConsumer, {
+                             test_id: 'abc',
+                             some_int: 3,
+                             updated_at: 1.day.ago.to_i,
+                             some_datetime_int: Time.zone.now.to_i,
+                             timestamp: 2.minutes.ago.to_s
+                           }, { call_original: true, key: 5 })
+      expect(Widget.count).to eq(0)
+    end
   end
 end

data/spec/generators/active_record_generator_spec.rb

@@ -16,7 +16,7 @@ RSpec.describe Deimos::Generators::ActiveRecordGenerator do
     files = Dir['db/migrate/*.rb']
     expect(files.length).to eq(1)
     results = <<~MIGRATION
-      class CreateGeneratedTable < ActiveRecord::Migration[6.0]
+      class CreateGeneratedTable < ActiveRecord::Migration[6.1]
         def up
           if table_exists?(:generated_table)
             warn "generated_table already exists, exiting"

data/spec/kafka_source_spec.rb

@@ -308,5 +308,74 @@ module KafkaSourceSpec
                                             }, widgets[1].id)
       end
     end
+
+    context 'with AR models that implement the kafka_producer interface' do
+      before(:each) do
+        # Dummy class we can include the mixin in. Has a backing table created
+        # earlier and has the import hook disabled
+        deprecated_class = Class.new(ActiveRecord::Base) do
+          include Deimos::KafkaSource
+          self.table_name = 'widgets'
+
+          # :nodoc:
+          def self.kafka_config
+            {
+              update: true,
+              delete: true,
+              import: false,
+              create: true
+            }
+          end
+
+          # :nodoc:
+          def self.kafka_producer
+            WidgetProducer
+          end
+        end
+        stub_const('WidgetDeprecated', deprecated_class)
+        WidgetDeprecated.reset_column_information
+      end
+
+      it 'logs a warning and sends the message as usual' do
+        expect(Deimos.config.logger).to receive(:warn).with({ message: WidgetDeprecated::DEPRECATION_WARNING })
+        widget = WidgetDeprecated.create(widget_id: 1, name: 'Widget 1')
+        expect('my-topic').to have_sent({
+                                          widget_id: 1,
+                                          name: 'Widget 1',
+                                          id: widget.id,
+                                          created_at: anything,
+                                          updated_at: anything
+                                        }, widget.id)
+      end
+    end
+
+    context 'with AR models that do not implement any producer interface' do
+      before(:each) do
+        # Dummy class we can include the mixin in. Has a backing table created
+        # earlier and has the import hook disabled
+        buggy_class = Class.new(ActiveRecord::Base) do
+          include Deimos::KafkaSource
+          self.table_name = 'widgets'
+
+          # :nodoc:
+          def self.kafka_config
+            {
+              update: true,
+              delete: true,
+              import: false,
+              create: true
+            }
+          end
+        end
+        stub_const('WidgetBuggy', buggy_class)
+        WidgetBuggy.reset_column_information
+      end
+
+      it 'raises a NotImplementedError exception' do
+        expect {
+          WidgetBuggy.create(widget_id: 1, name: 'Widget 1')
+        }.to raise_error(NotImplementedError)
+      end
+    end
   end
 end

data/spec/kafka_topic_info_spec.rb

@@ -51,13 +51,13 @@ each_db_config(Deimos::KafkaTopicInfo) do
       expect(record.locked_at).to eq(nil)
       expect(record.error).to eq(false)
       expect(record.retries).to eq(0)
-      expect(record.last_processed_at.to_s).to eq(Time.zone.now.to_s)
+      expect(record.last_processed_at.in_time_zone.to_s).to eq(Time.zone.now.to_s)
       record = Deimos::KafkaTopicInfo.last
       expect(record.locked_by).not_to eq(nil)
       expect(record.locked_at).not_to eq(nil)
       expect(record.error).not_to eq(false)
       expect(record.retries).not_to eq(0)
-      expect(record.last_processed_at.to_s).to eq(20.seconds.ago.to_s)
+      expect(record.last_processed_at.in_time_zone.to_s).to eq(20.seconds.ago.to_s)
     end
   end
 
@@ -70,11 +70,11 @@ each_db_config(Deimos::KafkaTopicInfo) do
                                           locked_by: 'me', locked_at: 1.minute.ago)
 
       expect(Deimos::KafkaTopicInfo.count).to eq(3)
-      Deimos::KafkaTopicInfo.all.each { |t| expect(t.last_processed_at.to_s).to eq(old_time) }
+      Deimos::KafkaTopicInfo.all.each { |t| expect(t.last_processed_at.in_time_zone.to_s).to eq(old_time) }
       Deimos::KafkaTopicInfo.ping_empty_topics(%w(topic1))
-      expect(t1.reload.last_processed_at.to_s).to eq(old_time) # was passed as an exception
-      expect(t2.reload.last_processed_at.to_s).to eq(Time.zone.now.to_s)
-      expect(t3.reload.last_processed_at.to_s).to eq(old_time) # is locked
+      expect(t1.reload.last_processed_at.in_time_zone.to_s).to eq(old_time) # was passed as an exception
+      expect(t2.reload.last_processed_at.in_time_zone.to_s).to eq(Time.zone.now.to_s)
+      expect(t3.reload.last_processed_at.in_time_zone.to_s).to eq(old_time) # is locked
     end
   end
 

data/spec/schema_backends/avro_base_shared.rb

@@ -42,6 +42,20 @@ RSpec.shared_examples_for('an Avro backend') do
         {
           'name' => 'union-int-field',
           'type' => %w(null int)
+        },
+        {
+          'name' => 'timestamp-millis-field',
+          'type' => {
+            'type' => 'long',
+            'logicalType' => 'timestamp-millis'
+          }
+        },
+        {
+          'name' => 'timestamp-micros-field',
+          'type' => {
+            'type' => 'long',
+            'logicalType' => 'timestamp-micros'
+          }
         }
       ]
     }
@@ -95,7 +109,9 @@ RSpec.shared_examples_for('an Avro backend') do
         'string-field' => 'hi mom',
         'boolean-field' => true,
         'union-field' => nil,
-        'union-int-field' => nil
+        'union-int-field' => nil,
+        'timestamp-millis-field' => Time.utc(2020, 11, 12, 13, 14, 15, 909_090),
+        'timestamp-micros-field' => Time.utc(2020, 11, 12, 13, 14, 15, 909_090)
       }
     end
 
@@ -169,6 +185,15 @@ RSpec.shared_examples_for('an Avro backend') do
       expect(result['union-field']).to eq('itsme')
     end
 
+    it 'should not convert timestamp-millis' do
+      result = backend.coerce(payload)
+      expect(result['timestamp-millis-field']).to eq(Time.utc(2020, 11, 12, 13, 14, 15, 909_090))
+    end
+
+    it 'should not convert timestamp-micros' do
+      result = backend.coerce(payload)
+      expect(result['timestamp-micros-field']).to eq(Time.utc(2020, 11, 12, 13, 14, 15, 909_090))
+    end
   end
 
 end

data/spec/spec_helper.rb

@@ -87,7 +87,7 @@ module DbConfigs
       port: 3306,
       username: 'root',
       database: 'test',
-      host: ENV['MYSQL_HOST'] || 'localhost'
+      host: ENV['MYSQL_HOST'] || '127.0.0.1'
     },
     {
       adapter: 'sqlite3',

data/spec/utils/inline_consumer_spec.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+describe Deimos::Utils::SeekListener do
+
+  describe '#start_listener' do
+    let(:consumer) { instance_double(Kafka::Consumer) }
+    let(:handler) { class_double(Deimos::Utils::MessageBankHandler) }
+
+    before(:each) do
+      allow(handler).to receive(:start)
+      allow(consumer).to receive(:subscribe)
+      allow_any_instance_of(Phobos::Listener).to receive(:create_kafka_consumer).and_return(consumer)
+      allow_any_instance_of(Kafka::Client).to receive(:last_offset_for).and_return(100)
+      stub_const('Deimos::Utils::SeekListener::MAX_SEEK_RETRIES', 2)
+    end
+
+    it 'should seek offset' do
+      allow(consumer).to receive(:seek)
+      expect(consumer).to receive(:seek).once
+      seek_listener = described_class.new({ handler: handler, group_id: 999, topic: 'test_topic' })
+      seek_listener.start_listener
+    end
+
+    it 'should retry on errors when seeking offset' do
+      allow(consumer).to receive(:seek).and_raise(StandardError)
+      expect(consumer).to receive(:seek).twice
+      seek_listener = described_class.new({ handler: handler, group_id: 999, topic: 'test_topic' })
+      seek_listener.start_listener
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.8.3
+  version: 1.9.0
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-11-18 00:00:00.000000000 Z
+date: 2021-01-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf
@@ -28,30 +28,36 @@ dependencies:
   name: phobos
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.9'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.9'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: ruby-kafka
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: sigurd
   requirement: !ruby/object:Gem::Requirement
@@ -280,30 +286,30 @@ dependencies:
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0.72'
+        version: 0.88.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0.72'
+        version: 0.88.0
 - !ruby/object:Gem::Dependency
   name: rubocop-rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '1.27'
+        version: 1.42.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '1.27'
+        version: 1.42.0
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -465,6 +471,7 @@ files:
 - spec/utils/db_poller_spec.rb
 - spec/utils/db_producer_spec.rb
 - spec/utils/deadlock_retry_spec.rb
+- spec/utils/inline_consumer_spec.rb
 - spec/utils/lag_reporter_spec.rb
 - spec/utils/platform_schema_validation_spec.rb
 - spec/utils/schema_controller_mixin_spec.rb
@@ -491,7 +498,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.3
+rubygems_version: 3.0.9
 signing_key:
 specification_version: 4
 summary: Kafka libraries for Ruby.
@@ -548,6 +555,7 @@ test_files:
 - spec/utils/db_poller_spec.rb
 - spec/utils/db_producer_spec.rb
 - spec/utils/deadlock_retry_spec.rb
+- spec/utils/inline_consumer_spec.rb
 - spec/utils/lag_reporter_spec.rb
 - spec/utils/platform_schema_validation_spec.rb
 - spec/utils/schema_controller_mixin_spec.rb