deimos-ruby 1.8.2.pre.beta2 → 1.8.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b773de54759babb229e8f3bc6b42931dab7695b7197d601ad4ae08015c0b401f
-  data.tar.gz: 4af1aa98ffdfc013c4cac02fa316371ad138ec0d807ed58426a7ad57d6c1e719
+  metadata.gz: 65a333a1b518855f322e74ee096dc555c70dffa1b258d406bdea574584336f74
+  data.tar.gz: 6e6b64330307f03ed0311c96ff897c6573500a1de9cac997e4fade0b5987e6d9
 SHA512:
-  metadata.gz: 319d01a4c42c5efe72bd4959fb57548102c3f2b4305f3d94510747fd372d516ebad3e34ff74f9e6744641728efce9eb7a20f2d89c37d6bd90847253698a05238
-  data.tar.gz: 9e6999cca87ca448167a9e61a0bfa5914b36fd74c053f132bd02ab5200be840984546469affccfd07ce77d4d098a2096b13e42febd4ce7059ec882c436580cd8
+  metadata.gz: dd188a47a95c60a1d129314da2eec7f80cd842eb713e975d582edfa07a3cf4e966cd4433082439d162e0a29b84d8c72b92408f936eeedd3d9f058c002581d1da
+  data.tar.gz: 0c2a7e11df203cda96dba6ca16809c88e18d149b8e23a05d76641cd1039a70284315b497ca324155f4fb060099a3dd32be83b9fe9da5be5708cbc91954dfeddc

data/CHANGELOG.md

@@ -7,6 +7,50 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+## 1.8.6 - 2021-01-14
+
+- ### Fixes :wrench:
+- Fix for configuration bug with Ruby 3.0 (** instead of passing hash)
+
+## 1.8.5 - 2021-01-13
+
+- ### Fixes :wrench:
+- Fixes for Rails 6.1 (remove usage of `update_attributes!`)
+
+## 1.8.4 - 2020-12-02
+
+### Features :star:
+- Add overridable "process_message?" method to ActiveRecordConsumer to allow for skipping of saving/updating records
+
+### Fixes :wrench:
+
+- Do not apply type coercion to `timestamp-millis` and `timestamp-micros` logical types (fixes [#97](https://github.com/flipp-oss/deimos/issues/97))
+
+## 1.8.3 - 2020-11-18
+
+### Fixes :wrench:
+- Do not resend already sent messages when splitting up batches
+  (fixes [#24](https://github.com/flipp-oss/deimos/issues/24))
+- KafkaSource crashing on bulk-imports if import hooks are disabled
+  (fixes [#73](https://github.com/flipp-oss/deimos/issues/73))
+- #96 Use string-safe encoding for partition keys
+- Retry on offset seek failures in inline consumer
+  (fixes [#5](Inline consumer should use retries when seeking))
+
+## 1.8.2 - 2020-09-25
+
+### Features :star:
+- Add "disabled" config field to consumers to allow disabling
+  individual consumers without having to comment out their
+  entries and possibly affecting unit tests.
+
+### Fixes :wrench:
+- Prepend topic_prefix while encoding messages
+  (fixes [#37](https://github.com/flipp-oss/deimos/issues/37))
+- Raise error if producing without a topic
+  (fixes [#50](https://github.com/flipp-oss/deimos/issues/50))
+- Don't try to load producers/consumers when running rake tasks involving webpacker or assets
+
 ## 1.8.2-beta2 - 2020-09-15
 
 ### Features :star:

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    deimos-ruby (1.8.2.pre.beta2)
+    deimos-ruby (1.8.6)
       avro_turf (~> 0.11)
       phobos (~> 1.9)
       ruby-kafka (~> 0.7)
@@ -10,63 +10,67 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (6.0.3.2)
-      actionpack (= 6.0.3.2)
+    actioncable (6.1.1)
+      actionpack (= 6.1.1)
+      activesupport (= 6.1.1)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailbox (6.0.3.2)
-      actionpack (= 6.0.3.2)
-      activejob (= 6.0.3.2)
-      activerecord (= 6.0.3.2)
-      activestorage (= 6.0.3.2)
-      activesupport (= 6.0.3.2)
+    actionmailbox (6.1.1)
+      actionpack (= 6.1.1)
+      activejob (= 6.1.1)
+      activerecord (= 6.1.1)
+      activestorage (= 6.1.1)
+      activesupport (= 6.1.1)
       mail (>= 2.7.1)
-    actionmailer (6.0.3.2)
-      actionpack (= 6.0.3.2)
-      actionview (= 6.0.3.2)
-      activejob (= 6.0.3.2)
+    actionmailer (6.1.1)
+      actionpack (= 6.1.1)
+      actionview (= 6.1.1)
+      activejob (= 6.1.1)
+      activesupport (= 6.1.1)
       mail (~> 2.5, >= 2.5.4)
       rails-dom-testing (~> 2.0)
-    actionpack (6.0.3.2)
-      actionview (= 6.0.3.2)
-      activesupport (= 6.0.3.2)
-      rack (~> 2.0, >= 2.0.8)
+    actionpack (6.1.1)
+      actionview (= 6.1.1)
+      activesupport (= 6.1.1)
+      rack (~> 2.0, >= 2.0.9)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actiontext (6.0.3.2)
-      actionpack (= 6.0.3.2)
-      activerecord (= 6.0.3.2)
-      activestorage (= 6.0.3.2)
-      activesupport (= 6.0.3.2)
+    actiontext (6.1.1)
+      actionpack (= 6.1.1)
+      activerecord (= 6.1.1)
+      activestorage (= 6.1.1)
+      activesupport (= 6.1.1)
       nokogiri (>= 1.8.5)
-    actionview (6.0.3.2)
-      activesupport (= 6.0.3.2)
+    actionview (6.1.1)
+      activesupport (= 6.1.1)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.1, >= 1.2.0)
-    activejob (6.0.3.2)
-      activesupport (= 6.0.3.2)
+    activejob (6.1.1)
+      activesupport (= 6.1.1)
       globalid (>= 0.3.6)
-    activemodel (6.0.3.2)
-      activesupport (= 6.0.3.2)
-    activerecord (6.0.3.2)
-      activemodel (= 6.0.3.2)
-      activesupport (= 6.0.3.2)
-    activerecord-import (1.0.5)
+    activemodel (6.1.1)
+      activesupport (= 6.1.1)
+    activerecord (6.1.1)
+      activemodel (= 6.1.1)
+      activesupport (= 6.1.1)
+    activerecord-import (1.0.7)
       activerecord (>= 3.2)
-    activestorage (6.0.3.2)
-      actionpack (= 6.0.3.2)
-      activejob (= 6.0.3.2)
-      activerecord (= 6.0.3.2)
+    activestorage (6.1.1)
+      actionpack (= 6.1.1)
+      activejob (= 6.1.1)
+      activerecord (= 6.1.1)
+      activesupport (= 6.1.1)
       marcel (~> 0.3.1)
-    activesupport (6.0.3.2)
+      mimemagic (~> 0.3.2)
+    activesupport (6.1.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (>= 0.7, < 2)
-      minitest (~> 5.1)
-      tzinfo (~> 1.1)
-      zeitwerk (~> 2.2, >= 2.2.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
     ast (2.4.1)
     avro (1.9.2)
       multi_json
@@ -83,11 +87,11 @@ GEM
     ddtrace (0.37.0)
       msgpack
     diff-lcs (1.4.4)
-    digest-crc (0.6.1)
-      rake (~> 13.0)
+    digest-crc (0.6.3)
+      rake (>= 12.0.0, < 14.0.0)
     dogstatsd-ruby (4.8.1)
-    erubi (1.9.0)
-    excon (0.76.0)
+    erubi (1.10.0)
+    excon (0.78.1)
     exponential-backoff (0.0.4)
     ffi (1.13.1)
     formatador (0.2.5)
@@ -110,7 +114,7 @@ GEM
     guard-rubocop (1.3.0)
       guard (~> 2.0)
       rubocop (~> 0.20)
-    i18n (1.8.4)
+    i18n (1.8.7)
       concurrent-ruby (~> 1.0)
     listen (3.2.1)
       rb-fsevent (~> 0.10, >= 0.10.3)
@@ -119,7 +123,7 @@ GEM
     logging (2.3.0)
       little-plugger (~> 1.1)
       multi_json (~> 1.14)
-    loofah (2.6.0)
+    loofah (2.8.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     lumberjack (1.2.6)
@@ -130,15 +134,16 @@ GEM
     method_source (1.0.0)
     mimemagic (0.3.5)
     mini_mime (1.0.2)
-    mini_portile2 (2.4.0)
-    minitest (5.14.1)
+    mini_portile2 (2.5.0)
+    minitest (5.14.3)
     msgpack (1.3.3)
     multi_json (1.15.0)
     mysql2 (0.5.3)
     nenv (0.3.0)
-    nio4r (2.5.2)
-    nokogiri (1.10.10)
-      mini_portile2 (~> 2.4.0)
+    nio4r (2.5.4)
+    nokogiri (1.11.1)
+      mini_portile2 (~> 2.5.0)
+      racc (~> 1.4)
     notiffany (0.1.3)
       nenv (~> 0.1)
       shellany (~> 0.0)
@@ -157,37 +162,38 @@ GEM
     pry (0.13.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
+    racc (1.5.2)
     rack (2.2.3)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
-    rails (6.0.3.2)
-      actioncable (= 6.0.3.2)
-      actionmailbox (= 6.0.3.2)
-      actionmailer (= 6.0.3.2)
-      actionpack (= 6.0.3.2)
-      actiontext (= 6.0.3.2)
-      actionview (= 6.0.3.2)
-      activejob (= 6.0.3.2)
-      activemodel (= 6.0.3.2)
-      activerecord (= 6.0.3.2)
-      activestorage (= 6.0.3.2)
-      activesupport (= 6.0.3.2)
-      bundler (>= 1.3.0)
-      railties (= 6.0.3.2)
+    rails (6.1.1)
+      actioncable (= 6.1.1)
+      actionmailbox (= 6.1.1)
+      actionmailer (= 6.1.1)
+      actionpack (= 6.1.1)
+      actiontext (= 6.1.1)
+      actionview (= 6.1.1)
+      activejob (= 6.1.1)
+      activemodel (= 6.1.1)
+      activerecord (= 6.1.1)
+      activestorage (= 6.1.1)
+      activesupport (= 6.1.1)
+      bundler (>= 1.15.0)
+      railties (= 6.1.1)
       sprockets-rails (>= 2.0.0)
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
     rails-html-sanitizer (1.3.0)
       loofah (~> 2.3)
-    railties (6.0.3.2)
-      actionpack (= 6.0.3.2)
-      activesupport (= 6.0.3.2)
+    railties (6.1.1)
+      actionpack (= 6.1.1)
+      activesupport (= 6.1.1)
       method_source
       rake (>= 0.8.7)
-      thor (>= 0.20.3, < 2.0)
+      thor (~> 1.0)
     rainbow (3.0.0)
-    rake (13.0.1)
+    rake (13.0.3)
     rb-fsevent (0.10.4)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
@@ -239,20 +245,19 @@ GEM
     sprockets (4.0.2)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
-    sprockets-rails (3.2.1)
+    sprockets-rails (3.2.2)
       actionpack (>= 4.0)
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)
     sqlite3 (1.4.2)
     thor (1.0.1)
-    thread_safe (0.3.6)
-    tzinfo (1.2.7)
-      thread_safe (~> 0.1)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
     unicode-display_width (1.7.0)
     websocket-driver (0.7.3)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
-    zeitwerk (2.4.0)
+    zeitwerk (2.4.2)
 
 PLATFORMS
   ruby
@@ -274,9 +279,9 @@ DEPENDENCIES
   rspec (~> 3)
   rspec-rails (~> 4)
   rspec_junit_formatter (~> 0.3)
-  rubocop (~> 0.72)
-  rubocop-rspec (~> 1.27)
+  rubocop (= 0.88.0)
+  rubocop-rspec (= 1.42.0)
   sqlite3 (~> 1.3)
 
 BUNDLED WITH
-   2.1.4
+   2.2.5

data/README.md

@@ -42,6 +42,7 @@ Please see the following for further information not covered by this readme:
 * [Configuration Reference](docs/CONFIGURATION.md)
 * [Database Backend Feature](docs/DATABASE_BACKEND.md)
 * [Upgrading Deimos](docs/UPGRADING.md)
+* [Contributing to Integration Tests](docs/INTEGRATION_TESTS.md)
 
 # Installation
 
@@ -625,6 +626,15 @@ class MyConsumer < Deimos::ActiveRecordConsumer
   def record_key(payload)
     super
   end
+
+  # Optional override, returns true by default.
+  # When this method returns true, a record corresponding to the message
+  # is created/updated.
+  # When this method returns false, message processing is skipped and a
+  # corresponding record will NOT be created/updated.
+  def process_message?(payload)
+    super
+  end
 end
 ```
 

data/deimos-ruby.gemspec

@@ -38,7 +38,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency('rspec', '~> 3')
   spec.add_development_dependency('rspec_junit_formatter', '~>0.3')
   spec.add_development_dependency('rspec-rails', '~> 4')
-  spec.add_development_dependency('rubocop', '~> 0.72')
-  spec.add_development_dependency('rubocop-rspec', '~> 1.27')
+  spec.add_development_dependency('rubocop', '0.88.0')
+  spec.add_development_dependency('rubocop-rspec', '1.42.0')
   spec.add_development_dependency('sqlite3', '~> 1.3')
 end

data/docs/CONFIGURATION.md

@@ -79,6 +79,7 @@ topic|nil|Topic to produce to.
 schema|nil|This is optional but strongly recommended for testing purposes; this will validate against a local schema file used as the reader schema, as well as being able to write tests against this schema. This is recommended since it ensures you are always getting the values  you expect.
 namespace|nil|Namespace of the schema to use when finding it locally.
 key_config|nil|Configuration hash for message keys. See [Kafka Message Keys](../README.md#installation)
+disabled|false|Set to true to skip starting an actual listener for this consumer on startup.
 group_id|nil|ID of the consumer group.
 max_concurrency|1|Number of threads created for this listener. Each thread will behave as an independent consumer. They don't share any state.
 start_from_beginning|true|Once the consumer group has checkpointed its progress in the topic's partitions, the consumers will always start from the checkpointed offsets, regardless of config. As such, this setting only applies when the consumer initially starts consuming from a topic

data/docs/INTEGRATION_TESTS.md

@@ -0,0 +1,52 @@
+# Running Integration Tests
+
+This repo includes integration tests in the [spec/utils](spec/utils) directory.
+Here, there are tests for more deimos features that include a database integration like
+* [Database Poller](README.md#Database Poller)
+* [Database Backend](docs/DATABASE_BACKEND.md)
+* [Deadlock Retrying](lib/deimos/utils/deadlock_retry.rb)
+
+You will need to set up the following databases to develop and create unit tests in these test suites.
+* [SQLite](#SQLite)
+* [MySQL](#MySQL)
+* [PostgreSQL](#PostgreSQL)
+
+### SQLite
+This database is covered through the `sqlite3` gem. 
+
+## MySQL
+### Setting up a local MySQL server (Mac)
+```bash
+# Download MySQL (Optionally, choose a version you are comfortable with)
+brew install mysql
+# Start automatically after rebooting your machine
+brew services start mysql
+
+# Cleanup once you are done with MySQL
+brew services stop mysql
+```
+
+## PostgreSQL
+### Setting up a local PostgreSQL server (Mac)
+```bash
+# Install postgres if it's not already installed
+brew install postgres
+
+# Initialize and Start up postgres db
+brew services start postgres
+initdb /usr/local/var/postgres
+# Create the default database and user
+# Use the password "root"
+createuser -s --password postgres
+
+# Cleanup once done with Postgres
+killall postgres
+brew services stop postgres
+```
+
+## Running Integration Tests
+You must specify the tag "integration" when running these these test suites.
+This can be done through the CLI with the `--tag integration` argument.
+```bash
+rspec spec/utils/ --tag integration
+```

data/docs/PULL_REQUEST_TEMPLATE.md

@@ -28,6 +28,7 @@ Please describe the tests that you ran to verify your changes. Provide instructi
 - [ ] I have performed a self-review of my own code
 - [ ] I have commented my code, particularly in hard-to-understand areas
 - [ ] I have made corresponding changes to the documentation
+- [ ] I have added a line in the CHANGELOG describing this change, under the UNRELEASED heading
 - [ ] My changes generate no new warnings
 - [ ] I have added tests that prove my fix is effective or that my feature works
 - [ ] New and existing unit tests pass locally with my changes

data/lib/deimos/active_record_consume/message_consumption.rb

@@ -26,6 +26,15 @@ module Deimos
 
       # :nodoc:
       def consume(payload, metadata)
+        unless self.process_message?(payload)
+          Deimos.config.logger.debug(
+            message: 'Skipping processing of message',
+            payload: payload,
+            metadata: metadata
+          )
+          return
+        end
+
         key = metadata.with_indifferent_access[:key]
         klass = self.class.config[:record_class]
         record = fetch_record(klass, (payload || {}).with_indifferent_access, key)

data/lib/deimos/active_record_consumer.rb

@@ -55,5 +55,13 @@ module Deimos
     def record_attributes(payload, _key=nil)
       @converter.convert(payload)
     end
+
+    # Override this message to conditionally save records
+    # @param payload [Hash] The kafka message as a hash
+    # @return [Boolean] if true, record is created/update.
+    #   If false, record processing is skipped but message offset is still committed.
+    def process_message?(_payload)
+      true
+    end
   end
 end

data/lib/deimos/backends/db.rb

@@ -14,7 +14,7 @@ module Deimos
             message = Deimos::KafkaMessage.new(
               message: m.encoded_payload ? m.encoded_payload.to_s.b : nil,
               topic: m.topic,
-              partition_key: m.partition_key || m.key
+              partition_key: partition_key_for(m)
             )
             message.key = m.encoded_key.to_s.b unless producer_class.config[:no_keys]
             message
@@ -26,6 +26,15 @@ module Deimos
             by: records.size
           )
         end
+
+        # @param message [Deimos::Message]
+        # @return [String] the partition key to use for this message
+        def partition_key_for(message)
+          return message.partition_key if message.partition_key.present?
+          return message.key unless message.key.is_a?(Hash)
+
+          message.key.to_yaml
+        end
       end
     end
   end

data/lib/deimos/config/configurable.rb

@@ -15,6 +15,11 @@ module Deimos
   #       enabled true
   #       ca_cert_file 'my_file'
   #     end
+  #     config.kafka do
+  #       ssl do
+  #         enabled true
+  #       end
+  #     end
   #   end
   # - Allows for arrays of configurations:
   #   Deimos.configure do |config|
@@ -245,6 +250,13 @@ module Deimos
 
       # Configure the settings with values.
       def configure(&block)
+        if defined?(Rake) && defined?(Rake.application)
+          tasks = Rake.application.top_level_tasks
+          if tasks.any? { |t| %w(assets webpacker yarn).include?(t.split(':').first) }
+            puts 'Skipping Deimos configuration since we are in JS/CSS compilation'
+            return
+          end
+        end
         config.run_callbacks(:configure) do
           config.instance_eval(&block)
         end

data/lib/deimos/config/configuration.rb

@@ -67,7 +67,7 @@ module Deimos
       topic(kafka_config.topic) if kafka_config.topic.present? && klass.respond_to?(:topic)
       schema(kafka_config.schema) if kafka_config.schema.present?
       namespace(kafka_config.namespace) if kafka_config.namespace.present?
-      key_config(kafka_config.key_config) if kafka_config.key_config.present?
+      key_config(**kafka_config.key_config) if kafka_config.key_config.present?
     end
   end
 
@@ -319,6 +319,10 @@ module Deimos
       # Key configuration (see docs).
       # @return [Hash]
       setting :key_config
+      # Set to true to ignore the consumer in the Phobos config and not actually start up a
+      # listener.
+      # @return [Boolean]
+      setting :disabled, false
 
       # These are the phobos "listener" configs. See CONFIGURATION.md for more
       # info.

data/lib/deimos/config/phobos_config.rb

@@ -63,8 +63,10 @@ module Deimos
       }
 
       p_config[:listeners] = self.consumer_objects.map do |consumer|
+        next nil if consumer.disabled
+
         hash = consumer.to_h.reject do |k, _|
-          %i(class_name schema namespace key_config backoff).include?(k)
+          %i(class_name schema namespace key_config backoff disabled).include?(k)
         end
         hash = hash.map { |k, v| [k, v.is_a?(Symbol) ? v.to_s : v] }.to_h
         hash[:handler] = consumer.class_name
@@ -73,6 +75,7 @@ module Deimos
         end
         hash
       end
+      p_config[:listeners].compact!
 
       if self.kafka.ssl.enabled
         %w(ca_cert client_cert client_cert_key).each do |key|

data/lib/deimos/kafka_source.rb

@@ -88,8 +88,9 @@ module Deimos
                                                   array_of_attributes,
                                                   options={})
         results = super
-        return unless self.kafka_config[:import]
-        return if array_of_attributes.empty?
+        if !self.kafka_config[:import] || array_of_attributes.empty?
+          return results
+        end
 
         # This will contain an array of hashes, where each hash is the actual
         # attribute hash that created the object.

data/lib/deimos/kafka_topic_info.rb

@@ -85,11 +85,8 @@ module Deimos
                       locked_at: Time.zone.now,
                       error: true,
                       retries: record.retries + 1 }
-        if ActiveRecord::VERSION::MAJOR >= 4
-          record.update!(attr_hash)
-        else
-          record.update_attributes!(attr_hash)
-        end
+        record.attributes = attr_hash
+        record.save!
       end
 
       # Update the locked_at timestamp to indicate that the producer is still

data/lib/deimos/producer.rb

@@ -104,6 +104,8 @@ module Deimos
                   Deimos.config.producers.disabled ||
                   Deimos.producers_disabled?(self)
 
+        raise 'Topic not specified. Please specify the topic.' if topic.blank?
+
         backend_class = determine_backend_class(sync, force_send)
         Deimos.instrument(
           'encode_messages',
@@ -183,7 +185,7 @@ module Deimos
                                     nil
                                   else
                                     encoder.encode(message.payload,
-                                                   topic: "#{config[:topic]}-value")
+                                                   topic: "#{Deimos.config.producers.topic_prefix}#{config[:topic]}-value")
                                   end
       end
 
@@ -201,9 +203,9 @@ module Deimos
         end
 
         if config[:key_field]
-          encoder.encode_key(config[:key_field], key, topic: "#{config[:topic]}-key")
+          encoder.encode_key(config[:key_field], key, topic: "#{Deimos.config.producers.topic_prefix}#{config[:topic]}-key")
         elsif config[:key_schema]
-          key_encoder.encode(key, topic: "#{config[:topic]}-key")
+          key_encoder.encode(key, topic: "#{Deimos.config.producers.topic_prefix}#{config[:topic]}-key")
         else
           key
         end

data/lib/deimos/schema_backends/avro_schema_coercer.rb

@@ -44,9 +44,11 @@ module Deimos
 
       case field_type
       when :int, :long
-        if val.is_a?(Integer) ||
-           _is_integer_string?(val) ||
-           int_classes.any? { |klass| val.is_a?(klass) }
+        if %w(timestamp-millis timestamp-micros).include?(type.logical_type)
+          val
+        elsif val.is_a?(Integer) ||
+              _is_integer_string?(val) ||
+              int_classes.any? { |klass| val.is_a?(klass) }
           val.to_i
         else
           val # this will fail

data/lib/deimos/utils/db_poller.rb

@@ -142,7 +142,8 @@ module Deimos
         last_id = record.public_send(id_method)
         last_updated_at = last_updated(record)
         @producer.send_events(batch)
-        @info.update_attributes!(last_sent: last_updated_at, last_sent_id: last_id)
+        @info.attributes = { last_sent: last_updated_at, last_sent_id: last_id }
+        @info.save!
       end
     end
   end

data/lib/deimos/utils/db_producer.rb

@@ -190,11 +190,14 @@ module Deimos
         end
       end
 
+      # Produce messages in batches, reducing the size 1/10 if the batch is too
+      # large. Does not retry batches of messages that have already been sent.
       # @param batch [Array<Hash>]
       def produce_messages(batch)
         batch_size = batch.size
+        current_index = 0
         begin
-          batch.in_groups_of(batch_size, false).each do |group|
+          batch[current_index..-1].in_groups_of(batch_size, false).each do |group|
             @logger.debug("Publishing #{group.size} messages to #{@current_topic}")
             producer.publish_list(group)
             Deimos.config.metrics&.increment(
@@ -202,6 +205,7 @@ module Deimos
               tags: %W(status:success topic:#{@current_topic}),
               by: group.size
             )
+            current_index += group.size
             @logger.info("Sent #{group.size} messages to #{@current_topic}")
           end
         rescue Kafka::BufferOverflow, Kafka::MessageSizeTooLarge,

data/lib/deimos/utils/inline_consumer.rb

@@ -6,6 +6,7 @@ module Deimos
   module Utils
     # Listener that can seek to get the last X messages in a topic.
     class SeekListener < Phobos::Listener
+      MAX_SEEK_RETRIES = 3
       attr_accessor :num_messages
 
       # :nodoc:
@@ -13,8 +14,10 @@ module Deimos
         @num_messages ||= 10
         @consumer = create_kafka_consumer
         @consumer.subscribe(topic, @subscribe_opts)
+        attempt = 0
 
         begin
+          attempt += 1
           last_offset = @kafka_client.last_offset_for(topic, 0)
           offset = last_offset - num_messages
           if offset.positive?
@@ -22,7 +25,11 @@ module Deimos
             @consumer.seek(topic, 0, offset)
           end
         rescue StandardError => e
-          "Could not seek to offset: #{e.message}"
+          if attempt < MAX_SEEK_RETRIES
+            sleep(1.seconds * attempt)
+            retry
+          end
+          log_error("Could not seek to offset: #{e.message} after #{MAX_SEEK_RETRIES} retries", listener_metadata)
         end
 
         instrument('listener.start_handler', listener_metadata) do
@@ -50,7 +57,6 @@ module Deimos
 
       # :nodoc:
       def consume(payload, metadata)
-        puts "Got #{payload}"
         self.class.total_messages << {
           key: metadata[:key],
           payload: payload

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.8.2-beta2'
+  VERSION = '1.8.6'
 end

data/spec/active_record_consumer_spec.rb

@@ -137,5 +137,18 @@ module ActiveRecordConsumerTest
       expect(Widget.find_by_test_id('id1').some_int).to eq(3)
       expect(Widget.find_by_test_id('id2').some_int).to eq(4)
     end
+
+    it 'should not create record of process_message returns false' do
+      MyConsumer.any_instance.stub(:process_message?).and_return(false)
+      expect(Widget.count).to eq(0)
+      test_consume_message(MyConsumer, {
+                             test_id: 'abc',
+                             some_int: 3,
+                             updated_at: 1.day.ago.to_i,
+                             some_datetime_int: Time.zone.now.to_i,
+                             timestamp: 2.minutes.ago.to_s
+                           }, { call_original: true, key: 5 })
+      expect(Widget.count).to eq(0)
+    end
   end
 end

data/spec/backends/db_spec.rb

@@ -43,6 +43,12 @@ each_db_config(Deimos::Backends::Db) do
     described_class.publish(producer_class: MyNoKeyProducer,
                             messages: [messages.first])
     expect(Deimos::KafkaMessage.count).to eq(4)
+  end
 
+  it 'should add messages with Hash keys with JSON encoding' do
+    described_class.publish(producer_class: MyProducer,
+                            messages: [build_message({ foo: 0 }, 'my-topic', { 'test_id' => 0 })])
+    expect(Deimos::KafkaMessage.count).to eq(1)
+    expect(Deimos::KafkaMessage.last.partition_key).to eq(%(---\ntest_id: 0\n))
   end
 end

data/spec/config/configuration_spec.rb

@@ -6,6 +6,14 @@ class MyConfigConsumer < Deimos::Consumer
   def consume
   end
 end
+
+# Mock consumer 2
+class MyConfigConsumer2 < Deimos::Consumer
+  # :no-doc:
+  def consume
+  end
+end
+
 describe Deimos, 'configuration' do
   it 'should configure with deprecated fields' do
     logger = Logger.new(nil)
@@ -171,6 +179,13 @@ describe Deimos, 'configuration' do
         offset_retention_time 13
         heartbeat_interval 13
       end
+      consumer do
+        disabled true
+        class_name 'MyConfigConsumer2'
+        schema 'blah2'
+        topic 'blah2'
+        group_id 'myconsumerid2'
+      end
     end
 
     expect(described_class.config.phobos_config).

data/spec/generators/active_record_generator_spec.rb

@@ -16,7 +16,7 @@ RSpec.describe Deimos::Generators::ActiveRecordGenerator do
     files = Dir['db/migrate/*.rb']
     expect(files.length).to eq(1)
     results = <<~MIGRATION
-      class CreateGeneratedTable < ActiveRecord::Migration[6.0]
+      class CreateGeneratedTable < ActiveRecord::Migration[6.1]
         def up
           if table_exists?(:generated_table)
             warn "generated_table already exists, exiting"

data/spec/kafka_source_spec.rb

@@ -225,5 +225,88 @@ module KafkaSourceSpec
         expect(Deimos::KafkaMessage.count).to eq(0)
       end
     end
+
+    context 'with import hooks disabled' do
+      before(:each) do
+        # Dummy class we can include the mixin in. Has a backing table created
+        # earlier and has the import hook disabled
+        class WidgetNoImportHook < ActiveRecord::Base
+          include Deimos::KafkaSource
+          self.table_name = 'widgets'
+
+          # :nodoc:
+          def self.kafka_config
+            {
+              update: true,
+              delete: true,
+              import: false,
+              create: true
+            }
+          end
+
+          # :nodoc:
+          def self.kafka_producers
+            [WidgetProducer]
+          end
+        end
+        WidgetNoImportHook.reset_column_information
+      end
+
+      it 'should not fail when bulk-importing with existing records' do
+        widget1 = WidgetNoImportHook.create(widget_id: 1, name: 'Widget 1')
+        widget2 = WidgetNoImportHook.create(widget_id: 2, name: 'Widget 2')
+        widget1.name = 'New Widget No Import Hook 1'
+        widget2.name = 'New Widget No Import Hook 2'
+
+        expect {
+          WidgetNoImportHook.import([widget1, widget2], on_duplicate_key_update: %i(widget_id name))
+        }.not_to raise_error
+
+        expect('my-topic').not_to have_sent({
+                                              widget_id: 1,
+                                              name: 'New Widget No Import Hook 1',
+                                              id: widget1.id,
+                                              created_at: anything,
+                                              updated_at: anything
+                                            }, widget1.id)
+        expect('my-topic').not_to have_sent({
+                                              widget_id: 2,
+                                              name: 'New Widget No Import Hook 2',
+                                              id: widget2.id,
+                                              created_at: anything,
+                                              updated_at: anything
+                                            }, widget2.id)
+      end
+
+      it 'should not fail when mixing existing and new records' do
+        widget1 = WidgetNoImportHook.create(widget_id: 1, name: 'Widget 1')
+        expect('my-topic').to have_sent({
+                                          widget_id: 1,
+                                          name: 'Widget 1',
+                                          id: widget1.id,
+                                          created_at: anything,
+                                          updated_at: anything
+                                        }, widget1.id)
+
+        widget2 = WidgetNoImportHook.new(widget_id: 2, name: 'Widget 2')
+        widget1.name = 'New Widget 1'
+        WidgetNoImportHook.import([widget1, widget2], on_duplicate_key_update: %i(widget_id))
+        widgets = WidgetNoImportHook.all
+        expect('my-topic').not_to have_sent({
+                                              widget_id: 1,
+                                              name: 'New Widget 1',
+                                              id: widgets[0].id,
+                                              created_at: anything,
+                                              updated_at: anything
+                                            }, widgets[0].id)
+        expect('my-topic').not_to have_sent({
+                                              widget_id: 2,
+                                              name: 'Widget 2',
+                                              id: widgets[1].id,
+                                              created_at: anything,
+                                              updated_at: anything
+                                            }, widgets[1].id)
+      end
+    end
   end
 end

data/spec/kafka_topic_info_spec.rb

@@ -51,13 +51,13 @@ each_db_config(Deimos::KafkaTopicInfo) do
       expect(record.locked_at).to eq(nil)
       expect(record.error).to eq(false)
       expect(record.retries).to eq(0)
-      expect(record.last_processed_at.to_s).to eq(Time.zone.now.to_s)
+      expect(record.last_processed_at.in_time_zone.to_s).to eq(Time.zone.now.to_s)
       record = Deimos::KafkaTopicInfo.last
       expect(record.locked_by).not_to eq(nil)
       expect(record.locked_at).not_to eq(nil)
       expect(record.error).not_to eq(false)
       expect(record.retries).not_to eq(0)
-      expect(record.last_processed_at.to_s).to eq(20.seconds.ago.to_s)
+      expect(record.last_processed_at.in_time_zone.to_s).to eq(20.seconds.ago.to_s)
     end
   end
 
@@ -70,11 +70,11 @@ each_db_config(Deimos::KafkaTopicInfo) do
                                           locked_by: 'me', locked_at: 1.minute.ago)
 
       expect(Deimos::KafkaTopicInfo.count).to eq(3)
-      Deimos::KafkaTopicInfo.all.each { |t| expect(t.last_processed_at.to_s).to eq(old_time) }
+      Deimos::KafkaTopicInfo.all.each { |t| expect(t.last_processed_at.in_time_zone.to_s).to eq(old_time) }
       Deimos::KafkaTopicInfo.ping_empty_topics(%w(topic1))
-      expect(t1.reload.last_processed_at.to_s).to eq(old_time) # was passed as an exception
-      expect(t2.reload.last_processed_at.to_s).to eq(Time.zone.now.to_s)
-      expect(t3.reload.last_processed_at.to_s).to eq(old_time) # is locked
+      expect(t1.reload.last_processed_at.in_time_zone.to_s).to eq(old_time) # was passed as an exception
+      expect(t2.reload.last_processed_at.in_time_zone.to_s).to eq(Time.zone.now.to_s)
+      expect(t3.reload.last_processed_at.in_time_zone.to_s).to eq(old_time) # is locked
     end
   end
 

data/spec/producer_spec.rb

@@ -64,6 +64,14 @@ module ProducerTest
       end
       stub_const('MyErrorProducer', producer_class)
 
+      producer_class = Class.new(Deimos::Producer) do
+        schema 'MySchema'
+        namespace 'com.my-namespace'
+        topic nil
+        key_config none: true
+      end
+      stub_const('MyNoTopicProducer', producer_class)
+
     end
 
     it 'should fail on invalid message with error handler' do
@@ -228,6 +236,7 @@ module ProducerTest
     end
 
     it 'should encode the key' do
+      Deimos.configure { |c| c.producers.topic_prefix = nil }
       expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'my-topic-key')
       expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'bar', topic: 'my-topic-key')
       expect(MyProducer.encoder).to receive(:encode).with({
@@ -245,6 +254,21 @@ module ProducerTest
       )
     end
 
+    it 'should encode the key with topic prefix' do
+      Deimos.configure { |c| c.producers.topic_prefix = 'prefix.' }
+      expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'prefix.my-topic-key')
+      expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'bar', topic: 'prefix.my-topic-key')
+      expect(MyProducer.encoder).to receive(:encode).with({  'test_id' => 'foo',
+                                                             'some_int' => 123 },
+                                                          { topic: 'prefix.my-topic-value' })
+      expect(MyProducer.encoder).to receive(:encode).with({  'test_id' => 'bar',
+                                                             'some_int' => 124 },
+                                                          { topic: 'prefix.my-topic-value' })
+
+      MyProducer.publish_list([{ 'test_id' => 'foo', 'some_int' => 123 },
+                               { 'test_id' => 'bar', 'some_int' => 124 }])
+    end
+
     it 'should not encode with plaintext key' do
       expect(MyNonEncodedProducer.key_encoder).not_to receive(:encode_key)
 
@@ -296,6 +320,31 @@ module ProducerTest
       )
     end
 
+    it 'should raise error if blank topic is passed in explicitly' do
+      expect {
+        MyProducer.publish_list(
+          [{  'test_id' => 'foo',
+              'some_int' => 123 },
+           {  'test_id' => 'bar',
+              'some_int' => 124 }],
+          topic: ''
+        )
+      }.to raise_error(RuntimeError,
+                       'Topic not specified. Please specify the topic.')
+    end
+
+    it 'should raise error if the producer has not been initialized with a topic' do
+      expect {
+        MyNoTopicProducer.publish_list(
+          [{  'test_id' => 'foo',
+              'some_int' => 123 },
+           {  'test_id' => 'bar',
+              'some_int' => 124 }]
+        )
+      }.to raise_error(RuntimeError,
+                       'Topic not specified. Please specify the topic.')
+    end
+
     it 'should error with nothing set' do
       expect {
         MyErrorProducer.publish_list(

data/spec/schema_backends/avro_base_shared.rb

@@ -42,6 +42,20 @@ RSpec.shared_examples_for('an Avro backend') do
         {
           'name' => 'union-int-field',
           'type' => %w(null int)
+        },
+        {
+          'name' => 'timestamp-millis-field',
+          'type' => {
+            'type' => 'long',
+            'logicalType' => 'timestamp-millis'
+          }
+        },
+        {
+          'name' => 'timestamp-micros-field',
+          'type' => {
+            'type' => 'long',
+            'logicalType' => 'timestamp-micros'
+          }
         }
       ]
     }
@@ -95,7 +109,9 @@ RSpec.shared_examples_for('an Avro backend') do
         'string-field' => 'hi mom',
         'boolean-field' => true,
         'union-field' => nil,
-        'union-int-field' => nil
+        'union-int-field' => nil,
+        'timestamp-millis-field' => Time.utc(2020, 11, 12, 13, 14, 15, 909_090),
+        'timestamp-micros-field' => Time.utc(2020, 11, 12, 13, 14, 15, 909_090)
       }
     end
 
@@ -169,6 +185,15 @@ RSpec.shared_examples_for('an Avro backend') do
       expect(result['union-field']).to eq('itsme')
     end
 
+    it 'should not convert timestamp-millis' do
+      result = backend.coerce(payload)
+      expect(result['timestamp-millis-field']).to eq(Time.utc(2020, 11, 12, 13, 14, 15, 909_090))
+    end
+
+    it 'should not convert timestamp-micros' do
+      result = backend.coerce(payload)
+      expect(result['timestamp-micros-field']).to eq(Time.utc(2020, 11, 12, 13, 14, 15, 909_090))
+    end
   end
 
 end

data/spec/spec_helper.rb

@@ -87,7 +87,7 @@ module DbConfigs
       port: 3306,
       username: 'root',
       database: 'test',
-      host: ENV['MYSQL_HOST'] || 'localhost'
+      host: ENV['MYSQL_HOST'] || '127.0.0.1'
     },
     {
       adapter: 'sqlite3',

data/spec/utils/db_producer_spec.rb

@@ -96,7 +96,32 @@ each_db_config(Deimos::Utils::DbProducer) do
       expect(phobos_producer).to have_received(:publish_list).with(['A'] * 100).once
       expect(phobos_producer).to have_received(:publish_list).with(['A'] * 10).once
       expect(phobos_producer).to have_received(:publish_list).with(['A']).once
+    end
+
+    it 'should not resend batches of sent messages' do
+      allow(phobos_producer).to receive(:publish_list) do |group|
+        raise Kafka::BufferOverflow if group.any?('A') && group.size >= 1000
+        raise Kafka::BufferOverflow if group.any?('BIG') && group.size >= 10
+      end
+      allow(Deimos.config.metrics).to receive(:increment)
+      batch = ['A'] * 450 + ['BIG'] * 550
+      producer.produce_messages(batch)
+
+      expect(phobos_producer).to have_received(:publish_list).with(batch)
+      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 100).exactly(4).times
+      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 50 + ['BIG'] * 50)
+      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 10).exactly(5).times
+      expect(phobos_producer).to have_received(:publish_list).with(['BIG'] * 1).exactly(550).times
 
+      expect(Deimos.config.metrics).to have_received(:increment).with('publish',
+                                                                      tags: %w(status:success topic:),
+                                                                      by: 100).exactly(4).times
+      expect(Deimos.config.metrics).to have_received(:increment).with('publish',
+                                                                      tags: %w(status:success topic:),
+                                                                      by: 10).exactly(5).times
+      expect(Deimos.config.metrics).to have_received(:increment).with('publish',
+                                                                      tags: %w(status:success topic:),
+                                                                      by: 1).exactly(550).times
     end
 
     describe '#compact_messages' do
@@ -289,6 +314,8 @@ each_db_config(Deimos::Utils::DbProducer) do
           message: "mess#{i}",
           partition_key: "key#{i}"
         )
+      end
+      (5..8).each do |i|
         Deimos::KafkaMessage.create!(
           id: i,
           topic: 'my-topic2',

data/spec/utils/inline_consumer_spec.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+describe Deimos::Utils::SeekListener do
+
+  describe '#start_listener' do
+    let(:consumer) { instance_double(Kafka::Consumer) }
+    let(:handler) { class_double(Deimos::Utils::MessageBankHandler) }
+
+    before(:each) do
+      allow(handler).to receive(:start)
+      allow(consumer).to receive(:subscribe)
+      allow_any_instance_of(Phobos::Listener).to receive(:create_kafka_consumer).and_return(consumer)
+      allow_any_instance_of(Kafka::Client).to receive(:last_offset_for).and_return(100)
+      stub_const('Deimos::Utils::SeekListener::MAX_SEEK_RETRIES', 2)
+    end
+
+    it 'should seek offset' do
+      allow(consumer).to receive(:seek)
+      expect(consumer).to receive(:seek).once
+      seek_listener = described_class.new({ handler: handler, group_id: 999, topic: 'test_topic' })
+      seek_listener.start_listener
+    end
+
+    it 'should retry on errors when seeking offset' do
+      allow(consumer).to receive(:seek).and_raise(StandardError)
+      expect(consumer).to receive(:seek).twice
+      seek_listener = described_class.new({ handler: handler, group_id: 999, topic: 'test_topic' })
+      seek_listener.start_listener
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.8.2.pre.beta2
+  version: 1.8.6
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-09-15 00:00:00.000000000 Z
+date: 2021-01-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf
@@ -280,30 +280,30 @@ dependencies:
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0.72'
+        version: 0.88.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0.72'
+        version: 0.88.0
 - !ruby/object:Gem::Dependency
   name: rubocop-rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '1.27'
+        version: 1.42.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '1.27'
+        version: 1.42.0
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -348,6 +348,7 @@ files:
 - docs/ARCHITECTURE.md
 - docs/CONFIGURATION.md
 - docs/DATABASE_BACKEND.md
+- docs/INTEGRATION_TESTS.md
 - docs/PULL_REQUEST_TEMPLATE.md
 - docs/UPGRADING.md
 - lib/deimos.rb
@@ -464,6 +465,7 @@ files:
 - spec/utils/db_poller_spec.rb
 - spec/utils/db_producer_spec.rb
 - spec/utils/deadlock_retry_spec.rb
+- spec/utils/inline_consumer_spec.rb
 - spec/utils/lag_reporter_spec.rb
 - spec/utils/platform_schema_validation_spec.rb
 - spec/utils/schema_controller_mixin_spec.rb
@@ -486,11 +488,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.0.9
 signing_key:
 specification_version: 4
 summary: Kafka libraries for Ruby.
@@ -547,6 +549,7 @@ test_files:
 - spec/utils/db_poller_spec.rb
 - spec/utils/db_producer_spec.rb
 - spec/utils/deadlock_retry_spec.rb
+- spec/utils/inline_consumer_spec.rb
 - spec/utils/lag_reporter_spec.rb
 - spec/utils/platform_schema_validation_spec.rb
 - spec/utils/schema_controller_mixin_spec.rb