deimos-ruby 1.6.1 → 1.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0226368aa5a493ec167e0cd4b0d03eeb8ac9e3efe8c2518fbe869275f6b7b10b'
-  data.tar.gz: 94afab34fb599d3cc0a5336a0af1ab8458621c8655c22de6882d1b87a645d8bf
+  metadata.gz: 53aab4ddcc19808cf7149ed226de9b7ddda07d12ed4237c157cce75780379c6c
+  data.tar.gz: 74ebf879894b88fc4ce5082d4f8b82f8f79a0ef581bf368f25423a755d71d635
 SHA512:
-  metadata.gz: ee0516157c9fedd3e0ad9ff5dab95afbd9d3eee0975c899ede0f8104c1c9b5e992117221a8e42772f397abc3b8e70c755a90423a8d3daf93fd949751b3095f38
-  data.tar.gz: e42ad9fca0314bfa6cf5f764f7369a22a28d60c4d395b219df2d0c1ad1bb0c2d45a37d9e5eeb76984dbd7d122f211f4f8bbf8a5d0198108346bc8c16df795266
+  metadata.gz: 4c0fa67f0b958ddc3083c278c5c195f896b134f822168df245e906e89f5f5414f063175e2ed1a444891a210d28e7a0d96836806b6ba2efd6f55bf46d929450c1
+  data.tar.gz: 9fc31899551ec75dc0ca1e7ac02cc3f4ed429ec8eb25f42057024d2dc9cde5bef70eabafcf8276e7a8a4ae2b7a88b4efff5efe047715612bd5f5480ce1f52576

data/CHANGELOG.md

@@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+## 1.6.2 - 2020-05-04
+### Fixes :wrench:
+- When saving records via `ActiveRecordConsumer`, update `updated_at` to today's time
+  even if nothing else was saved.
+- When logging payloads and metadata, decode them first.
+- Fixes bug in `KafkaSource` that crashes when importing a mix of existing and new records with the `:on_duplicate_key_update` option.
+
 ## [1.6.1] - 2020-04-20
 ### Fixes :wrench:
 - Re-consuming a message after crashing would try to re-decode message keys.

data/Gemfile.lock

@@ -9,45 +9,45 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (5.2.4)
-      actionpack (= 5.2.4)
+    actioncable (5.2.4.2)
+      actionpack (= 5.2.4.2)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailer (5.2.4)
-      actionpack (= 5.2.4)
-      actionview (= 5.2.4)
-      activejob (= 5.2.4)
+    actionmailer (5.2.4.2)
+      actionpack (= 5.2.4.2)
+      actionview (= 5.2.4.2)
+      activejob (= 5.2.4.2)
       mail (~> 2.5, >= 2.5.4)
       rails-dom-testing (~> 2.0)
-    actionpack (5.2.4)
-      actionview (= 5.2.4)
-      activesupport (= 5.2.4)
-      rack (~> 2.0)
+    actionpack (5.2.4.2)
+      actionview (= 5.2.4.2)
+      activesupport (= 5.2.4.2)
+      rack (~> 2.0, >= 2.0.8)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.0.2)
-    actionview (5.2.4)
-      activesupport (= 5.2.4)
+    actionview (5.2.4.2)
+      activesupport (= 5.2.4.2)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.0.3)
-    activejob (5.2.4)
-      activesupport (= 5.2.4)
+    activejob (5.2.4.2)
+      activesupport (= 5.2.4.2)
       globalid (>= 0.3.6)
-    activemodel (5.2.4)
-      activesupport (= 5.2.4)
-    activerecord (5.2.4)
-      activemodel (= 5.2.4)
-      activesupport (= 5.2.4)
+    activemodel (5.2.4.2)
+      activesupport (= 5.2.4.2)
+    activerecord (5.2.4.2)
+      activemodel (= 5.2.4.2)
+      activesupport (= 5.2.4.2)
       arel (>= 9.0)
     activerecord-import (1.0.3)
       activerecord (>= 3.2)
-    activestorage (5.2.4)
-      actionpack (= 5.2.4)
-      activerecord (= 5.2.4)
+    activestorage (5.2.4.2)
+      actionpack (= 5.2.4.2)
+      activerecord (= 5.2.4.2)
       marcel (~> 0.3.1)
-    activesupport (5.2.4)
+    activesupport (5.2.4.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
@@ -64,7 +64,7 @@ GEM
     concurrent-ruby (1.1.5)
     concurrent-ruby-ext (1.1.5)
       concurrent-ruby (= 1.1.5)
-    crass (1.0.5)
+    crass (1.0.6)
     ddtrace (0.30.0)
       msgpack
     diff-lcs (1.3)
@@ -94,7 +94,7 @@ GEM
     guard-rubocop (1.3.0)
       guard (~> 2.0)
       rubocop (~> 0.20)
-    i18n (1.7.0)
+    i18n (1.8.2)
       concurrent-ruby (~> 1.0)
     jaro_winkler (1.5.4)
     listen (3.2.1)
@@ -104,7 +104,7 @@ GEM
     logging (2.2.2)
       little-plugger (~> 1.1)
       multi_json (~> 1.10)
-    loofah (2.4.0)
+    loofah (2.5.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     lumberjack (1.0.13)
@@ -113,16 +113,16 @@ GEM
     marcel (0.3.3)
       mimemagic (~> 0.3.2)
     method_source (0.9.2)
-    mimemagic (0.3.3)
+    mimemagic (0.3.4)
     mini_mime (1.0.2)
     mini_portile2 (2.4.0)
-    minitest (5.13.0)
+    minitest (5.14.0)
     msgpack (1.3.1)
     multi_json (1.14.1)
     mysql2 (0.5.3)
     nenv (0.3.0)
     nio4r (2.5.2)
-    nokogiri (1.10.8)
+    nokogiri (1.10.9)
       mini_portile2 (~> 2.4.0)
     notiffany (0.1.3)
       nenv (~> 0.1)
@@ -142,30 +142,30 @@ GEM
     pry (0.12.2)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
-    rack (2.0.8)
+    rack (2.2.2)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
-    rails (5.2.4)
-      actioncable (= 5.2.4)
-      actionmailer (= 5.2.4)
-      actionpack (= 5.2.4)
-      actionview (= 5.2.4)
-      activejob (= 5.2.4)
-      activemodel (= 5.2.4)
-      activerecord (= 5.2.4)
-      activestorage (= 5.2.4)
-      activesupport (= 5.2.4)
+    rails (5.2.4.2)
+      actioncable (= 5.2.4.2)
+      actionmailer (= 5.2.4.2)
+      actionpack (= 5.2.4.2)
+      actionview (= 5.2.4.2)
+      activejob (= 5.2.4.2)
+      activemodel (= 5.2.4.2)
+      activerecord (= 5.2.4.2)
+      activestorage (= 5.2.4.2)
+      activesupport (= 5.2.4.2)
       bundler (>= 1.3.0)
-      railties (= 5.2.4)
+      railties (= 5.2.4.2)
       sprockets-rails (>= 2.0.0)
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
     rails-html-sanitizer (1.3.0)
       loofah (~> 2.3)
-    railties (5.2.4)
-      actionpack (= 5.2.4)
-      activesupport (= 5.2.4)
+    railties (5.2.4.2)
+      actionpack (= 5.2.4.2)
+      activesupport (= 5.2.4.2)
       method_source
       rake (>= 0.8.7)
       thor (>= 0.19.0, < 2.0)
@@ -212,7 +212,7 @@ GEM
     sqlite3 (1.4.1)
     thor (1.0.1)
     thread_safe (0.3.6)
-    tzinfo (1.2.5)
+    tzinfo (1.2.7)
       thread_safe (~> 0.1)
     unicode-display_width (1.6.0)
     websocket-driver (0.7.1)
@@ -235,7 +235,7 @@ DEPENDENCIES
   guard-rubocop (~> 1)
   mysql2 (~> 0.5)
   pg (~> 1.1)
-  rails (~> 5.2)
+  rails (~> 5.2, >= 5.2.4.2)
   rake (~> 13)
   rspec (~> 3)
   rspec_junit_formatter (~> 0.3)

data/deimos-ruby.gemspec

@@ -33,7 +33,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency('guard-rubocop', '~> 1')
   spec.add_development_dependency('mysql2', '~> 0.5')
   spec.add_development_dependency('pg', '~> 1.1')
-  spec.add_development_dependency('rails', '~> 5.2')
+  spec.add_development_dependency('rails', '~> 5.2', '>= 5.2.4.2')
   spec.add_development_dependency('rake', '~> 13')
   spec.add_development_dependency('rspec', '~> 3')
   spec.add_development_dependency('rspec_junit_formatter', '~>0.3')

data/lib/deimos/active_record_consumer.rb

@@ -51,7 +51,7 @@ module Deimos
         record.send("#{k}=", v)
       end
       record.created_at ||= Time.zone.now if record.respond_to?(:created_at)
-      record.updated_at ||= Time.zone.now if record.respond_to?(:updated_at)
+      record.updated_at = Time.zone.now if record.respond_to?(:updated_at)
       record.save!
     end
 

data/lib/deimos/consumer.rb

@@ -15,12 +15,12 @@ module Deimos
 
     # :nodoc:
     def around_consume(payload, metadata)
-      _received_message(payload, metadata)
       benchmark = Benchmark.measure do
         _with_error_span(payload, metadata) do
           new_metadata = metadata.dup
           new_metadata[:key] = decode_key(metadata[:key]) if self.class.config[:key_configured]
           decoded_payload = payload ? self.class.decoder.decode(payload) : nil
+          _received_message(payload, metadata)
           yield decoded_payload, new_metadata
         end
       end

data/lib/deimos/kafka_source.rb

@@ -93,31 +93,37 @@ module Deimos
 
         # This will contain an array of hashes, where each hash is the actual
         # attribute hash that created the object.
-        ids = if results.is_a?(Array)
-                results[1]
-              elsif results.respond_to?(:ids)
-                results.ids
-              else
-                []
-              end
-        if ids.blank?
-          # re-fill IDs based on what was just entered into the DB.
-          if self.connection.adapter_name.downcase =~ /sqlite/
-            last_id = self.connection.select_value('select last_insert_rowid()')
-            ids = ((last_id - array_of_attributes.size + 1)..last_id).to_a
-          else # mysql
-            last_id = self.connection.select_value('select LAST_INSERT_ID()')
-            ids = (last_id..(last_id + array_of_attributes.size)).to_a
-          end
-        end
         array_of_hashes = []
-        array_of_attributes.each_with_index do |array, i|
-          hash = column_names.zip(array).to_h.with_indifferent_access
-          hash[self.primary_key] = ids[i] if hash[self.primary_key].blank?
-          array_of_hashes << hash
+        array_of_attributes.each do |array|
+          array_of_hashes << column_names.zip(array).to_h.with_indifferent_access
+        end
+        hashes_with_id, hashes_without_id = array_of_hashes.partition { |arr| arr[:id].present? }
+
+        self.kafka_producers.each { |p| p.send_events(hashes_with_id) }
+
+        if hashes_without_id.any?
+          if options[:on_duplicate_key_update].present? &&
+             options[:on_duplicate_key_update] != [:updated_at]
+            unique_columns = column_names.map(&:to_s) -
+                             options[:on_duplicate_key_update].map(&:to_s) - %w(id created_at)
+            records = hashes_without_id.map do |hash|
+              self.where(unique_columns.map { |c| [c, hash[c]] }.to_h).first
+            end
+            self.kafka_producers.each { |p| p.send_events(records) }
+          else
+            # re-fill IDs based on what was just entered into the DB.
+            last_id = if self.connection.adapter_name.downcase =~ /sqlite/
+                        self.connection.select_value('select last_insert_rowid()') -
+                          hashes_without_id.size + 1
+                      else # mysql
+                        self.connection.select_value('select LAST_INSERT_ID()')
+                      end
+            hashes_without_id.each_with_index do |attrs, i|
+              attrs[:id] = last_id + i
+            end
+            self.kafka_producers.each { |p| p.send_events(hashes_without_id) }
+          end
         end
-
-        self.kafka_producers.each { |p| p.send_events(array_of_hashes) }
         results
       end
     end

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.6.1'
+  VERSION = '1.6.2'
 end

data/spec/active_record_consumer_spec.rb

@@ -22,6 +22,10 @@ module ActiveRecordProducerTest
       Widget.reset_column_information
     end
 
+    before(:each) do
+      Widget.delete_all
+    end
+
     after(:all) do
       ActiveRecord::Base.connection.drop_table(:widgets)
     end
@@ -101,6 +105,21 @@ module ActiveRecordProducerTest
 
     end
 
+    it 'should update only updated_at' do
+      travel_to Time.local(2020, 5, 5, 5, 5, 5)
+      widget1 = Widget.create!(test_id: 'id1', some_int: 3)
+      expect(widget1.updated_at.in_time_zone).to eq(Time.local(2020, 5, 5, 5, 5, 5))
+
+      travel 1.day
+      test_consume_message(MyCustomFetchConsumer, {
+                             test_id: 'id1',
+                             some_int: 3
+                           }, { call_original: true })
+      expect(widget1.reload.updated_at.in_time_zone).
+        to eq(Time.local(2020, 5, 6, 5, 5, 5))
+      travel_back
+    end
+
     it 'should find widgets by custom logic' do
       widget1 = Widget.create!(test_id: 'id1')
       expect(widget1.some_int).to be_nil

data/spec/kafka_source_spec.rb

@@ -126,6 +126,59 @@ module KafkaSourceSpec
                                       }, widgets[2].id)
     end
 
+    it 'should send events on import with on_duplicate_key_update and existing records' do
+      widget1 = Widget.create(widget_id: 1, name: 'Widget 1')
+      widget2 = Widget.create(widget_id: 2, name: 'Widget 2')
+      widget1.name = 'New Widget 1'
+      widget2.name = 'New Widget 2'
+      Widget.import([widget1, widget2], on_duplicate_key_update: %i(widget_id name))
+
+      expect('my-topic').to have_sent({
+                                        widget_id: 1,
+                                        name: 'New Widget 1',
+                                        id: widget1.id,
+                                        created_at: anything,
+                                        updated_at: anything
+                                      }, widget1.id)
+      expect('my-topic').to have_sent({
+                                        widget_id: 2,
+                                        name: 'New Widget 2',
+                                        id: widget2.id,
+                                        created_at: anything,
+                                        updated_at: anything
+                                      }, widget2.id)
+    end
+
+    it 'should not fail when mixing existing and new records for import :on_duplicate_key_update' do
+      widget1 = Widget.create(widget_id: 1, name: 'Widget 1')
+      expect('my-topic').to have_sent({
+                                        widget_id: 1,
+                                        name: 'Widget 1',
+                                        id: widget1.id,
+                                        created_at: anything,
+                                        updated_at: anything
+                                      }, widget1.id)
+
+      widget2 = Widget.new(widget_id: 2, name: 'Widget 2')
+      widget1.name = 'New Widget 1'
+      Widget.import([widget1, widget2], on_duplicate_key_update: %i(widget_id))
+      widgets = Widget.all
+      expect('my-topic').to have_sent({
+                                        widget_id: 1,
+                                        name: 'New Widget 1',
+                                        id: widgets[0].id,
+                                        created_at: anything,
+                                        updated_at: anything
+                                      }, widgets[0].id)
+      expect('my-topic').to have_sent({
+                                        widget_id: 2,
+                                        name: 'Widget 2',
+                                        id: widgets[1].id,
+                                        created_at: anything,
+                                        updated_at: anything
+                                      }, widgets[1].id)
+    end
+
     it 'should send events even if the save fails' do
       widget = Widget.create!(widget_id: 1, name: 'widget')
       expect('my-topic').to have_sent({

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.6.1
+  version: 1.6.2
 platform: ruby
 authors:
 - Daniel Orner
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-20 00:00:00.000000000 Z
+date: 2020-05-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf
@@ -213,6 +213,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.2.4.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -220,6 +223,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.2.4.2
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement