deimos-ruby 1.8.3 → 1.8.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2badd671866a8fbde743e03acbc2da623279ac08abd5ed983ae9a15a0bef415d
-  data.tar.gz: e4b3cd2e80b13fc00ff54ebc85b59ab2e1daad052ab170dafd9a75f57eb80ce3
+  metadata.gz: 497160ac15346d0acb471818c0f02e10f745761ba3777a70039b1a762e8f4edb
+  data.tar.gz: d85717deea2ac68d6c995d1dce1c42187d4fe5585b577c4468cafff7f0f71097
 SHA512:
-  metadata.gz: 98b41fa7354e624a4538ac5006a091e1a144a2caaa57e2ef77c214e350cd8dcbf2b126c01a3663365d28bbed19baa7525e3c52390cc60f6378537ad8fe6bee12
-  data.tar.gz: 208ba417a35c7042155383d5c0eb9d9e35be95c0459cf2e58cdc24168c8fdf30af16434fcc66c612d197aab38279d790907f146e19b6a36157cde008c5aa60a3
+  metadata.gz: 441577e56812dbc9d0dbbf22b43cb76e390c792165f7d21febcdf17a309fe85ec57bd9ddaad0929ddd3d49dd602a94601fc2660b124419e5943460e2fab94aa5
+  data.tar.gz: 2fdd3a94d91130c2430d189165504bdf437907fa1a9a459841bd4e260a2ff906469fb0f6989a8faf6c140cc133d22639af91a89549c85a9aeef2145636aa7b58

data/CHANGELOG.md

@@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+## 1.8.4 - 2020-12-02
+
+### Features :star:
+- Add overridable "process_message?" method to ActiveRecordConsumer to allow for skipping of saving/updating records
+
+### Fixes :wrench:
+
+- Do not apply type coercion to `timestamp-millis` and `timestamp-micros` logical types (fixes [#97](https://github.com/flipp-oss/deimos/issues/97))
+
 ## 1.8.3 - 2020-11-18
 
 ### Fixes :wrench:
@@ -15,6 +24,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - KafkaSource crashing on bulk-imports if import hooks are disabled
   (fixes [#73](https://github.com/flipp-oss/deimos/issues/73))
 - #96 Use string-safe encoding for partition keys
+- Retry on offset seek failures in inline consumer
+  (fixes [#5](Inline consumer should use retries when seeking))
 
 ## 1.8.2 - 2020-09-25
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    deimos-ruby (1.8.2)
+    deimos-ruby (1.8.4)
       avro_turf (~> 0.11)
       phobos (~> 1.9)
       ruby-kafka (~> 0.7)

data/README.md

@@ -626,6 +626,15 @@ class MyConsumer < Deimos::ActiveRecordConsumer
   def record_key(payload)
     super
   end
+
+  # Optional override, returns true by default.
+  # When this method returns true, a record corresponding to the message
+  # is created/updated.
+  # When this method returns false, message processing is skipped and a
+  # corresponding record will NOT be created/updated.
+  def process_message?(payload)
+    super
+  end
 end
 ```
 

data/docs/PULL_REQUEST_TEMPLATE.md

@@ -28,6 +28,7 @@ Please describe the tests that you ran to verify your changes. Provide instructi
 - [ ] I have performed a self-review of my own code
 - [ ] I have commented my code, particularly in hard-to-understand areas
 - [ ] I have made corresponding changes to the documentation
+- [ ] I have added a line in the CHANGELOG describing this change, under the UNRELEASED heading
 - [ ] My changes generate no new warnings
 - [ ] I have added tests that prove my fix is effective or that my feature works
 - [ ] New and existing unit tests pass locally with my changes

data/lib/deimos/active_record_consume/message_consumption.rb

@@ -26,6 +26,15 @@ module Deimos
 
       # :nodoc:
       def consume(payload, metadata)
+        unless self.process_message?(payload)
+          Deimos.config.logger.debug(
+            message: 'Skipping processing of message',
+            payload: payload,
+            metadata: metadata
+          )
+          return
+        end
+
         key = metadata.with_indifferent_access[:key]
         klass = self.class.config[:record_class]
         record = fetch_record(klass, (payload || {}).with_indifferent_access, key)

data/lib/deimos/active_record_consumer.rb

@@ -55,5 +55,13 @@ module Deimos
     def record_attributes(payload, _key=nil)
       @converter.convert(payload)
     end
+
+    # Override this message to conditionally save records
+    # @param payload [Hash] The kafka message as a hash
+    # @return [Boolean] if true, record is created/update.
+    #   If false, record processing is skipped but message offset is still committed.
+    def process_message?(_payload)
+      true
+    end
   end
 end

data/lib/deimos/schema_backends/avro_schema_coercer.rb

@@ -44,9 +44,11 @@ module Deimos
 
       case field_type
       when :int, :long
-        if val.is_a?(Integer) ||
-           _is_integer_string?(val) ||
-           int_classes.any? { |klass| val.is_a?(klass) }
+        if %w(timestamp-millis timestamp-micros).include?(type.logical_type)
+          val
+        elsif val.is_a?(Integer) ||
+              _is_integer_string?(val) ||
+              int_classes.any? { |klass| val.is_a?(klass) }
           val.to_i
         else
           val # this will fail

data/lib/deimos/utils/inline_consumer.rb

@@ -6,6 +6,7 @@ module Deimos
   module Utils
     # Listener that can seek to get the last X messages in a topic.
     class SeekListener < Phobos::Listener
+      MAX_SEEK_RETRIES = 3
       attr_accessor :num_messages
 
       # :nodoc:
@@ -13,8 +14,10 @@ module Deimos
         @num_messages ||= 10
         @consumer = create_kafka_consumer
         @consumer.subscribe(topic, @subscribe_opts)
+        attempt = 0
 
         begin
+          attempt += 1
           last_offset = @kafka_client.last_offset_for(topic, 0)
           offset = last_offset - num_messages
           if offset.positive?
@@ -22,7 +25,11 @@ module Deimos
             @consumer.seek(topic, 0, offset)
           end
         rescue StandardError => e
-          "Could not seek to offset: #{e.message}"
+          if attempt < MAX_SEEK_RETRIES
+            sleep(1.seconds * attempt)
+            retry
+          end
+          log_error("Could not seek to offset: #{e.message} after #{MAX_SEEK_RETRIES} retries", listener_metadata)
         end
 
         instrument('listener.start_handler', listener_metadata) do
@@ -50,7 +57,6 @@ module Deimos
 
       # :nodoc:
       def consume(payload, metadata)
-        puts "Got #{payload}"
         self.class.total_messages << {
           key: metadata[:key],
           payload: payload

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.8.3'
+  VERSION = '1.8.4'
 end

data/spec/active_record_consumer_spec.rb

@@ -137,5 +137,18 @@ module ActiveRecordConsumerTest
       expect(Widget.find_by_test_id('id1').some_int).to eq(3)
       expect(Widget.find_by_test_id('id2').some_int).to eq(4)
     end
+
+    it 'should not create record of process_message returns false' do
+      MyConsumer.any_instance.stub(:process_message?).and_return(false)
+      expect(Widget.count).to eq(0)
+      test_consume_message(MyConsumer, {
+                             test_id: 'abc',
+                             some_int: 3,
+                             updated_at: 1.day.ago.to_i,
+                             some_datetime_int: Time.zone.now.to_i,
+                             timestamp: 2.minutes.ago.to_s
+                           }, { call_original: true, key: 5 })
+      expect(Widget.count).to eq(0)
+    end
   end
 end

data/spec/schema_backends/avro_base_shared.rb

@@ -42,6 +42,20 @@ RSpec.shared_examples_for('an Avro backend') do
         {
           'name' => 'union-int-field',
           'type' => %w(null int)
+        },
+        {
+          'name' => 'timestamp-millis-field',
+          'type' => {
+            'type' => 'long',
+            'logicalType' => 'timestamp-millis'
+          }
+        },
+        {
+          'name' => 'timestamp-micros-field',
+          'type' => {
+            'type' => 'long',
+            'logicalType' => 'timestamp-micros'
+          }
         }
       ]
     }
@@ -95,7 +109,9 @@ RSpec.shared_examples_for('an Avro backend') do
         'string-field' => 'hi mom',
         'boolean-field' => true,
         'union-field' => nil,
-        'union-int-field' => nil
+        'union-int-field' => nil,
+        'timestamp-millis-field' => Time.utc(2020, 11, 12, 13, 14, 15, 909_090),
+        'timestamp-micros-field' => Time.utc(2020, 11, 12, 13, 14, 15, 909_090)
       }
     end
 
@@ -169,6 +185,15 @@ RSpec.shared_examples_for('an Avro backend') do
       expect(result['union-field']).to eq('itsme')
     end
 
+    it 'should not convert timestamp-millis' do
+      result = backend.coerce(payload)
+      expect(result['timestamp-millis-field']).to eq(Time.utc(2020, 11, 12, 13, 14, 15, 909_090))
+    end
+
+    it 'should not convert timestamp-micros' do
+      result = backend.coerce(payload)
+      expect(result['timestamp-micros-field']).to eq(Time.utc(2020, 11, 12, 13, 14, 15, 909_090))
+    end
   end
 
 end

data/spec/utils/inline_consumer_spec.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+describe Deimos::Utils::SeekListener do
+
+  describe '#start_listener' do
+    let(:consumer) { instance_double(Kafka::Consumer) }
+    let(:handler) { class_double(Deimos::Utils::MessageBankHandler) }
+
+    before(:each) do
+      allow(handler).to receive(:start)
+      allow(consumer).to receive(:subscribe)
+      allow_any_instance_of(Phobos::Listener).to receive(:create_kafka_consumer).and_return(consumer)
+      allow_any_instance_of(Kafka::Client).to receive(:last_offset_for).and_return(100)
+      stub_const('Deimos::Utils::SeekListener::MAX_SEEK_RETRIES', 2)
+    end
+
+    it 'should seek offset' do
+      allow(consumer).to receive(:seek)
+      expect(consumer).to receive(:seek).once
+      seek_listener = described_class.new({ handler: handler, group_id: 999, topic: 'test_topic' })
+      seek_listener.start_listener
+    end
+
+    it 'should retry on errors when seeking offset' do
+      allow(consumer).to receive(:seek).and_raise(StandardError)
+      expect(consumer).to receive(:seek).twice
+      seek_listener = described_class.new({ handler: handler, group_id: 999, topic: 'test_topic' })
+      seek_listener.start_listener
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.8.3
+  version: 1.8.4
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-11-18 00:00:00.000000000 Z
+date: 2020-12-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf
@@ -465,6 +465,7 @@ files:
 - spec/utils/db_poller_spec.rb
 - spec/utils/db_producer_spec.rb
 - spec/utils/deadlock_retry_spec.rb
+- spec/utils/inline_consumer_spec.rb
 - spec/utils/lag_reporter_spec.rb
 - spec/utils/platform_schema_validation_spec.rb
 - spec/utils/schema_controller_mixin_spec.rb
@@ -548,6 +549,7 @@ test_files:
 - spec/utils/db_poller_spec.rb
 - spec/utils/db_producer_spec.rb
 - spec/utils/deadlock_retry_spec.rb
+- spec/utils/inline_consumer_spec.rb
 - spec/utils/lag_reporter_spec.rb
 - spec/utils/platform_schema_validation_spec.rb
 - spec/utils/schema_controller_mixin_spec.rb