deimos-ruby 1.6.3 → 1.8.1.pre.beta1

data/spec/utils/deadlock_retry_spec.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+RSpec.describe Deimos::Utils::DeadlockRetry do
+  include_context 'with widgets'
+
+  before(:each) do
+    allow(described_class).to receive(:sleep)
+  end
+
+  describe 'deadlock handling' do
+    let(:batch) { [{ key: 1, payload: { test_id: 'abc', some_int: 3 } }] }
+
+    it 'should retry deadlocks 3 times' do
+      # Should receive original attempt + 2 retries
+      expect(Widget).
+        to receive(:create).
+        and_raise(ActiveRecord::Deadlocked.new('Lock wait timeout exceeded')).
+        exactly(3).times
+
+      # After 3 tries, should let it bubble up
+      expect {
+        described_class.wrap do
+          Widget.create(test_id: 'abc')
+        end
+      }.to raise_error(ActiveRecord::Deadlocked)
+    end
+
+    it 'should stop retrying deadlocks after success' do
+      allow(Widget).
+        to receive(:create).
+        with(hash_including(test_id: 'first')).
+        and_call_original
+
+      # Fail on first attempt, succeed on second
+      expect(Widget).
+        to receive(:create).
+        with(hash_including(test_id: 'second')).
+        and_raise(ActiveRecord::Deadlocked.new('Deadlock found when trying to get lock')).
+        once.
+        ordered
+
+      expect(Widget).
+        to receive(:create).
+        with(hash_including(test_id: 'second')).
+        once.
+        ordered.
+        and_call_original
+
+      # Should not raise anything
+      described_class.wrap do
+        Widget.create(test_id: 'first')
+        Widget.create(test_id: 'second')
+      end
+
+      expect(Widget.all).to match_array([
+                                          have_attributes(test_id: 'first'),
+                                          have_attributes(test_id: 'second')
+                                        ])
+    end
+
+    it 'should not retry non-deadlock exceptions' do
+      expect(Widget).
+        to receive(:create).
+        and_raise(ActiveRecord::StatementInvalid.new('Oops!!')).
+        once
+
+      expect {
+        described_class.wrap do
+          Widget.create(test_id: 'abc')
+        end
+      }.to raise_error(ActiveRecord::StatementInvalid, 'Oops!!')
+    end
+  end
+end

data/spec/utils/lag_reporter_spec.rb

@@ -2,8 +2,11 @@
 
 describe Deimos::Utils::LagReporter do
 
+  let(:kafka_client) { instance_double(Kafka::Client) }
+  let(:partition1_tags) { %w(consumer_group:group1 partition:1 topic:my-topic) }
+  let(:partition2_tags) { %w(consumer_group:group1 partition:2 topic:my-topic) }
+
   before(:each) do
-    kafka_client = instance_double(Kafka::Client)
     allow(kafka_client).to receive(:last_offset_for).and_return(100)
     allow(Phobos).to receive(:create_kafka_client).and_return(kafka_client)
     Deimos.configure { |c| c.consumers.report_lag = true }
@@ -20,38 +23,22 @@ describe Deimos::Utils::LagReporter do
       'heartbeat.consumer.kafka',
       group_id: 'group1', topic_partitions: { 'my-topic': [1] }
     )
-
   end
 
   it 'should report lag' do
     expect(Deimos.config.metrics).to receive(:gauge).ordered.twice.
-      with('consumer_lag', 95,
-           tags: %w(
-             consumer_group:group1
-             partition:1
-             topic:my-topic
-           ))
+      with('consumer_lag', 95, tags: partition1_tags)
     expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
-      with('consumer_lag', 80,
-           tags: %w(
-             consumer_group:group1
-             partition:2
-             topic:my-topic
-           ))
+      with('consumer_lag', 80, tags: partition2_tags)
     expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
-      with('consumer_lag', 0,
-           tags: %w(
-             consumer_group:group1
-             partition:2
-             topic:my-topic
-           ))
+      with('consumer_lag', 0, tags: partition2_tags)
     ActiveSupport::Notifications.instrument(
       'seek.consumer.kafka',
       offset: 5, topic: 'my-topic', group_id: 'group1', partition: 1
     )
     ActiveSupport::Notifications.instrument(
       'start_process_message.consumer.kafka',
-      offset_lag: 80, topic: 'my-topic', group_id: 'group1', partition: 2
+      offset: 20, topic: 'my-topic', group_id: 'group1', partition: 2
     )
     ActiveSupport::Notifications.instrument(
       'heartbeat.consumer.kafka',
@@ -59,8 +46,28 @@ describe Deimos::Utils::LagReporter do
     )
     ActiveSupport::Notifications.instrument(
       'start_process_batch.consumer.kafka',
-      offset_lag: 0, topic: 'my-topic', group_id: 'group1', partition: 2
+      last_offset: 100, topic: 'my-topic', group_id: 'group1', partition: 2
+    )
+    ActiveSupport::Notifications.instrument(
+      'heartbeat.consumer.kafka',
+      group_id: 'group1', topic_partitions: { 'my-topic': [1, 2] }
+    )
+  end
+
+  it 'should update lag after heartbeat' do
+    expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
+      with('consumer_lag', 94, tags: partition2_tags)
+    expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
+      with('consumer_lag', 95, tags: partition2_tags)
+    ActiveSupport::Notifications.instrument(
+      'seek.consumer.kafka',
+      offset: 6, topic: 'my-topic', group_id: 'group1', partition: 2
+    )
+    ActiveSupport::Notifications.instrument(
+      'heartbeat.consumer.kafka',
+      group_id: 'group1', topic_partitions: { 'my-topic': [1, 2] }
     )
+    allow(kafka_client).to receive(:last_offset_for).and_return(101)
     ActiveSupport::Notifications.instrument(
       'heartbeat.consumer.kafka',
       group_id: 'group1', topic_partitions: { 'my-topic': [1, 2] }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.6.3
+  version: 1.8.1.pre.beta1
 platform: ruby
 authors:
 - Daniel Orner
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-05-04 00:00:00.000000000 Z
+date: 2020-07-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf
@@ -52,20 +52,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: sigurd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.1
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '6'
 - !ruby/object:Gem::Dependency
   name: activerecord-import
   requirement: !ruby/object:Gem::Requirement
@@ -95,19 +109,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.9'
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: database_cleaner
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '1.7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '1.7'
 - !ruby/object:Gem::Dependency
   name: ddtrace
   requirement: !ruby/object:Gem::Requirement
@@ -212,20 +226,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.2'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 5.2.4.2
+        version: '6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.2'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 5.2.4.2
+        version: '6'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -310,7 +318,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
-description: 
+description:
 email:
 - daniel.orner@wishabi.com
 executables:
@@ -337,10 +345,15 @@ files:
 - bin/deimos
 - deimos-ruby.gemspec
 - docker-compose.yml
+- docs/ARCHITECTURE.md
 - docs/CONFIGURATION.md
 - docs/DATABASE_BACKEND.md
 - docs/PULL_REQUEST_TEMPLATE.md
 - lib/deimos.rb
+- lib/deimos/active_record_consume/batch_consumption.rb
+- lib/deimos/active_record_consume/batch_slicer.rb
+- lib/deimos/active_record_consume/message_consumption.rb
+- lib/deimos/active_record_consume/schema_model_converter.rb
 - lib/deimos/active_record_consumer.rb
 - lib/deimos/active_record_producer.rb
 - lib/deimos/backends/base.rb
@@ -348,11 +361,12 @@ files:
 - lib/deimos/backends/kafka.rb
 - lib/deimos/backends/kafka_async.rb
 - lib/deimos/backends/test.rb
-- lib/deimos/base_consumer.rb
 - lib/deimos/batch_consumer.rb
 - lib/deimos/config/configurable.rb
 - lib/deimos/config/configuration.rb
 - lib/deimos/config/phobos_config.rb
+- lib/deimos/consume/batch_consumption.rb
+- lib/deimos/consume/message_consumption.rb
 - lib/deimos/consumer.rb
 - lib/deimos/instrumentation.rb
 - lib/deimos/kafka_message.rb
@@ -365,6 +379,7 @@ files:
 - lib/deimos/monkey_patches/phobos_cli.rb
 - lib/deimos/monkey_patches/phobos_producer.rb
 - lib/deimos/monkey_patches/ruby_kafka_heartbeat.rb
+- lib/deimos/poll_info.rb
 - lib/deimos/producer.rb
 - lib/deimos/railtie.rb
 - lib/deimos/schema_backends/avro_base.rb
@@ -379,17 +394,25 @@ files:
 - lib/deimos/tracing/datadog.rb
 - lib/deimos/tracing/mock.rb
 - lib/deimos/tracing/provider.rb
+- lib/deimos/utils/db_poller.rb
 - lib/deimos/utils/db_producer.rb
-- lib/deimos/utils/executor.rb
+- lib/deimos/utils/deadlock_retry.rb
 - lib/deimos/utils/inline_consumer.rb
 - lib/deimos/utils/lag_reporter.rb
-- lib/deimos/utils/platform_schema_validation.rb
-- lib/deimos/utils/signal_handler.rb
 - lib/deimos/version.rb
+- lib/generators/deimos/active_record/templates/migration.rb.tt
+- lib/generators/deimos/active_record/templates/model.rb.tt
+- lib/generators/deimos/active_record_generator.rb
 - lib/generators/deimos/db_backend/templates/migration
 - lib/generators/deimos/db_backend/templates/rails3_migration
 - lib/generators/deimos/db_backend_generator.rb
+- lib/generators/deimos/db_poller/templates/migration
+- lib/generators/deimos/db_poller/templates/rails3_migration
+- lib/generators/deimos/db_poller_generator.rb
 - lib/tasks/deimos.rake
+- spec/active_record_batch_consumer_spec.rb
+- spec/active_record_consume/batch_slicer_spec.rb
+- spec/active_record_consume/schema_model_converter_spec.rb
 - spec/active_record_consumer_spec.rb
 - spec/active_record_producer_spec.rb
 - spec/backends/base_spec.rb
@@ -401,10 +424,12 @@ files:
 - spec/config/configuration_spec.rb
 - spec/consumer_spec.rb
 - spec/deimos_spec.rb
+- spec/generators/active_record_generator_spec.rb
 - spec/handlers/my_batch_consumer.rb
 - spec/handlers/my_consumer.rb
 - spec/kafka_source_spec.rb
 - spec/kafka_topic_info_spec.rb
+- spec/message_spec.rb
 - spec/phobos.bad_db.yml
 - spec/phobos.yml
 - spec/producer_spec.rb
@@ -414,20 +439,23 @@ files:
 - spec/schema_backends/avro_schema_registry_spec.rb
 - spec/schema_backends/avro_validation_spec.rb
 - spec/schema_backends/base_spec.rb
+- spec/schemas/com/my-namespace/Generated.avsc
 - spec/schemas/com/my-namespace/MySchema-key.avsc
 - spec/schemas/com/my-namespace/MySchema.avsc
+- spec/schemas/com/my-namespace/MySchemaCompound-key.avsc
 - spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc
 - spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc
 - spec/schemas/com/my-namespace/MySchemaWithId.avsc
 - spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc
+- spec/schemas/com/my-namespace/Wibble.avsc
 - spec/schemas/com/my-namespace/Widget.avsc
 - spec/schemas/com/my-namespace/WidgetTheSecond.avsc
 - spec/spec_helper.rb
+- spec/utils/db_poller_spec.rb
 - spec/utils/db_producer_spec.rb
-- spec/utils/executor_spec.rb
+- spec/utils/deadlock_retry_spec.rb
 - spec/utils/lag_reporter_spec.rb
 - spec/utils/platform_schema_validation_spec.rb
-- spec/utils/signal_handler_spec.rb
 - support/deimos-solo.png
 - support/deimos-with-name-next.png
 - support/deimos-with-name.png
@@ -436,7 +464,7 @@ homepage: ''
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -447,15 +475,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.1.3
+signing_key:
 specification_version: 4
 summary: Kafka libraries for Ruby.
 test_files:
+- spec/active_record_batch_consumer_spec.rb
+- spec/active_record_consume/batch_slicer_spec.rb
+- spec/active_record_consume/schema_model_converter_spec.rb
 - spec/active_record_consumer_spec.rb
 - spec/active_record_producer_spec.rb
 - spec/backends/base_spec.rb
@@ -467,10 +498,12 @@ test_files:
 - spec/config/configuration_spec.rb
 - spec/consumer_spec.rb
 - spec/deimos_spec.rb
+- spec/generators/active_record_generator_spec.rb
 - spec/handlers/my_batch_consumer.rb
 - spec/handlers/my_consumer.rb
 - spec/kafka_source_spec.rb
 - spec/kafka_topic_info_spec.rb
+- spec/message_spec.rb
 - spec/phobos.bad_db.yml
 - spec/phobos.yml
 - spec/producer_spec.rb
@@ -480,17 +513,20 @@ test_files:
 - spec/schema_backends/avro_schema_registry_spec.rb
 - spec/schema_backends/avro_validation_spec.rb
 - spec/schema_backends/base_spec.rb
+- spec/schemas/com/my-namespace/Generated.avsc
 - spec/schemas/com/my-namespace/MySchema-key.avsc
 - spec/schemas/com/my-namespace/MySchema.avsc
+- spec/schemas/com/my-namespace/MySchemaCompound-key.avsc
 - spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc
 - spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc
 - spec/schemas/com/my-namespace/MySchemaWithId.avsc
 - spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc
+- spec/schemas/com/my-namespace/Wibble.avsc
 - spec/schemas/com/my-namespace/Widget.avsc
 - spec/schemas/com/my-namespace/WidgetTheSecond.avsc
 - spec/spec_helper.rb
+- spec/utils/db_poller_spec.rb
 - spec/utils/db_producer_spec.rb
-- spec/utils/executor_spec.rb
+- spec/utils/deadlock_retry_spec.rb
 - spec/utils/lag_reporter_spec.rb
 - spec/utils/platform_schema_validation_spec.rb
-- spec/utils/signal_handler_spec.rb

data/lib/deimos/base_consumer.rb

@@ -1,104 +0,0 @@
-# frozen_string_literal: true
-
-module Deimos
-  # Shared methods for Kafka Consumers
-  class BaseConsumer
-    include SharedConfig
-
-    class << self
-      # @return [Deimos::SchemaBackends::Base]
-      def decoder
-        @decoder ||= Deimos.schema_backend(schema: config[:schema],
-                                           namespace: config[:namespace])
-      end
-
-      # @return [Deimos::SchemaBackends::Base]
-      def key_decoder
-        @key_decoder ||= Deimos.schema_backend(schema: config[:key_schema],
-                                               namespace: config[:namespace])
-      end
-    end
-
-    # Helper method to decode an encoded key.
-    # @param key [String]
-    # @return [Object] the decoded key.
-    def decode_key(key)
-      return nil if key.nil?
-
-      config = self.class.config
-      unless config[:key_configured]
-        raise 'No key config given - if you are not decoding keys, please use '\
-          '`key_config plain: true`'
-      end
-
-      if config[:key_field]
-        self.class.decoder.decode_key(key, config[:key_field])
-      elsif config[:key_schema]
-        self.class.key_decoder.decode(key, schema: config[:key_schema])
-      else # no encoding
-        key
-      end
-    end
-
-  protected
-
-    # @param payload [Hash|String]
-    # @param metadata [Hash]
-    def _with_error_span(payload, metadata)
-      @span = Deimos.config.tracer&.start(
-        'deimos-consumer',
-        resource: self.class.name.gsub('::', '-')
-      )
-      yield
-    rescue StandardError => e
-      _handle_error(e, payload, metadata)
-    ensure
-      Deimos.config.tracer&.finish(@span)
-    end
-
-    def _report_time_delayed(payload, metadata)
-      return if payload.nil? || payload['timestamp'].blank?
-
-      begin
-        time_delayed = Time.now.in_time_zone - payload['timestamp'].to_datetime
-      rescue ArgumentError
-        Deimos.config.logger.info(
-          message: "Error parsing timestamp! #{payload['timestamp']}"
-        )
-        return
-      end
-      Deimos.config.metrics&.histogram('handler', time_delayed, tags: %W(
-                                         time:time_delayed
-                                         topic:#{metadata[:topic]}
-                                       ))
-    end
-
-    # Overrideable method to determine if a given error should be considered
-    # "fatal" and always be reraised.
-    # @param error [Exception]
-    # @param payload [Hash]
-    # @param metadata [Hash]
-    # @return [Boolean]
-    def fatal_error?(_error, _payload, _metadata)
-      false
-    end
-
-    # @param exception [Exception]
-    # @param payload [Hash]
-    # @param metadata [Hash]
-    def _handle_error(exception, payload, metadata)
-      Deimos.config.tracer&.set_error(@span, exception)
-
-      raise if Deimos.config.consumers.reraise_errors ||
-               Deimos.config.consumers.fatal_error&.call(exception, payload, metadata) ||
-               fatal_error?(exception, payload, metadata)
-    end
-
-    # @param _time_taken [Float]
-    # @param _payload [Hash]
-    # @param _metadata [Hash]
-    def _handle_success(_time_taken, _payload, _metadata)
-      raise NotImplementedError
-    end
-  end
-end