deimos-ruby 1.7.0.pre.beta1 → 1.8.0.pre.beta1

data/spec/handlers/my_batch_consumer.rb

@@ -1,5 +1,10 @@
 # frozen_string_literal: true
 
 module ConsumerTest
-  class MyBatchConsumer < Deimos::BatchConsumer; end
+  # Mock consumer
+  class MyBatchConsumer < Deimos::Consumer
+    # :no-doc:
+    def consume_batch
+    end
+  end
 end

data/spec/handlers/my_consumer.rb

@@ -1,5 +1,10 @@
 # frozen_string_literal: true
 
 module ConsumerTest
-  class MyConsumer < Deimos::Consumer; end
+  # Mock consumer
+  class MyConsumer < Deimos::Consumer
+    # :no-doc:
+    def consume
+    end
+  end
 end

data/spec/message_spec.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+RSpec.describe(Deimos::Message) do
+  it 'should detect tombstones' do
+    expect(described_class.new(nil, nil, key: 'key1')).
+      to be_tombstone
+    expect(described_class.new({ v: 'val1' }, nil, key: 'key1')).
+      not_to be_tombstone
+    expect(described_class.new({ v: '' }, nil, key: 'key1')).
+      not_to be_tombstone
+    expect(described_class.new({ v: 'val1' }, nil, key: nil)).
+      not_to be_tombstone
+  end
+
+  it 'can support complex keys/values' do
+    expect { described_class.new({ a: 1, b: 2 }, nil, key: { c: 3, d: 4 }) }.
+      not_to raise_exception
+  end
+end

data/spec/schemas/com/my-namespace/MySchemaCompound-key.avsc

@@ -0,0 +1,18 @@
+{
+  "namespace": "com.my-namespace",
+  "name": "MySchemaCompound-key",
+  "type": "record",
+  "doc": "Test schema",
+  "fields": [
+    {
+      "name": "part_one",
+      "type": "string",
+      "doc": "test string one"
+    },
+    {
+      "name": "part_two",
+      "type": "string",
+      "doc": "test string two"
+    }
+  ]
+}

data/spec/schemas/com/my-namespace/Wibble.avsc

@@ -0,0 +1,43 @@
+{
+  "namespace": "com.my-namespace",
+  "name": "Wibble",
+  "type": "record",
+  "fields": [
+    {
+      "name": "id",
+      "type": "long"
+    },
+    {
+      "name": "wibble_id",
+      "type": "long"
+    },
+    {
+      "name": "name",
+      "type": "string"
+    },
+    {
+      "name": "floop",
+      "type": "string"
+    },
+    {
+      "name": "birthday_int",
+      "type": "int"
+    },
+    {
+      "name": "birthday_long",
+      "type": "long"
+    },
+    {
+      "name": "birthday_optional",
+      "type": ["null", "int"]
+    },
+    {
+      "name": "updated_at",
+      "type": "long"
+    },
+    {
+      "name": "created_at",
+      "type": "long"
+    }
+  ]
+}

data/spec/spec_helper.rb

@@ -2,6 +2,7 @@
 
 $LOAD_PATH.unshift(File.expand_path('../lib', __dir__))
 require 'active_record'
+require 'database_cleaner'
 require 'deimos'
 require 'deimos/metrics/mock'
 require 'deimos/tracing/mock'
@@ -147,6 +148,9 @@ RSpec.configure do |config|
   # true by default for RSpec 4.0
   config.shared_context_metadata_behavior = :apply_to_host_groups
 
+  config.filter_run(focus: true)
+  config.run_all_when_everything_filtered = true
+
   config.before(:all) do
     Time.zone = 'Eastern Time (US & Canada)'
     ActiveRecord::Base.logger = Logger.new('/dev/null')
@@ -159,6 +163,9 @@ RSpec.configure do |config|
   config.include ActiveSupport::Testing::TimeHelpers
   config.before(:suite) do
     setup_db(DbConfigs::DB_OPTIONS.last)
+
+    DatabaseCleaner.strategy = :transaction
+    DatabaseCleaner.clean_with(:truncation)
   end
 
   config.mock_with(:rspec) do |mocks|
@@ -179,6 +186,16 @@ RSpec.configure do |config|
       deimos_config.schema.backend = :avro_validation
     end
   end
+
+  config.around(:each) do |example|
+    use_cleaner = !example.metadata[:integration]
+
+    DatabaseCleaner.start if use_cleaner
+
+    example.run
+
+    DatabaseCleaner.clean if use_cleaner
+  end
 end
 
 RSpec.shared_context('with widgets') do

data/spec/utils/db_poller_spec.rb

@@ -48,8 +48,8 @@ each_db_config(Deimos::Utils::DbPoller) do
       end
 
       allow(Deimos::Utils::DbPoller).to receive(:new)
-      signal_double = instance_double(Deimos::Utils::SignalHandler, run!: nil)
-      allow(Deimos::Utils::SignalHandler).to receive(:new).and_return(signal_double)
+      signal_double = instance_double(Sigurd::SignalHandler, run!: nil)
+      allow(Sigurd::SignalHandler).to receive(:new).and_return(signal_double)
       described_class.start!
       expect(Deimos::Utils::DbPoller).to have_received(:new).twice
       expect(Deimos::Utils::DbPoller).to have_received(:new).

data/spec/utils/deadlock_retry_spec.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+RSpec.describe Deimos::Utils::DeadlockRetry do
+  include_context 'with widgets'
+
+  before(:each) do
+    allow(described_class).to receive(:sleep)
+  end
+
+  describe 'deadlock handling' do
+    let(:batch) { [{ key: 1, payload: { test_id: 'abc', some_int: 3 } }] }
+
+    it 'should retry deadlocks 3 times' do
+      # Should receive original attempt + 2 retries
+      expect(Widget).
+        to receive(:create).
+        and_raise(ActiveRecord::Deadlocked.new('Lock wait timeout exceeded')).
+        exactly(3).times
+
+      # After 3 tries, should let it bubble up
+      expect {
+        described_class.wrap do
+          Widget.create(test_id: 'abc')
+        end
+      }.to raise_error(ActiveRecord::Deadlocked)
+    end
+
+    it 'should stop retrying deadlocks after success' do
+      allow(Widget).
+        to receive(:create).
+        with(hash_including(test_id: 'first')).
+        and_call_original
+
+      # Fail on first attempt, succeed on second
+      expect(Widget).
+        to receive(:create).
+        with(hash_including(test_id: 'second')).
+        and_raise(ActiveRecord::Deadlocked.new('Deadlock found when trying to get lock')).
+        once.
+        ordered
+
+      expect(Widget).
+        to receive(:create).
+        with(hash_including(test_id: 'second')).
+        once.
+        ordered.
+        and_call_original
+
+      # Should not raise anything
+      described_class.wrap do
+        Widget.create(test_id: 'first')
+        Widget.create(test_id: 'second')
+      end
+
+      expect(Widget.all).to match_array([
+                                          have_attributes(test_id: 'first'),
+                                          have_attributes(test_id: 'second')
+                                        ])
+    end
+
+    it 'should not retry non-deadlock exceptions' do
+      expect(Widget).
+        to receive(:create).
+        and_raise(ActiveRecord::StatementInvalid.new('Oops!!')).
+        once
+
+      expect {
+        described_class.wrap do
+          Widget.create(test_id: 'abc')
+        end
+      }.to raise_error(ActiveRecord::StatementInvalid, 'Oops!!')
+    end
+  end
+end

data/spec/utils/lag_reporter_spec.rb

@@ -2,8 +2,11 @@
 
 describe Deimos::Utils::LagReporter do
 
+  let(:kafka_client) { instance_double(Kafka::Client) }
+  let(:partition1_tags) { %w(consumer_group:group1 partition:1 topic:my-topic) }
+  let(:partition2_tags) { %w(consumer_group:group1 partition:2 topic:my-topic) }
+
   before(:each) do
-    kafka_client = instance_double(Kafka::Client)
     allow(kafka_client).to receive(:last_offset_for).and_return(100)
     allow(Phobos).to receive(:create_kafka_client).and_return(kafka_client)
     Deimos.configure { |c| c.consumers.report_lag = true }
@@ -20,38 +23,22 @@ describe Deimos::Utils::LagReporter do
       'heartbeat.consumer.kafka',
       group_id: 'group1', topic_partitions: { 'my-topic': [1] }
     )
-
   end
 
   it 'should report lag' do
     expect(Deimos.config.metrics).to receive(:gauge).ordered.twice.
-      with('consumer_lag', 95,
-           tags: %w(
-             consumer_group:group1
-             partition:1
-             topic:my-topic
-           ))
+      with('consumer_lag', 95, tags: partition1_tags)
     expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
-      with('consumer_lag', 80,
-           tags: %w(
-             consumer_group:group1
-             partition:2
-             topic:my-topic
-           ))
+      with('consumer_lag', 80, tags: partition2_tags)
     expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
-      with('consumer_lag', 0,
-           tags: %w(
-             consumer_group:group1
-             partition:2
-             topic:my-topic
-           ))
+      with('consumer_lag', 0, tags: partition2_tags)
     ActiveSupport::Notifications.instrument(
       'seek.consumer.kafka',
       offset: 5, topic: 'my-topic', group_id: 'group1', partition: 1
     )
     ActiveSupport::Notifications.instrument(
       'start_process_message.consumer.kafka',
-      offset_lag: 80, topic: 'my-topic', group_id: 'group1', partition: 2
+      offset: 20, topic: 'my-topic', group_id: 'group1', partition: 2
     )
     ActiveSupport::Notifications.instrument(
       'heartbeat.consumer.kafka',
@@ -59,8 +46,28 @@ describe Deimos::Utils::LagReporter do
     )
     ActiveSupport::Notifications.instrument(
       'start_process_batch.consumer.kafka',
-      offset_lag: 0, topic: 'my-topic', group_id: 'group1', partition: 2
+      last_offset: 100, topic: 'my-topic', group_id: 'group1', partition: 2
+    )
+    ActiveSupport::Notifications.instrument(
+      'heartbeat.consumer.kafka',
+      group_id: 'group1', topic_partitions: { 'my-topic': [1, 2] }
+    )
+  end
+
+  it 'should update lag after heartbeat' do
+    expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
+      with('consumer_lag', 94, tags: partition2_tags)
+    expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
+      with('consumer_lag', 95, tags: partition2_tags)
+    ActiveSupport::Notifications.instrument(
+      'seek.consumer.kafka',
+      offset: 6, topic: 'my-topic', group_id: 'group1', partition: 2
+    )
+    ActiveSupport::Notifications.instrument(
+      'heartbeat.consumer.kafka',
+      group_id: 'group1', topic_partitions: { 'my-topic': [1, 2] }
     )
+    allow(kafka_client).to receive(:last_offset_for).and_return(101)
     ActiveSupport::Notifications.instrument(
       'heartbeat.consumer.kafka',
       group_id: 'group1', topic_partitions: { 'my-topic': [1, 2] }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.7.0.pre.beta1
+  version: 1.8.0.pre.beta1
 platform: ruby
 authors:
 - Daniel Orner
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-05-12 00:00:00.000000000 Z
+date: 2020-07-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: sigurd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.1
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
@@ -94,6 +108,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.9'
+- !ruby/object:Gem::Dependency
+  name: database_cleaner
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
 - !ruby/object:Gem::Dependency
   name: ddtrace
   requirement: !ruby/object:Gem::Requirement
@@ -296,7 +324,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
-description: 
+description:
 email:
 - daniel.orner@wishabi.com
 executables:
@@ -323,10 +351,15 @@ files:
 - bin/deimos
 - deimos-ruby.gemspec
 - docker-compose.yml
+- docs/ARCHITECTURE.md
 - docs/CONFIGURATION.md
 - docs/DATABASE_BACKEND.md
 - docs/PULL_REQUEST_TEMPLATE.md
 - lib/deimos.rb
+- lib/deimos/active_record_consume/batch_consumption.rb
+- lib/deimos/active_record_consume/batch_slicer.rb
+- lib/deimos/active_record_consume/message_consumption.rb
+- lib/deimos/active_record_consume/schema_model_converter.rb
 - lib/deimos/active_record_consumer.rb
 - lib/deimos/active_record_producer.rb
 - lib/deimos/backends/base.rb
@@ -334,11 +367,12 @@ files:
 - lib/deimos/backends/kafka.rb
 - lib/deimos/backends/kafka_async.rb
 - lib/deimos/backends/test.rb
-- lib/deimos/base_consumer.rb
 - lib/deimos/batch_consumer.rb
 - lib/deimos/config/configurable.rb
 - lib/deimos/config/configuration.rb
 - lib/deimos/config/phobos_config.rb
+- lib/deimos/consume/batch_consumption.rb
+- lib/deimos/consume/message_consumption.rb
 - lib/deimos/consumer.rb
 - lib/deimos/instrumentation.rb
 - lib/deimos/kafka_message.rb
@@ -368,11 +402,9 @@ files:
 - lib/deimos/tracing/provider.rb
 - lib/deimos/utils/db_poller.rb
 - lib/deimos/utils/db_producer.rb
-- lib/deimos/utils/executor.rb
+- lib/deimos/utils/deadlock_retry.rb
 - lib/deimos/utils/inline_consumer.rb
 - lib/deimos/utils/lag_reporter.rb
-- lib/deimos/utils/platform_schema_validation.rb
-- lib/deimos/utils/signal_handler.rb
 - lib/deimos/version.rb
 - lib/generators/deimos/db_backend/templates/migration
 - lib/generators/deimos/db_backend/templates/rails3_migration
@@ -381,6 +413,9 @@ files:
 - lib/generators/deimos/db_poller/templates/rails3_migration
 - lib/generators/deimos/db_poller_generator.rb
 - lib/tasks/deimos.rake
+- spec/active_record_batch_consumer_spec.rb
+- spec/active_record_consume/batch_slicer_spec.rb
+- spec/active_record_consume/schema_model_converter_spec.rb
 - spec/active_record_consumer_spec.rb
 - spec/active_record_producer_spec.rb
 - spec/backends/base_spec.rb
@@ -396,6 +431,7 @@ files:
 - spec/handlers/my_consumer.rb
 - spec/kafka_source_spec.rb
 - spec/kafka_topic_info_spec.rb
+- spec/message_spec.rb
 - spec/phobos.bad_db.yml
 - spec/phobos.yml
 - spec/producer_spec.rb
@@ -407,19 +443,20 @@ files:
 - spec/schema_backends/base_spec.rb
 - spec/schemas/com/my-namespace/MySchema-key.avsc
 - spec/schemas/com/my-namespace/MySchema.avsc
+- spec/schemas/com/my-namespace/MySchemaCompound-key.avsc
 - spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc
 - spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc
 - spec/schemas/com/my-namespace/MySchemaWithId.avsc
 - spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc
+- spec/schemas/com/my-namespace/Wibble.avsc
 - spec/schemas/com/my-namespace/Widget.avsc
 - spec/schemas/com/my-namespace/WidgetTheSecond.avsc
 - spec/spec_helper.rb
 - spec/utils/db_poller_spec.rb
 - spec/utils/db_producer_spec.rb
-- spec/utils/executor_spec.rb
+- spec/utils/deadlock_retry_spec.rb
 - spec/utils/lag_reporter_spec.rb
 - spec/utils/platform_schema_validation_spec.rb
-- spec/utils/signal_handler_spec.rb
 - support/deimos-solo.png
 - support/deimos-with-name-next.png
 - support/deimos-with-name.png
@@ -428,7 +465,7 @@ homepage: ''
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -443,12 +480,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
-signing_key: 
+rubygems_version: 3.1.3
+signing_key:
 specification_version: 4
 summary: Kafka libraries for Ruby.
 test_files:
+- spec/active_record_batch_consumer_spec.rb
+- spec/active_record_consume/batch_slicer_spec.rb
+- spec/active_record_consume/schema_model_converter_spec.rb
 - spec/active_record_consumer_spec.rb
 - spec/active_record_producer_spec.rb
 - spec/backends/base_spec.rb
@@ -464,6 +503,7 @@ test_files:
 - spec/handlers/my_consumer.rb
 - spec/kafka_source_spec.rb
 - spec/kafka_topic_info_spec.rb
+- spec/message_spec.rb
 - spec/phobos.bad_db.yml
 - spec/phobos.yml
 - spec/producer_spec.rb
@@ -475,16 +515,17 @@ test_files:
 - spec/schema_backends/base_spec.rb
 - spec/schemas/com/my-namespace/MySchema-key.avsc
 - spec/schemas/com/my-namespace/MySchema.avsc
+- spec/schemas/com/my-namespace/MySchemaCompound-key.avsc
 - spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc
 - spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc
 - spec/schemas/com/my-namespace/MySchemaWithId.avsc
 - spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc
+- spec/schemas/com/my-namespace/Wibble.avsc
 - spec/schemas/com/my-namespace/Widget.avsc
 - spec/schemas/com/my-namespace/WidgetTheSecond.avsc
 - spec/spec_helper.rb
 - spec/utils/db_poller_spec.rb
 - spec/utils/db_producer_spec.rb
-- spec/utils/executor_spec.rb
+- spec/utils/deadlock_retry_spec.rb
 - spec/utils/lag_reporter_spec.rb
 - spec/utils/platform_schema_validation_spec.rb
-- spec/utils/signal_handler_spec.rb