deimos-ruby 1.19.7 → 1.20.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 786ff723866bcb2f983f384e7dd8dbf6675ceb8c454a80d58a7c5eed7e730f3e
-  data.tar.gz: 7bcd9693ce7fe0db27a63285c9ebc47a2299449c95f5526806fe4e2cb5be4f24
+  metadata.gz: 51c4ea2e760bba16fc5bc251bdd842f578340f40543d54287f408e76ea4c6dfd
+  data.tar.gz: 4bffdae791e9b8b2aa491c43ff7f4c0ba16a98582f9bee872af648da6afdf846
 SHA512:
-  metadata.gz: 8fe1b840ebb4255e8df49e0600d5b8a85aad3c40807170b557cb43a236ed7b53af83a0348f8c3e6d4511f87987ade29be1855cf9f3c4a88bf7d254ba556c460e
-  data.tar.gz: 43b43d127aff1b275453d86bd46ea38097a231419863afee300adb2914ef8a9734d086871d3dc4a96ae8148ddb7ab1423f113ca3fc6ff6d39dc411428a6f8472
+  metadata.gz: 68565725b1ff47b86ba70e4a993befadb95f3a8e99fbef5a1d7f2dd51c25e5749e66c0ac8e82f5dafbb31f2605b3f03743cc774486bf616b1cd10c746df937bf
+  data.tar.gz: d1aa1f426f7c6b6894724ccc7a84661c6e4ca151e11f419cea024d5b562f0f40b80cfc44c3f1410ed125a4eae482261200870a5136af301de85bf2e5f9f5d3d4

data/CHANGELOG.md

@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+# 1.20.0 - 2023-04-17
+
+- Feature: Updated the DB Poller logic to allow for inherited db poller classes to initialize producers and
+  be able to publish to multiple Kafka topics using the results from a single poll_query method
+
 # 1.19.7 - 2023-04-17
 
 - Fix: Update Datadog metrics backend so it doesn't crash on newer versions of dogstatsd-ruby

data/docs/CONFIGURATION.md

@@ -114,7 +114,7 @@ end
 ```
 
 Config name|Default|Description
------------|--|-----------
+-----------|-------|-----------
 producer_class|nil|ActiveRecordProducer class to use for sending messages.
 mode|:time_based|Whether to use time-based polling or state-based polling.
 run_every|60|Amount of time in seconds to wait between runs.
@@ -127,6 +127,7 @@ state_column|nil|If set, this represents the DB column to use to update publishi
 publish_timestamp_column|nil|If set, this represents the DB column to use to update when publishing is done. State-based only.
 published_state|nil|If set, the poller will update the `state_column` to this value when publishing succeeds. State-based only.
 failed_state|nil|If set, the poller will update the `state_column` to this value when publishing fails. State-based only.
+poller_class|nil|Inherited poller class name to use for publishing to multiple kafka topics from a single poller.
 
 ## Kafka Configuration
 

data/lib/deimos/config/configuration.rb

@@ -453,7 +453,7 @@ module Deimos
       # Mode to use for querying - :time_based (via updated_at) or :state_based.
       setting :mode, :time_based
       # Producer class to use for the poller.
-      setting :producer_class
+      setting :producer_class, nil
       # How often to run the poller, in seconds. If the poll takes longer than this
       # time, it will run again immediately and the timeout
       # will be pushed to the next e.g. 1 minute.
@@ -481,6 +481,9 @@ module Deimos
       setting :published_state
       # Value to set the state_column to if publishing fails - state-based only.
       setting :failed_state
+
+      # Inherited poller class name to use for publishing to multiple kafka topics from a single poller
+      setting :poller_class, nil
     end
 
     deprecate 'kafka_logger', 'kafka.logger'

data/lib/deimos/utils/db_poller/base.rb

@@ -21,18 +21,30 @@ module Deimos
         # @return [Hash]
         attr_reader :config
 
+        # Method to define producers if a single poller needs to publish to multiple topics.
+        # Producer classes should be constantized
+        # @return [Array<Producer>]
+        def self.producers
+          []
+        end
+
         # @param config [FigTree::ConfigStruct]
         def initialize(config)
           @config = config
           @id = SecureRandom.hex
           begin
-            @producer = @config.producer_class.constantize
+            if @config.poller_class.nil? && @config.producer_class.nil?
+              raise 'No producers have been set for this DB poller!'
+            end
+
+            @resource_class = self.class.producers.any? ? self.class : @config.producer_class.constantize
+
+            producer_classes.each do |producer_class|
+              validate_producer_class(producer_class)
+            end
           rescue NameError
             raise "Class #{@config.producer_class} not found!"
           end
-          unless @producer < Deimos::ActiveRecordProducer
-            raise "Class #{@producer.class.name} is not an ActiveRecordProducer!"
-          end
         end
 
         # Start the poll:
@@ -65,12 +77,12 @@ module Deimos
         # Grab the PollInfo or create if it doesn't exist.
         # @return [void]
         def retrieve_poll_info
-          @info = Deimos::PollInfo.find_by_producer(@config.producer_class) || create_poll_info
+          @info = Deimos::PollInfo.find_by_producer(@resource_class.to_s) || create_poll_info
         end
 
         # @return [Deimos::PollInfo]
         def create_poll_info
-          Deimos::PollInfo.create!(producer: @config.producer_class, last_sent: Time.new(0))
+          Deimos::PollInfo.create!(producer: @resource_class.to_s, last_sent: Time.new(0))
         end
 
         # Indicate whether this current loop should process updates. Most loops
@@ -101,7 +113,7 @@ module Deimos
           begin
             span = Deimos.config.tracer&.start(
               'deimos-db-poller',
-              resource: @producer.class.name.gsub('::', '-')
+              resource: @resource_class.name.gsub('::', '-')
             )
             process_batch(batch)
             Deimos.config.tracer&.finish(span)
@@ -128,10 +140,35 @@ module Deimos
           true
         end
 
+        # Publish batch using the configured producers
         # @param batch [Array<ActiveRecord::Base>]
         # @return [void]
         def process_batch(batch)
-          @producer.send_events(batch)
+          producer_classes.each do |producer|
+            producer.send_events(batch)
+          end
+        end
+
+        # Configure log identifier and messages to be used in subclasses
+        # @return [String]
+        def log_identifier
+          "#{@resource_class.name}: #{producer_classes.map(&:topic)}"
+        end
+
+        # Return array of configured producers depending on poller class
+        # @return [Array<ActiveRecordProducer>]
+        def producer_classes
+          return self.class.producers if self.class.producers.any?
+
+          [@config.producer_class.constantize]
+        end
+
+        # Validate if a producer class is an ActiveRecordProducer or not
+        # @return [void]
+        def validate_producer_class(producer_class)
+          unless producer_class < Deimos::ActiveRecordProducer
+            raise "Class #{producer_class.class.name} is not an ActiveRecordProducer!"
+          end
         end
       end
     end

data/lib/deimos/utils/db_poller/state_based.rb

@@ -10,13 +10,13 @@ module Deimos
         # Send messages for updated data.
         # @return [void]
         def process_updates
-          Deimos.config.logger.info("Polling #{@producer.topic}")
+          Deimos.config.logger.info("Polling #{log_identifier}")
           status = PollStatus.new(0, 0, 0)
 
           # poll_query gets all the relevant data from the database, as defined
           # by the producer itself.
           loop do
-            Deimos.config.logger.debug("Polling #{@producer.topic}, batch #{status.current_batch}")
+            Deimos.config.logger.debug("Polling #{log_identifier}, batch #{status.current_batch}")
             batch = fetch_results.to_a
             if batch.empty?
               @info.touch(:last_sent)
@@ -26,12 +26,12 @@ module Deimos
             success = process_batch_with_span(batch, status)
             finalize_batch(batch, success)
           end
-          Deimos.config.logger.info("Poll #{@producer.topic} complete (#{status.report}")
+          Deimos.config.logger.info("Poll #{log_identifier} complete (#{status.report}")
         end
 
         # @return [ActiveRecord::Relation]
         def fetch_results
-          @producer.poll_query.limit(BATCH_SIZE).order(@config.timestamp_column)
+          @resource_class.poll_query.limit(BATCH_SIZE).order(@config.timestamp_column)
         end
 
         # @param batch [Array<ActiveRecord::Base>]

data/lib/deimos/utils/db_poller/time_based.rb

@@ -11,7 +11,7 @@ module Deimos
         # :nodoc:
         def create_poll_info
           new_time = @config.start_from_beginning ? Time.new(0) : Time.zone.now
-          Deimos::PollInfo.create!(producer: @config.producer_class,
+          Deimos::PollInfo.create!(producer: @resource_class.to_s,
                                    last_sent: new_time,
                                    last_sent_id: 0)
         end
@@ -28,13 +28,13 @@ module Deimos
         def process_updates
           time_from = @config.full_table ? Time.new(0) : @info.last_sent.in_time_zone
           time_to = Time.zone.now - @config.delay_time
-          Deimos.config.logger.info("Polling #{@producer.topic} from #{time_from} to #{time_to}")
+          Deimos.config.logger.info("Polling #{log_identifier} from #{time_from} to #{time_to}")
           status = PollStatus.new(0, 0, 0)
 
           # poll_query gets all the relevant data from the database, as defined
           # by the producer itself.
           loop do
-            Deimos.config.logger.debug("Polling #{@producer.topic}, batch #{status.current_batch}")
+            Deimos.config.logger.debug("Polling #{log_identifier}, batch #{status.current_batch}")
             batch = fetch_results(time_from, time_to).to_a
             if batch.empty?
               @info.touch(:last_sent)
@@ -44,20 +44,20 @@ module Deimos
             process_and_touch_info(batch, status)
             time_from = last_updated(batch.last)
           end
-          Deimos.config.logger.info("Poll #{@producer.topic} complete at #{time_to} (#{status.report})")
+          Deimos.config.logger.info("Poll #{log_identifier} complete at #{time_to} (#{status.report})")
         end
 
         # @param time_from [ActiveSupport::TimeWithZone]
         # @param time_to [ActiveSupport::TimeWithZone]
         # @return [ActiveRecord::Relation]
         def fetch_results(time_from, time_to)
-          id = @producer.config[:record_class].primary_key
+          id = self.producer_classes.first.config[:record_class].primary_key
           quoted_timestamp = ActiveRecord::Base.connection.quote_column_name(@config.timestamp_column)
           quoted_id = ActiveRecord::Base.connection.quote_column_name(id)
-          @producer.poll_query(time_from: time_from,
-                               time_to: time_to,
-                               column_name: @config.timestamp_column,
-                               min_id: @info.last_sent_id).
+          @resource_class.poll_query(time_from: time_from,
+                                     time_to: time_to,
+                                     column_name: @config.timestamp_column,
+                                     min_id: @info.last_sent_id).
             limit(BATCH_SIZE).
             order("#{quoted_timestamp}, #{quoted_id}")
         end

data/lib/deimos/utils/db_poller.rb

@@ -12,7 +12,7 @@ module Deimos
         end
 
         pollers = Deimos.config.db_poller_objects.map do |poller_config|
-          self.class_for_config(poller_config.mode).new(poller_config)
+          self.class_for_config(poller_config).new(poller_config)
         end
         executor = Sigurd::Executor.new(pollers,
                                         sleep_seconds: 5,
@@ -21,15 +21,21 @@ module Deimos
         signal_handler.run!
       end
 
-      # @param config_name [Symbol]
+      # @param config_name [DBPollerConfig]
       # @return [Class<Deimos::Utils::DbPoller>]
       def self.class_for_config(config_name)
-        case config_name
-        when :state_based
-          Deimos::Utils::DbPoller::StateBased
+        if config_name.poller_class.present?
+          config_name.poller_class.constantize
         else
-          Deimos::Utils::DbPoller::TimeBased
+          case config_name.mode
+          when :state_based
+            Deimos::Utils::DbPoller::StateBased
+          else
+            Deimos::Utils::DbPoller::TimeBased
+          end
         end
+      rescue NameError
+          raise "Class #{config_name.poller_class} not found!"
       end
 
       PollStatus = Struct.new(:batches_processed, :batches_errored, :messages_processed) do

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.19.7'
+  VERSION = '1.20.0'
 end

data/spec/spec_helper.rb

@@ -221,6 +221,8 @@ RSpec.shared_context('with widgets') do
       t.string(:test_id)
       t.integer(:some_int)
       t.boolean(:some_bool)
+      t.string(:publish_status)
+      t.datetime(:published_at)
       t.timestamps
     end
 

data/spec/utils/db_poller_spec.rb

@@ -68,7 +68,7 @@ each_db_config(Deimos::Utils::DbPoller::Base) do
     include_context 'with widgets'
 
     let(:poller) do
-      poller = Deimos::Utils::DbPoller.class_for_config(config.mode).new(config)
+      poller = Deimos::Utils::DbPoller.class_for_config(config).new(config)
       allow(poller).to receive(:sleep)
       poller
     end
@@ -387,5 +387,103 @@ each_db_config(Deimos::Utils::DbPoller::Base) do
       end
     end
   end
+
+  describe 'multi_producer_pollers' do
+    include_context 'with widgets'
+
+    let(:poller) do
+      poller = Deimos::Utils::DbPoller.class_for_config(config).new(config)
+      allow(poller).to receive(:sleep)
+      poller
+    end
+
+    let(:config) { Deimos.config.db_poller_objects.first.dup }
+
+    before(:each) do
+      Widget.delete_all
+      producer_class = Class.new(Deimos::ActiveRecordProducer) do
+        schema 'MySchemaWithId'
+        namespace 'com.my-namespace'
+        topic 'my-topic-with-id'
+        key_config none: true
+        record_class Widget
+
+        # :nodoc:
+        def self.generate_payload(attrs, widget)
+          super.merge(message_id: widget.generated_id)
+        end
+      end
+      stub_const('ProducerOne', producer_class)
+
+      producer_class = Class.new(Deimos::ActiveRecordProducer) do
+        schema 'MySchemaWithId'
+        namespace 'com.my-namespace'
+        topic 'my-topic-with-id'
+        key_config none: true
+        record_class Widget
+
+        # :nodoc:
+        def self.generate_payload(attrs, widget)
+          super.merge(message_id: widget.generated_id)
+        end
+      end
+      stub_const('ProducerTwo', producer_class)
+
+      poller_class = Class.new(Deimos::Utils::DbPoller::StateBased) do
+        def self.producers
+          [ProducerOne, ProducerTwo]
+        end
+
+        def self.poll_query(*)
+          Widget.where(publish_status: nil)
+        end
+      end
+      stub_const('Deimos::Utils::DbPoller::MultiProducerPoller', poller_class)
+    end
+
+    it 'should publish to two different kafka topics from two producers' do
+      Deimos.configure do
+        db_poller do
+          poller_class 'Deimos::Utils::DbPoller::MultiProducerPoller'
+          mode :state_based
+          state_column :publish_status
+          publish_timestamp_column :published_at
+          published_state 'PUBLISHED'
+          failed_state 'PUBLISH_FAILED'
+          run_every 1.minute
+        end
+      end
+
+      widgets = (1..3).map do |i|
+        Widget.create!(test_id: 'some_id', some_int: i,
+                       updated_at: time_value(mins: -61, secs: 30 + i),
+                       publish_status: nil, published_at: nil)
+      end
+      poller.retrieve_poll_info
+      allow(Deimos::Utils::DbPoller::MultiProducerPoller).to receive(:poll_query).and_call_original
+      allow(ProducerOne).to receive(:send_events)
+      allow(ProducerTwo).to receive(:send_events)
+      expect(Deimos::Utils::DbPoller::MultiProducerPoller).to receive(:poll_query).at_least(:once)
+      poller.process_updates
+
+      expect(ProducerOne).to have_received(:send_events).with(widgets)
+      expect(ProducerTwo).to have_received(:send_events).with(widgets)
+      expect(widgets.map(&:reload).map(&:publish_status)).to eq(%w(PUBLISHED PUBLISHED PUBLISHED))
+    end
+
+    it 'should raise an error if producer_class and poller_class are both not configured' do
+      Deimos.configure do
+        db_poller do
+          mode :state_based
+          state_column :publish_status
+          publish_timestamp_column :published_at
+          published_state 'PUBLISHED'
+          failed_state 'PUBLISH_FAILED'
+          run_every 1.minute
+        end
+      end
+      expect { described_class.new(config) }.to raise_error('No producers have been set for this DB poller!')
+    end
+  end
 end
 # rubocop:enable Layout/LineLength

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.19.7
+  version: 1.20.0
 platform: ruby
 authors:
 - Daniel Orner