deimos-ruby 1.17.1 → 1.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a2626347406fc29d48c68e375692b600eff69b85d24abad15919fa21c4013e78
-  data.tar.gz: cbcab580a7f88581d6a984f908a44694fcf04f8f0a0802357a8f640ae5d0902f
+  metadata.gz: 93b418353fa710fd90425980ec2772f6cb4249b1d4e43c3cadd9fb7a099eb387
+  data.tar.gz: b319ef0e32b2e147b7ec0a3b131d8d8269e9818faa5313b6f56745498a2942c3
 SHA512:
-  metadata.gz: 79356ac5f75a0dae4c2b83875b11d1a6727c8c2268259a266b68caa3e29e16a9e1017ec45c53d86f6d6039abc37665c5b1a987e022272d8ec68ff319836593d0
-  data.tar.gz: 5982d3397e011b7684d8ed2f3c858c0426968ed907fa59773c6a2cdbfbbecce88aa7d18b33a95f33c650985a64186db85aee3247560288e68dfc08ba9f1f8989
+  metadata.gz: 60bd9c6f8cb7e4a63d791312b2fb784cfed4f2b0c60f679d71f1fff1adeac8ca86cbf285e42bf24fb1e65748fe5c1a0127710925d068c5d6bf6feeb4753ed203
+  data.tar.gz: 9e30a8666f75d62fcee4c5f99de3737af5b58c6bafc9c54f786784785472233dc6a9135c525b288d04320acb716b58274b9dcaeeba90965a04721e0e77588c83

data/.rubocop.yml

@@ -81,7 +81,7 @@ Lint/UnusedMethodArgument:
 
 Metrics/AbcSize:
   Severity: refactor
-  Max: 25
+  Max: 35
 
 Metrics/BlockLength:
   Enabled: false

data/CHANGELOG.md

@@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+# 1.18.0 - 2022-11-01
+
+### Features :star:
+
+- Add the `state_based` mode for DB pollers.
+
+### Fixes :wrench:
+- Fix the mock schema backend's `encode_key` method so it doesn't crash when used in application code.
+
 # 1.17.1 - 2022-10-20
 
 - Fix the log message for publishing messages so it uses the topic of the actual message instead of 

data/README.md

@@ -803,6 +803,25 @@ end
 Note that the poller will retry infinitely if it encounters a Kafka-related error such
 as a communication failure. For all other errors, it will retry once by default.
 
+### State-based pollers
+
+By default, pollers use timestamps and IDs to determine the records to publish. However, you can
+set a different mode whereby it will include all records that match your query, and when done,
+will update a state and/or timestamp column which should remove it from that query. With this
+algorithm, you can ignore the `updated_at` and `id` columns.
+
+To configure a state-based poller:
+
+```ruby
+db_poller do
+  mode :state_based
+  state_column :publish_state # the name of the column to update state to
+  publish_timestamp_column :published_at # the column to update when publishing succeeds
+  published_state 'published' # the value to put into the state_column when publishing succeeds
+  failed_state 'publish_failed' the value to put into the state_column when publishing fails
+end
+```
+
 ## Running consumers
 
 Deimos includes a rake task. Once it's in your gemfile, just run

data/docs/CONFIGURATION.md

@@ -112,14 +112,19 @@ end
 ```
 
 Config name|Default|Description
------------|-------|-----------
+-----------|--|-----------
 producer_class|nil|ActiveRecordProducer class to use for sending messages.
+mode|:time_based|Whether to use time-based polling or state-based polling.
 run_every|60|Amount of time in seconds to wait between runs.
 timestamp_column|`:updated_at`|Name of the column to query. Remember to add an index to this column!
 delay_time|2|Amount of time in seconds to wait before picking up records, to allow for transactions to finish.
-full_table|false|If set to true, do a full table dump to Kafka each run. Good for very small tables.
-start_from_beginning|true|If false, start from the current time instead of the beginning of time if this is the first time running the poller.
 retries|1|The number of times to retry for a *non-Kafka* error.
+full_table|false|If set to true, do a full table dump to Kafka each run. Good for very small tables. Time-based only.
+start_from_beginning|true|If false, start from the current time instead of the beginning of time if this is the first time running the poller. Time-based only.
+state_column|nil|If set, this represents the DB column to use to update publishing status. State-based only.
+publish_timestamp_column|nil|If set, this represents the DB column to use to update when publishing is done. State-based only.
+published_state|nil|If set, the poller will update the `state_column` to this value when publishing succeeds. State-based only.
+failed_state|nil|If set, the poller will update the `state_column` to this value when publishing fails. State-based only.
 
 ## Kafka Configuration
 

data/lib/deimos/config/configuration.rb

@@ -444,25 +444,37 @@ module Deimos
     end
 
     setting_object :db_poller do
+      # Mode to use for querying - :time_based (via updated_at) or :state_based.
+      setting :mode, :time_based
       # Producer class to use for the poller.
       setting :producer_class
       # How often to run the poller, in seconds. If the poll takes longer than this
       # time, it will run again immediately and the timeout
       # will be pushed to the next e.g. 1 minute.
       setting :run_every, 60
-      # Column to use to find updates. Must have an index on it.
-      setting :timestamp_column, :updated_at
+      # The number of times to retry production when encountering a *non-Kafka* error.
+      setting :retries, 1
       # Amount of time, in seconds, to wait before catching updates, to allow transactions
-      # to complete but still pick up the right records.
+      # to complete but still pick up the right records. Should only be set for time-based mode.
       setting :delay_time, 2
+      # Column to use to find updates. Must have an index on it.
+      setting :timestamp_column, :updated_at
+
       # If true, dump the full table rather than incremental changes. Should
-      # only be used for very small tables.
+      # only be used for very small tables. Time-based only.
       setting :full_table, false
       # If false, start from the current time instead of the beginning of time
-      # if this is the first time running the poller.
+      # if this is the first time running the poller. Time-based only.
       setting :start_from_beginning, true
-      # The number of times to retry production when encountering a *non-Kafka* error.
-      setting :retries, 1
+
+      # Column to set once publishing is complete - state-based only.
+      setting :state_column
+      # Column to update with e.g. published_at. State-based only.
+      setting :publish_timestamp_column
+      # Value to set the state_column to once published - state-based only.
+      setting :published_state
+      # Value to set the state_column to if publishing fails - state-based only.
+      setting :failed_state
     end
 
     deprecate 'kafka_logger', 'kafka.logger'

data/lib/deimos/schema_backends/mock.rb

@@ -29,7 +29,7 @@ module Deimos
       end
 
       # @override
-      def encode_key(key_id, key)
+      def encode_key(key_id, key, topic: nil)
         { key_id => key }
       end
 

data/lib/deimos/utils/db_poller/base.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require 'deimos/utils/db_poller'
+require 'deimos/poll_info'
+require 'sigurd'
+
+module Deimos
+  module Utils
+    # Class which continually polls the database and sends Kafka messages.
+    module DbPoller
+      # Base poller class for retrieving and publishing messages.
+      class Base
+
+        # @return [Integer]
+        BATCH_SIZE = 1000
+
+        # Needed for Executor so it can identify the worker
+        # @return [Integer]
+        attr_reader :id
+
+        # @return [Hash]
+        attr_reader :config
+
+        # @param config [FigTree::ConfigStruct]
+        def initialize(config)
+          @config = config
+          @id = SecureRandom.hex
+          begin
+            @producer = @config.producer_class.constantize
+          rescue NameError
+            raise "Class #{@config.producer_class} not found!"
+          end
+          unless @producer < Deimos::ActiveRecordProducer
+            raise "Class #{@producer.class.name} is not an ActiveRecordProducer!"
+          end
+        end
+
+        # Start the poll:
+        # 1) Grab the current PollInfo from the database indicating the last
+        # time we ran
+        # 2) On a loop, process all the recent updates between the last time
+        # we ran and now.
+        # @return [void]
+        def start
+          # Don't send asynchronously
+          if Deimos.config.producers.backend == :kafka_async
+            Deimos.config.producers.backend = :kafka
+          end
+          Deimos.config.logger.info('Starting...')
+          @signal_to_stop = false
+          ActiveRecord::Base.connection.reconnect! unless ActiveRecord::Base.connection.open_transactions.positive?
+
+          retrieve_poll_info
+          loop do
+            if @signal_to_stop
+              Deimos.config.logger.info('Shutting down')
+              break
+            end
+            process_updates if should_run?
+            sleep(0.1)
+          end
+        end
+
+        # @return [void]
+        # Grab the PollInfo or create if it doesn't exist.
+        # @return [void]
+        def retrieve_poll_info
+          @info = Deimos::PollInfo.find_by_producer(@config.producer_class) || create_poll_info
+        end
+
+        # @return [Deimos::PollInfo]
+        def create_poll_info
+          Deimos::PollInfo.create!(producer: @config.producer_class, last_sent: Time.new(0))
+        end
+
+        # Indicate whether this current loop should process updates. Most loops
+        # will busy-wait (sleeping 0.1 seconds) until it's ready.
+        # @return [Boolean]
+        def should_run?
+          Time.zone.now - @info.last_sent - @config.delay_time >= @config.run_every
+        end
+
+        # Stop the poll.
+        # @return [void]
+        def stop
+          Deimos.config.logger.info('Received signal to stop')
+          @signal_to_stop = true
+        end
+
+        # Send messages for updated data.
+        # @return [void]
+        def process_updates
+          raise Deimos::MissingImplementationError
+        end
+
+        # @param batch [Array<ActiveRecord::Base>]
+        # @param status [PollStatus]
+        # @return [Boolean]
+        def process_batch_with_span(batch, status)
+          retries = 0
+          begin
+            span = Deimos.config.tracer&.start(
+              'deimos-db-poller',
+              resource: @producer.class.name.gsub('::', '-')
+            )
+            process_batch(batch)
+            Deimos.config.tracer&.finish(span)
+            status.batches_processed += 1
+          rescue Kafka::Error => e # keep trying till it fixes itself
+            Deimos.config.logger.error("Error publishing through DB Poller: #{e.message}")
+            sleep(0.5)
+            retry
+          rescue StandardError => e
+            Deimos.config.logger.error("Error publishing through DB poller: #{e.message}}")
+            if retries < @config.retries
+              retries += 1
+              sleep(0.5)
+              retry
+            else
+              Deimos.config.logger.error('Retries exceeded, moving on to next batch')
+              Deimos.config.tracer&.set_error(span, e)
+              status.batches_errored += 1
+              return false
+            end
+          ensure
+            status.messages_processed += batch.size
+          end
+          true
+        end
+
+        # @param batch [Array<ActiveRecord::Base>]
+        # @return [void]
+        def process_batch(batch)
+          @producer.send_events(batch)
+        end
+      end
+    end
+  end
+end

data/lib/deimos/utils/db_poller/state_based.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require 'deimos/utils/db_poller/base'
+
+module Deimos
+  module Utils
+    module DbPoller
+      # Poller that uses state columns to determine the records to publish.
+      class StateBased < Base
+        # Send messages for updated data.
+        # @return [void]
+        def process_updates
+          Deimos.config.logger.info("Polling #{@producer.topic}")
+          status = PollStatus.new(0, 0, 0)
+
+          # poll_query gets all the relevant data from the database, as defined
+          # by the producer itself.
+          loop do
+            Deimos.config.logger.debug("Polling #{@producer.topic}, batch #{status.current_batch}")
+            batch = fetch_results.to_a
+            break if batch.empty?
+
+            success = process_batch_with_span(batch, status)
+            finalize_batch(batch, success)
+          end
+          Deimos.config.logger.info("Poll #{@producer.topic} complete (#{status.report}")
+        end
+
+        # @return [ActiveRecord::Relation]
+        def fetch_results
+          @producer.poll_query.limit(BATCH_SIZE).order(@config.timestamp_column)
+        end
+
+        # @param batch [Array<ActiveRecord::Base>]
+        # @param success [Boolean]
+        # @return [void]
+        def finalize_batch(batch, success)
+          @info.touch
+
+          state = success ? @config.published_state : @config.failed_state
+          klass = batch.first.class
+          id_col = record.class.primary_key
+          timestamp_col = @config.timestamp_column
+
+          attrs = { timestamp_col => Time.zone.now }
+          attrs[@config.state_column] = state if state
+          if @config.publish_timestamp_column
+            attrs[@config.publish_timestamp_column] = Time.zone.now
+          end
+
+          klass.where(id_col => batch.map(&id_col)).update_all(attrs)
+        end
+
+      end
+    end
+  end
+end

data/lib/deimos/utils/db_poller/time_based.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require 'deimos/utils/db_poller/base'
+
+module Deimos
+  module Utils
+    module DbPoller
+      # Poller that uses ID and updated_at to determine the records to publish.
+      class TimeBased < Base
+
+        # :nodoc:
+        def create_poll_info
+          new_time = @config.start_from_beginning ? Time.new(0) : Time.zone.now
+          Deimos::PollInfo.create!(producer: @config.producer_class,
+                                   last_sent: new_time,
+                                   last_sent_id: 0)
+        end
+
+        # @param batch [Array<ActiveRecord::Base>]
+        # @param status [Deimos::Utils::DbPoller::PollStatus]
+        def process_and_touch_info(batch, status)
+          process_batch_with_span(batch, status)
+          self.touch_info(batch)
+        end
+
+        # Send messages for updated data.
+        # @return [void]
+        def process_updates
+          time_from = @config.full_table ? Time.new(0) : @info.last_sent.in_time_zone
+          time_to = Time.zone.now - @config.delay_time
+          Deimos.config.logger.info("Polling #{@producer.topic} from #{time_from} to #{time_to}")
+          status = PollStatus.new(0, 0, 0)
+
+          # poll_query gets all the relevant data from the database, as defined
+          # by the producer itself.
+          loop do
+            Deimos.config.logger.debug("Polling #{@producer.topic}, batch #{status.current_batch}")
+            batch = fetch_results(time_from, time_to).to_a
+            break if batch.empty?
+
+            process_and_touch_info(batch, status)
+            time_from = last_updated(batch.last)
+          end
+          Deimos.config.logger.info("Poll #{@producer.topic} complete at #{time_to} (#{status.report})")
+        end
+
+        # @param time_from [ActiveSupport::TimeWithZone]
+        # @param time_to [ActiveSupport::TimeWithZone]
+        # @return [ActiveRecord::Relation]
+        def fetch_results(time_from, time_to)
+          id = @producer.config[:record_class].primary_key
+          quoted_timestamp = ActiveRecord::Base.connection.quote_column_name(@config.timestamp_column)
+          quoted_id = ActiveRecord::Base.connection.quote_column_name(id)
+          @producer.poll_query(time_from: time_from,
+                               time_to: time_to,
+                               column_name: @config.timestamp_column,
+                               min_id: @info.last_sent_id).
+            limit(BATCH_SIZE).
+            order("#{quoted_timestamp}, #{quoted_id}")
+        end
+
+        # @param record [ActiveRecord::Base]
+        # @return [ActiveSupport::TimeWithZone]
+        def last_updated(record)
+          record.public_send(@config.timestamp_column)
+        end
+
+        # @param batch [Array<ActiveRecord::Base>]
+        # @return [void]
+        def touch_info(batch)
+          record = batch.last
+          id_method = record.class.primary_key
+          last_id = record.public_send(id_method)
+          last_updated_at = last_updated(record)
+          @info.attributes = { last_sent: last_updated_at, last_sent_id: last_id }
+          @info.save!
+        end
+
+      end
+    end
+  end
+end

data/lib/deimos/utils/db_poller.rb

@@ -1,22 +1,9 @@
 # frozen_string_literal: true
 
-require 'deimos/poll_info'
-require 'sigurd'
-
 module Deimos
   module Utils
-    # Class which continually polls the database and sends Kafka messages.
-    class DbPoller
-      # @return [Integer]
-      BATCH_SIZE = 1000
-
-      # Needed for Executor so it can identify the worker
-      # @return [Integer]
-      attr_reader :id
-
-      # @return [Hash]
-      attr_reader :config
-
+    # Overall functionality related to DB poller.
+    module DbPoller
       # Begin the DB Poller process.
       # @return [void]
       def self.start!
@@ -25,7 +12,7 @@ module Deimos
         end
 
         pollers = Deimos.config.db_poller_objects.map do |poller_config|
-          self.new(poller_config)
+          self.class_for_config(poller_config.mode).new(poller_config)
         end
         executor = Sigurd::Executor.new(pollers,
                                         sleep_seconds: 5,
@@ -34,168 +21,33 @@ module Deimos
         signal_handler.run!
       end
 
-      # @param config [FigTree::ConfigStruct]
-      def initialize(config)
-        @config = config
-        @id = SecureRandom.hex
-        begin
-          @producer = @config.producer_class.constantize
-        rescue NameError
-          raise "Class #{@config.producer_class} not found!"
-        end
-        unless @producer < Deimos::ActiveRecordProducer
-          raise "Class #{@producer.class.name} is not an ActiveRecordProducer!"
-        end
-      end
-
-      # Start the poll:
-      # 1) Grab the current PollInfo from the database indicating the last
-      # time we ran
-      # 2) On a loop, process all the recent updates between the last time
-      # we ran and now.
-      # @return [void]
-      def start
-        # Don't send asynchronously
-        if Deimos.config.producers.backend == :kafka_async
-          Deimos.config.producers.backend = :kafka
+      # @param config_name [Symbol]
+      # @return [Class<Deimos::Utils::DbPoller>]
+      def self.class_for_config(config_name)
+        case config_name
+        when :state_based
+          Deimos::Utils::DbPoller::StateBased
+        else
+          Deimos::Utils::DbPoller::TimeBased
         end
-        Deimos.config.logger.info('Starting...')
-        @signal_to_stop = false
-        retrieve_poll_info
-        loop do
-          if @signal_to_stop
-            Deimos.config.logger.info('Shutting down')
-            break
-          end
-          process_updates
-          sleep 0.1
-        end
-      end
-
-      # Grab the PollInfo or create if it doesn't exist.
-      # @return [void]
-      def retrieve_poll_info
-        ActiveRecord::Base.connection.reconnect! unless ActiveRecord::Base.connection.open_transactions.positive?
-        new_time = @config.start_from_beginning ? Time.new(0) : Time.zone.now
-        @info = Deimos::PollInfo.find_by_producer(@config.producer_class) ||
-                Deimos::PollInfo.create!(producer: @config.producer_class,
-                                         last_sent: new_time,
-                                         last_sent_id: 0)
-      end
-
-      # Stop the poll.
-      # @return [void]
-      def stop
-        Deimos.config.logger.info('Received signal to stop')
-        @signal_to_stop = true
-      end
-
-      # Indicate whether this current loop should process updates. Most loops
-      # will busy-wait (sleeping 0.1 seconds) until it's ready.
-      # @return [Boolean]
-      def should_run?
-        Time.zone.now - @info.last_sent - @config.delay_time >= @config.run_every
-      end
-
-      # @param record [ActiveRecord::Base]
-      # @return [ActiveSupport::TimeWithZone]
-      def last_updated(record)
-        record.public_send(@config.timestamp_column)
       end
 
-      # Send messages for updated data.
-      # @return [void]
-      def process_updates
-        return unless should_run?
-
-        time_from = @config.full_table ? Time.new(0) : @info.last_sent.in_time_zone
-        time_to = Time.zone.now - @config.delay_time
-        Deimos.config.logger.info("Polling #{@producer.topic} from #{time_from} to #{time_to}")
-        message_count = 0
-        batch_count = 0
-        error_count = 0
+      PollStatus = Struct.new(:batches_processed, :batches_errored, :messages_processed) do
 
-        # poll_query gets all the relevant data from the database, as defined
-        # by the producer itself.
-        loop do
-          Deimos.config.logger.debug("Polling #{@producer.topic}, batch #{batch_count + 1}")
-          batch = fetch_results(time_from, time_to).to_a
-          break if batch.empty?
-
-          if process_batch_with_span(batch)
-            batch_count += 1
-          else
-            error_count += 1
-          end
-          message_count += batch.size
-          time_from = last_updated(batch.last)
+        # @return [Integer]
+        def current_batch
+          batches_processed + 1
         end
-        Deimos.config.logger.info("Poll #{@producer.topic} complete at #{time_to} (#{message_count} messages, #{batch_count} successful batches, #{error_count} batches errored}")
-      end
-
-      # @param time_from [ActiveSupport::TimeWithZone]
-      # @param time_to [ActiveSupport::TimeWithZone]
-      # @return [ActiveRecord::Relation]
-      def fetch_results(time_from, time_to)
-        id = @producer.config[:record_class].primary_key
-        quoted_timestamp = ActiveRecord::Base.connection.quote_column_name(@config.timestamp_column)
-        quoted_id = ActiveRecord::Base.connection.quote_column_name(id)
-        @producer.poll_query(time_from: time_from,
-                             time_to: time_to,
-                             column_name: @config.timestamp_column,
-                             min_id: @info.last_sent_id).
-          limit(BATCH_SIZE).
-          order("#{quoted_timestamp}, #{quoted_id}")
-      end
 
-      # @param batch [Array<ActiveRecord::Base>]
-      # @return [void]
-      def process_batch_with_span(batch)
-        retries = 0
-        begin
-          span = Deimos.config.tracer&.start(
-            'deimos-db-poller',
-            resource: @producer.class.name.gsub('::', '-')
-          )
-          process_batch(batch)
-          Deimos.config.tracer&.finish(span)
-        rescue Kafka::Error => e # keep trying till it fixes itself
-          Deimos.config.logger.error("Error publishing through DB Poller: #{e.message}")
-          sleep(0.5)
-          retry
-        rescue StandardError => e
-          Deimos.config.logger.error("Error publishing through DB poller: #{e.message}}")
-          if retries < @config.retries
-            retries += 1
-            sleep(0.5)
-            retry
-          else
-            Deimos.config.logger.error('Retries exceeded, moving on to next batch')
-            Deimos.config.tracer&.set_error(span, e)
-            self.touch_info(batch)
-            return false
-          end
+        # @return [String]
+        def report
+          "#{batches_processed} batches, #{batches_errored} errored batches, #{messages_processed} processed messages"
         end
-        true
-      end
-
-      # @param batch [Array<ActiveRecord::Base>]
-      # @return [void]
-      def touch_info(batch)
-        record = batch.last
-        id_method = record.class.primary_key
-        last_id = record.public_send(id_method)
-        last_updated_at = last_updated(record)
-        @info.attributes = { last_sent: last_updated_at, last_sent_id: last_id }
-        @info.save!
-      end
-
-      # @param batch [Array<ActiveRecord::Base>]
-      # @return [void]
-      def process_batch(batch)
-        @producer.send_events(batch)
-        self.touch_info(batch)
       end
     end
   end
 end
+
+require 'deimos/utils/db_poller/base'
+require 'deimos/utils/db_poller/time_based'
+require 'deimos/utils/db_poller/state_based'

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.17.1'
+  VERSION = '1.18.0'
 end