deimos-ruby 1.16.4 → 1.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7412853b7b878c8bcfbfa130f21b5d2c3d46b4d741f1f6adad9edcb358843a4b
-  data.tar.gz: 5672157db0da4ea4a67893d7a792737e858962a2011cd097fc4af23a9bea60f0
+  metadata.gz: 8b4270b2fc345a60377ead079f053ea92281bd606df2d24da782c583bf25e3b9
+  data.tar.gz: 6c824200ea66395439dfbc00a0478a30f1410d10032915fb063f94e0d7efbeb0
 SHA512:
-  metadata.gz: 444d6e14343916242caaa612d1e1c731ce45423ef41dc5e2cf4d9a85e8d618851001d52cf30ef2c2db060b2631bcaa4daa921ddd66eed93636d5d09ac5ad47e2
-  data.tar.gz: 253428dc65fc2d86108671a9e03dd85e9a9afa97681dd562efe2a6ec7d61317002a375705ccc7e0b267da31041a0d4c5b2fce960b7e73195dfdad651c5e77b09
+  metadata.gz: db33a95a08af5e4d5a6e42ea7d8d7193432647afe1c18186d8d4dbf58dca3fa7994cf06c67abda80c841a918746fe4059e72386c6ab2d98080bd1e9aa727e25e
+  data.tar.gz: 8d091dfb4478e66dc898949bc87304dc82a48fc29e1c89e2b1235ea199d338a59c62cc8b29667a03eb45e45db846187feaa28f8f490db3beb17c8e70ddfb7cf0

data/.github/workflows/ci.yml

@@ -25,18 +25,18 @@ jobs:
           ruby-version: 2.7
           bundler-cache: true
 
+      - name: Bundle install
+        run: bundle install --with development
+
       - name: Rubocop
         run: bundle exec rubocop --format progress
 
   build:
     runs-on: ubuntu-latest
-    env:
-      BUNDLE_GEMFILE: ${{ github.workspace }}/.gemfiles/${{ matrix.gemfile }}.gemfile
     strategy:
       fail-fast: false
       matrix:
         ruby: [ '2.6', '2.7', '3.0', '3.1' ]
-        gemfile: [ 'avro_turf-0', 'avro_turf-1' ]
 
     steps:
       - uses: actions/checkout@v2

data/.rubocop.yml

@@ -1,7 +1,9 @@
+inherit_from: .rubocop_todo.yml
+
 require: rubocop-rspec
 
 AllCops:
-  TargetRubyVersion: 2.4
+  TargetRubyVersion: 2.5
   Exclude:
     - lib/deimos/monkey_patches/*.rb
     - vendor/**/*
@@ -39,8 +41,9 @@ Layout/EmptyLinesAroundBlockBody:
 Layout/LineLength:
   Max: 120
   Severity: refactor
-  Exclude:
-  - 'spec/**/*'
+#  TODO: uncomment the files below once rubocop is fixed
+#  Exclude:
+#  - 'spec/**/*'
 
 # foo = if expression
 #         'bar'
@@ -149,18 +152,20 @@ Style/CollectionMethods:
 Style/DateTime:
   AllowCoercion: true
 
-Style/Documentation:
-  Exclude:
-  - 'app/controllers/**/*'
-  - 'app/helpers/**/*'
-  - 'db/**/*'
+# TODO: uncomment the lines below once rubocop is fixed
+#Style/Documentation:
+#  Exclude:
+#  - 'app/controllers/**/*'
+#  - 'app/helpers/**/*'
+#  - 'db/**/*'
 
 # Force documentation for public methods and classes
 Style/DocumentationMethod:
   Enabled: true
-  Exclude:
-  - 'app/controllers/**/*'
-  - 'db/**/*'
+# TODO: uncomment the file below once rubocop is fixed
+#  Exclude:
+#  - 'app/controllers/**/*'
+#  - 'db/**/*'
 
 # Allow else with just nil in it
 Style/EmptyElse:
@@ -231,9 +236,10 @@ Style/MethodCallWithArgsParentheses:
     - before_destroy
     - after_destroy
     - queue_as
-  Exclude:
-    - 'bin/**/*'
-    - 'Gemfile'
+# TODO: uncomment the file below once rubocop is fixed
+#  Exclude:
+#    - 'bin/**/*'
+#    - 'Gemfile'
 
 # Do not allow "end.something"
 Style/MethodCalledOnDoEndBlock:

data/.rubocop_todo.yml

@@ -0,0 +1,364 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config --auto-gen-only-exclude`
+# on 2022-09-08 19:46:52 UTC using RuboCop version 0.90.0.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 1
+# Configuration parameters: Include.
+# Include: **/*.gemspec
+Gemspec/RequiredRubyVersion:
+  Exclude:
+    - 'deimos-ruby.gemspec'
+
+# Offense count: 4
+# Cop supports --auto-correct.
+Layout/EmptyLines:
+  Exclude:
+    - 'spec/schemas/my_namespace/generated.rb'
+    - 'spec/schemas/my_namespace/my_nested_schema.rb'
+    - 'spec/schemas/my_namespace/my_schema_with_circular_reference.rb'
+    - 'spec/schemas/my_namespace/my_schema_with_complex_type.rb'
+
+# Offense count: 32
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle.
+# SupportedStyles: empty_lines, empty_lines_except_namespace, empty_lines_special, no_empty_lines, beginning_only, ending_only
+Layout/EmptyLinesAroundClassBody:
+  Enabled: false
+
+# Offense count: 1
+# Cop supports --auto-correct.
+Layout/EmptyLinesAroundMethodBody:
+  Exclude:
+    - 'lib/generators/deimos/schema_class_generator.rb'
+
+# Offense count: 42
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyleAlignWith, Severity.
+# SupportedStylesAlignWith: keyword, variable, start_of_line
+Layout/EndAlignment:
+  Enabled: false
+
+# Offense count: 4
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, IndentationWidth.
+# SupportedStyles: special_inside_parentheses, consistent, align_braces
+Layout/FirstHashElementIndentation:
+  Exclude:
+    - 'spec/producer_spec.rb'
+
+# Offense count: 19
+# Cop supports --auto-correct.
+# Configuration parameters: AllowMultipleStyles, EnforcedHashRocketStyle, EnforcedColonStyle, EnforcedLastArgumentHashStyle.
+# SupportedHashRocketStyles: key, separator, table
+# SupportedColonStyles: key, separator, table
+# SupportedLastArgumentHashStyles: always_inspect, always_ignore, ignore_implicit, ignore_explicit
+Layout/HashAlignment:
+  Exclude:
+    - 'spec/consumer_spec.rb'
+    - 'spec/generators/schema_class/my_schema_with_circular_reference_spec.rb'
+
+# Offense count: 21
+# Cop supports --auto-correct.
+# Configuration parameters: Width, IgnoredPatterns.
+Layout/IndentationWidth:
+  Enabled: false
+
+# Offense count: 9
+# Cop supports --auto-correct.
+# Configuration parameters: AllowForAlignment, EnforcedStyleForExponentOperator.
+# SupportedStylesForExponentOperator: space, no_space
+Layout/SpaceAroundOperators:
+  Exclude:
+    - 'spec/generators/schema_class/my_schema_with_circular_reference_spec.rb'
+    - 'spec/generators/schema_class_generator_spec.rb'
+    - 'spec/schemas/my_namespace/my_schema_with_complex_type.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, EnforcedStyleForEmptyBraces, SpaceBeforeBlockParameters.
+# SupportedStyles: space, no_space
+# SupportedStylesForEmptyBraces: space, no_space
+Layout/SpaceInsideBlockBraces:
+  Exclude:
+    - 'spec/generators/schema_class_generator_spec.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, EnforcedStyleForEmptyBraces.
+# SupportedStyles: space, no_space, compact
+# SupportedStylesForEmptyBraces: space, no_space
+Layout/SpaceInsideHashLiteralBraces:
+  Exclude:
+    - 'spec/active_record_producer_spec.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle.
+# SupportedStyles: space, no_space
+Layout/SpaceInsideParens:
+  Exclude:
+    - 'lib/deimos.rb'
+
+# Offense count: 23
+# Cop supports --auto-correct.
+# Configuration parameters: AllowInHeredoc.
+Layout/TrailingWhitespace:
+  Exclude:
+    - 'spec/schemas/my_namespace/generated.rb'
+    - 'spec/schemas/my_namespace/my_nested_schema.rb'
+    - 'spec/schemas/my_namespace/my_schema_with_circular_reference.rb'
+    - 'spec/schemas/my_namespace/my_schema_with_complex_type.rb'
+
+# Offense count: 5
+Lint/MissingSuper:
+  Exclude:
+    - 'lib/deimos/active_record_consumer.rb'
+    - 'lib/deimos/metrics/datadog.rb'
+    - 'lib/deimos/schema_class/enum.rb'
+    - 'lib/deimos/tracing/datadog.rb'
+    - 'lib/deimos/tracing/mock.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+Lint/RedundantCopDisableDirective:
+  Exclude:
+    - 'spec/spec_helper.rb'
+
+# Offense count: 3
+# Cop supports --auto-correct.
+# Configuration parameters: AllowUnusedKeywordArguments, IgnoreEmptyMethods, IgnoreNotImplementedMethods.
+Lint/UnusedMethodArgument:
+  Exclude:
+    - 'lib/deimos/tracing/mock.rb'
+    - 'spec/schemas/my_namespace/my_schema_with_circular_reference.rb'
+
+# Offense count: 1
+Lint/UselessAssignment:
+  Exclude:
+    - 'spec/generators/schema_class_generator_spec.rb'
+
+# Offense count: 19
+# Configuration parameters: IgnoredMethods, Max.
+Metrics/AbcSize:
+  Exclude:
+    - 'lib/deimos/active_record_consume/message_consumption.rb'
+    - 'lib/deimos/config/phobos_config.rb'
+    - 'lib/deimos/instrumentation.rb'
+    - 'lib/deimos/kafka_source.rb'
+    - 'lib/deimos/kafka_topic_info.rb'
+    - 'lib/deimos/producer.rb'
+    - 'lib/deimos/schema_backends/avro_schema_coercer.rb'
+    - 'lib/deimos/test_helpers.rb'
+    - 'lib/deimos/utils/db_poller.rb'
+    - 'lib/deimos/utils/db_producer.rb'
+    - 'lib/deimos/utils/inline_consumer.rb'
+    - 'lib/deimos/utils/schema_controller_mixin.rb'
+    - 'lib/generators/deimos/schema_class_generator.rb'
+
+# Offense count: 1
+# Configuration parameters: CountComments, Max, CountAsOne, ExcludedMethods.
+Metrics/MethodLength:
+  Exclude:
+    - 'lib/deimos/config/phobos_config.rb'
+
+# Offense count: 5
+# Configuration parameters: CountComments, Max, CountAsOne.
+Metrics/ModuleLength:
+  Exclude:
+    - 'lib/deimos/test_helpers.rb'
+    - 'spec/active_record_batch_consumer_spec.rb'
+    - 'spec/batch_consumer_spec.rb'
+    - 'spec/kafka_source_spec.rb'
+    - 'spec/producer_spec.rb'
+
+# Offense count: 6
+# Configuration parameters: IgnoredMethods, Max.
+Metrics/PerceivedComplexity:
+  Exclude:
+    - 'lib/deimos/config/phobos_config.rb'
+    - 'lib/deimos/consume/batch_consumption.rb'
+    - 'lib/deimos/kafka_source.rb'
+    - 'lib/deimos/schema_backends/avro_schema_coercer.rb'
+    - 'lib/deimos/test_helpers.rb'
+
+# Offense count: 12
+# Configuration parameters: Prefixes.
+# Prefixes: when, with, without
+RSpec/ContextWording:
+  Exclude:
+    - 'spec/generators/schema_class_generator_spec.rb'
+
+# Offense count: 4
+# Configuration parameters: Max.
+RSpec/ExampleLength:
+  Exclude:
+    - 'spec/config/configuration_spec.rb'
+    - 'spec/utils/db_producer_spec.rb'
+
+# Offense count: 6
+# Configuration parameters: Max.
+RSpec/MultipleExpectations:
+  Exclude:
+    - 'spec/active_record_consumer_spec.rb'
+    - 'spec/config/configuration_spec.rb'
+    - 'spec/kafka_topic_info_spec.rb'
+    - 'spec/utils/db_poller_spec.rb'
+    - 'spec/utils/db_producer_spec.rb'
+
+# Offense count: 1
+Style/Documentation:
+  Exclude:
+    - 'spec/**/*'
+    - 'test/**/*'
+    - 'regenerate_test_schema_classes.rb'
+
+# Offense count: 1
+# Configuration parameters: RequireForNonPublicMethods.
+Style/DocumentationMethod:
+  Exclude:
+    - 'spec/**/*'
+    - 'test/**/*'
+    - 'regenerate_test_schema_classes.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle.
+# SupportedStyles: compact, expanded
+Style/EmptyMethod:
+  Exclude:
+    - 'regenerate_test_schema_classes.rb'
+
+# Offense count: 2
+# Cop supports --auto-correct.
+Style/ExplicitBlockArgument:
+  Exclude:
+    - 'lib/deimos/metrics/datadog.rb'
+    - 'lib/deimos/utils/deadlock_retry.rb'
+
+# Offense count: 2
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle.
+# SupportedStyles: always, always_true, never
+Style/FrozenStringLiteralComment:
+  Exclude:
+    - 'regenerate_test_schema_classes.rb'
+    - 'spec/generators/schema_class/my_schema_spec.rb'
+
+# Offense count: 10
+# Cop supports --auto-correct.
+Style/GlobalStdStream:
+  Exclude:
+    - 'lib/deimos/config/configuration.rb'
+    - 'lib/deimos/config/phobos_config.rb'
+    - 'lib/deimos/metrics/mock.rb'
+    - 'lib/deimos/test_helpers.rb'
+    - 'lib/deimos/tracing/mock.rb'
+    - 'lib/deimos/utils/db_producer.rb'
+    - 'lib/tasks/deimos.rake'
+    - 'spec/rake_spec.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+Style/HashEachMethods:
+  Exclude:
+    - 'lib/generators/deimos/schema_class_generator.rb'
+
+# Offense count: 31
+# Cop supports --auto-correct.
+# Configuration parameters: IgnoreMacros, IgnoredMethods, IgnoredPatterns, IncludedMacros, AllowParenthesesInMultilineCall, AllowParenthesesInChaining, AllowParenthesesInCamelCaseMethod, EnforcedStyle.
+# SupportedStyles: require_parentheses, omit_parentheses
+Style/MethodCallWithArgsParentheses:
+  Exclude:
+    - 'lib/deimos/utils/db_poller.rb'
+    - 'lib/deimos/utils/inline_consumer.rb'
+    - 'spec/config/configuration_spec.rb'
+    - 'spec/deimos_spec.rb'
+    - 'spec/generators/schema_class_generator_spec.rb'
+
+# Offense count: 1
+Style/OptionalBooleanParameter:
+  Exclude:
+    - 'lib/deimos/test_helpers.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+Style/RedundantAssignment:
+  Exclude:
+    - 'spec/schemas/my_namespace/my_schema_with_circular_reference.rb'
+
+# Offense count: 2
+# Cop supports --auto-correct.
+Style/RedundantBegin:
+  Exclude:
+    - 'lib/deimos.rb'
+    - 'lib/deimos/utils/db_producer.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+Style/SingleArgumentDig:
+  Exclude:
+    - 'lib/deimos/active_record_consume/batch_slicer.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: AllowIfMethodIsEmpty.
+Style/SingleLineMethods:
+  Exclude:
+    - 'regenerate_test_schema_classes.rb'
+
+# Offense count: 5
+# Cop supports --auto-correct.
+Style/StringConcatenation:
+  Exclude:
+    - 'spec/producer_spec.rb'
+    - 'spec/utils/schema_controller_mixin_spec.rb'
+
+# Offense count: 10
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, ConsistentQuotesInMultiline.
+# SupportedStyles: single_quotes, double_quotes
+Style/StringLiterals:
+  Exclude:
+    - 'lib/generators/deimos/schema_class_generator.rb'
+    - 'lib/tasks/deimos.rake'
+    - 'regenerate_test_schema_classes.rb'
+    - 'spec/schemas/my_namespace/my_schema_with_complex_type.rb'
+    - 'spec/spec_helper.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, AllowSafeAssignment.
+# SupportedStyles: require_parentheses, require_no_parentheses, require_parentheses_when_complex
+Style/TernaryParentheses:
+  Exclude:
+    - 'lib/deimos/config/phobos_config.rb'
+
+# Offense count: 21
+# Cop supports --auto-correct.
+Style/TrailingBodyOnModule:
+  Enabled: false
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyleForMultiline.
+# SupportedStylesForMultiline: comma, consistent_comma, no_comma
+Style/TrailingCommaInHashLiteral:
+  Exclude:
+    - 'spec/config/configuration_spec.rb'
+
+# Offense count: 8
+# Cop supports --auto-correct.
+# Configuration parameters: AutoCorrect, Max, AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
+# URISchemes: http, https
+Layout/LineLength:
+  Exclude:
+    - 'lib/deimos/producer.rb'
+    - 'lib/deimos/utils/db_poller.rb'
+    - 'lib/deimos/utils/db_producer.rb'
+    - 'spec/active_record_batch_consumer_spec.rb'
+    - 'spec/generators/schema_class/my_schema_with_circular_reference_spec.rb'
+    - 'spec/generators/schema_class/my_schema_with_complex_types_spec.rb'

data/.ruby-version

@@ -1 +1,2 @@
-2.5.3
+2.7.0
+

data/CHANGELOG.md

@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+# 1.17.0 - 2022-10-19
+- Fix the linting step in the CI
+- CHANGE: Add retries to DB poller and bypass "bad batches".
+- Add tracing spans to DB poller production.
+
 # 1.16.4 - 2022-09-09
 
 - Now generates RBS types.

data/Gemfile

@@ -9,4 +9,3 @@ if !ENV['CI'] || ENV['CI'] == ''
   # TODO: once all PRs are merged, add this to gemspec as a development dependency
   gem 'sord', git: 'git@github.com:dorner/sord.git', ref: 'local-develop'
 end
-

data/README.md

@@ -800,6 +800,9 @@ class MyProducer < Deimos::ActiveRecordProducer
 end
 ```
 
+Note that the poller will retry infinitely if it encounters a Kafka-related error such
+as a communication failure. For all other errors, it will retry once by default.
+
 ## Running consumers
 
 Deimos includes a rake task. Once it's in your gemfile, just run
@@ -1200,13 +1203,14 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/flipp-
 
 You can/should re-generate RBS types when methods or classes change by running the following:
 
-    rbs collection install # if you haven't done it recently
+    rbs collection install # if you haven't done it 
+    rbs collection update
     bundle exec sord --hide-private --no-sord-comments sig/defs.rbs --tags 'override:Override'
 
 ### Linting
 
 Deimos uses Rubocop to lint the code. Please run Rubocop on your code 
-before submitting a PR.
+before submitting a PR. The GitHub CI will also run rubocop on your pull request. 
 
 ---
 <p align="center">

data/Steepfile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+target :app do
+  check 'lib'
+  signature 'sig'
+
+  library 'set', 'pathname'
+end

data/deimos-ruby.gemspec

@@ -18,11 +18,11 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
-  spec.add_runtime_dependency('avro_turf', '>= 0.11', '< 2')
+  spec.add_runtime_dependency('avro_turf', '>= 1.4', '< 2')
+  spec.add_runtime_dependency('fig_tree', '~> 0.0.2')
   spec.add_runtime_dependency('phobos', '>= 1.9', '< 3.0')
   spec.add_runtime_dependency('ruby-kafka', '< 2')
   spec.add_runtime_dependency('sigurd', '>= 0.1.0', '< 1.0')
-  spec.add_runtime_dependency('fig_tree', '~> 0.0.2')
 
   spec.add_development_dependency('activerecord-import')
   spec.add_development_dependency('avro', '~> 1.9')
@@ -43,4 +43,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency('rubocop', '0.89.0')
   spec.add_development_dependency('rubocop-rspec', '1.42.0')
   spec.add_development_dependency('sqlite3', '~> 1.3')
+  spec.add_development_dependency('steep', '~> 1.0')
 end

data/docs/CONFIGURATION.md

@@ -15,7 +15,7 @@ Config name|Default|Description
 logger|`Logger.new(STDOUT)`|The logger that Deimos will use.
 phobos_logger|`Deimos.config.logger`|The logger passed to Phobos.
 metrics|`Deimos::Metrics::Mock.new`|The metrics backend use for reporting.
-tracer|`Deimos::Tracer::Mock.new`|The tracer backend used for debugging.
+tracer|`Deimos::Tracing::Mock.new`|The tracer backend used for debugging.
 
 ## Defining Producers
 
@@ -119,6 +119,7 @@ timestamp_column|`:updated_at`|Name of the column to query. Remember to add an i
 delay_time|2|Amount of time in seconds to wait before picking up records, to allow for transactions to finish.
 full_table|false|If set to true, do a full table dump to Kafka each run. Good for very small tables.
 start_from_beginning|true|If false, start from the current time instead of the beginning of time if this is the first time running the poller.
+retries|1|The number of times to retry for a *non-Kafka* error.
 
 ## Kafka Configuration
 

data/lib/deimos/config/configuration.rb

@@ -461,6 +461,8 @@ module Deimos
       # If false, start from the current time instead of the beginning of time
       # if this is the first time running the poller.
       setting :start_from_beginning, true
+      # The number of times to retry production when encountering a *non-Kafka* error.
+      setting :retries, 1
     end
 
     deprecate 'kafka_logger', 'kafka.logger'

data/lib/deimos/metrics/mock.rb

@@ -7,7 +7,7 @@ module Deimos
     # A mock Metrics wrapper which just logs the metrics
     class Mock < Provider
       # @param logger [Logger,nil]
-      def initialize(logger=nil)
+      def initialize(logger=nil) # rubocop:disable Lint/MissingSuper
         @logger = logger || Logger.new(STDOUT)
         @logger.info('MockMetricsProvider initialized')
       end

data/lib/deimos/schema_backends/mock.rb

@@ -24,7 +24,7 @@ module Deimos
       end
 
       # @override
-      def coerce_field(field, value)
+      def coerce_field(_field, value)
         value
       end
 

data/lib/deimos/utils/db_poller.rb

@@ -14,6 +14,9 @@ module Deimos
       # @return [Integer]
       attr_reader :id
 
+      # @return [Hash]
+      attr_reader :config
+
       # Begin the DB Poller process.
       # @return [void]
       def self.start!
@@ -110,6 +113,7 @@ module Deimos
         Deimos.config.logger.info("Polling #{@producer.topic} from #{time_from} to #{time_to}")
         message_count = 0
         batch_count = 0
+        error_count = 0
 
         # poll_query gets all the relevant data from the database, as defined
         # by the producer itself.
@@ -118,12 +122,15 @@ module Deimos
           batch = fetch_results(time_from, time_to).to_a
           break if batch.empty?
 
-          batch_count += 1
-          process_batch(batch)
+          if process_batch_with_span(batch)
+            batch_count += 1
+          else
+            error_count += 1
+          end
           message_count += batch.size
           time_from = last_updated(batch.last)
         end
-        Deimos.config.logger.info("Poll #{@producer.topic} complete at #{time_to} (#{message_count} messages, #{batch_count} batches}")
+        Deimos.config.logger.info("Poll #{@producer.topic} complete at #{time_to} (#{message_count} messages, #{batch_count} successful batches, #{error_count} batches errored}")
       end
 
       # @param time_from [ActiveSupport::TimeWithZone]
@@ -143,15 +150,52 @@ module Deimos
 
       # @param batch [Array<ActiveRecord::Base>]
       # @return [void]
-      def process_batch(batch)
+      def process_batch_with_span(batch)
+        retries = 0
+        begin
+          span = Deimos.config.tracer&.start(
+            'deimos-db-poller',
+            resource: @producer.class.name.gsub('::', '-')
+          )
+          process_batch(batch)
+          Deimos.config.tracer&.finish(span)
+        rescue Kafka::Error => e # keep trying till it fixes itself
+          Deimos.config.logger.error("Error publishing through DB Poller: #{e.message}")
+          sleep(0.5)
+          retry
+        rescue StandardError => e
+          Deimos.config.logger.error("Error publishing through DB poller: #{e.message}}")
+          if retries < @config.retries
+            retries += 1
+            sleep(0.5)
+            retry
+          else
+            Deimos.config.logger.error('Retries exceeded, moving on to next batch')
+            Deimos.config.tracer&.set_error(span, e)
+            self.touch_info(batch)
+            return false
+          end
+        end
+        true
+      end
+
+      # @param batch [Array<ActiveRecord::Base>]
+      # @return [void]
+      def touch_info(batch)
         record = batch.last
         id_method = record.class.primary_key
         last_id = record.public_send(id_method)
         last_updated_at = last_updated(record)
-        @producer.send_events(batch)
         @info.attributes = { last_sent: last_updated_at, last_sent_id: last_id }
         @info.save!
       end
+
+      # @param batch [Array<ActiveRecord::Base>]
+      # @return [void]
+      def process_batch(batch)
+        @producer.send_events(batch)
+        self.touch_info(batch)
+      end
     end
   end
 end