deimos-ruby 1.6.1 → 1.8.0.pre.beta1

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.6.1
+  version: 1.8.0.pre.beta1
 platform: ruby
 authors:
 - Daniel Orner
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-04-20 00:00:00.000000000 Z
+date: 2020-07-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: sigurd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.1
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
@@ -95,19 +109,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.9'
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: database_cleaner
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '1.7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '1.7'
 - !ruby/object:Gem::Dependency
   name: ddtrace
   requirement: !ruby/object:Gem::Requirement
@@ -213,6 +227,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.2.4.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -220,6 +237,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.2.4.2
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -304,7 +324,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
-description: 
+description:
 email:
 - daniel.orner@wishabi.com
 executables:
@@ -331,10 +351,15 @@ files:
 - bin/deimos
 - deimos-ruby.gemspec
 - docker-compose.yml
+- docs/ARCHITECTURE.md
 - docs/CONFIGURATION.md
 - docs/DATABASE_BACKEND.md
 - docs/PULL_REQUEST_TEMPLATE.md
 - lib/deimos.rb
+- lib/deimos/active_record_consume/batch_consumption.rb
+- lib/deimos/active_record_consume/batch_slicer.rb
+- lib/deimos/active_record_consume/message_consumption.rb
+- lib/deimos/active_record_consume/schema_model_converter.rb
 - lib/deimos/active_record_consumer.rb
 - lib/deimos/active_record_producer.rb
 - lib/deimos/backends/base.rb
@@ -342,11 +367,12 @@ files:
 - lib/deimos/backends/kafka.rb
 - lib/deimos/backends/kafka_async.rb
 - lib/deimos/backends/test.rb
-- lib/deimos/base_consumer.rb
 - lib/deimos/batch_consumer.rb
 - lib/deimos/config/configurable.rb
 - lib/deimos/config/configuration.rb
 - lib/deimos/config/phobos_config.rb
+- lib/deimos/consume/batch_consumption.rb
+- lib/deimos/consume/message_consumption.rb
 - lib/deimos/consumer.rb
 - lib/deimos/instrumentation.rb
 - lib/deimos/kafka_message.rb
@@ -359,6 +385,7 @@ files:
 - lib/deimos/monkey_patches/phobos_cli.rb
 - lib/deimos/monkey_patches/phobos_producer.rb
 - lib/deimos/monkey_patches/ruby_kafka_heartbeat.rb
+- lib/deimos/poll_info.rb
 - lib/deimos/producer.rb
 - lib/deimos/railtie.rb
 - lib/deimos/schema_backends/avro_base.rb
@@ -373,17 +400,22 @@ files:
 - lib/deimos/tracing/datadog.rb
 - lib/deimos/tracing/mock.rb
 - lib/deimos/tracing/provider.rb
+- lib/deimos/utils/db_poller.rb
 - lib/deimos/utils/db_producer.rb
-- lib/deimos/utils/executor.rb
+- lib/deimos/utils/deadlock_retry.rb
 - lib/deimos/utils/inline_consumer.rb
 - lib/deimos/utils/lag_reporter.rb
-- lib/deimos/utils/platform_schema_validation.rb
-- lib/deimos/utils/signal_handler.rb
 - lib/deimos/version.rb
 - lib/generators/deimos/db_backend/templates/migration
 - lib/generators/deimos/db_backend/templates/rails3_migration
 - lib/generators/deimos/db_backend_generator.rb
+- lib/generators/deimos/db_poller/templates/migration
+- lib/generators/deimos/db_poller/templates/rails3_migration
+- lib/generators/deimos/db_poller_generator.rb
 - lib/tasks/deimos.rake
+- spec/active_record_batch_consumer_spec.rb
+- spec/active_record_consume/batch_slicer_spec.rb
+- spec/active_record_consume/schema_model_converter_spec.rb
 - spec/active_record_consumer_spec.rb
 - spec/active_record_producer_spec.rb
 - spec/backends/base_spec.rb
@@ -399,6 +431,7 @@ files:
 - spec/handlers/my_consumer.rb
 - spec/kafka_source_spec.rb
 - spec/kafka_topic_info_spec.rb
+- spec/message_spec.rb
 - spec/phobos.bad_db.yml
 - spec/phobos.yml
 - spec/producer_spec.rb
@@ -410,18 +443,20 @@ files:
 - spec/schema_backends/base_spec.rb
 - spec/schemas/com/my-namespace/MySchema-key.avsc
 - spec/schemas/com/my-namespace/MySchema.avsc
+- spec/schemas/com/my-namespace/MySchemaCompound-key.avsc
 - spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc
 - spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc
 - spec/schemas/com/my-namespace/MySchemaWithId.avsc
 - spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc
+- spec/schemas/com/my-namespace/Wibble.avsc
 - spec/schemas/com/my-namespace/Widget.avsc
 - spec/schemas/com/my-namespace/WidgetTheSecond.avsc
 - spec/spec_helper.rb
+- spec/utils/db_poller_spec.rb
 - spec/utils/db_producer_spec.rb
-- spec/utils/executor_spec.rb
+- spec/utils/deadlock_retry_spec.rb
 - spec/utils/lag_reporter_spec.rb
 - spec/utils/platform_schema_validation_spec.rb
-- spec/utils/signal_handler_spec.rb
 - support/deimos-solo.png
 - support/deimos-with-name-next.png
 - support/deimos-with-name.png
@@ -430,7 +465,7 @@ homepage: ''
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -441,15 +476,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.1.3
+signing_key:
 specification_version: 4
 summary: Kafka libraries for Ruby.
 test_files:
+- spec/active_record_batch_consumer_spec.rb
+- spec/active_record_consume/batch_slicer_spec.rb
+- spec/active_record_consume/schema_model_converter_spec.rb
 - spec/active_record_consumer_spec.rb
 - spec/active_record_producer_spec.rb
 - spec/backends/base_spec.rb
@@ -465,6 +503,7 @@ test_files:
 - spec/handlers/my_consumer.rb
 - spec/kafka_source_spec.rb
 - spec/kafka_topic_info_spec.rb
+- spec/message_spec.rb
 - spec/phobos.bad_db.yml
 - spec/phobos.yml
 - spec/producer_spec.rb
@@ -476,15 +515,17 @@ test_files:
 - spec/schema_backends/base_spec.rb
 - spec/schemas/com/my-namespace/MySchema-key.avsc
 - spec/schemas/com/my-namespace/MySchema.avsc
+- spec/schemas/com/my-namespace/MySchemaCompound-key.avsc
 - spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc
 - spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc
 - spec/schemas/com/my-namespace/MySchemaWithId.avsc
 - spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc
+- spec/schemas/com/my-namespace/Wibble.avsc
 - spec/schemas/com/my-namespace/Widget.avsc
 - spec/schemas/com/my-namespace/WidgetTheSecond.avsc
 - spec/spec_helper.rb
+- spec/utils/db_poller_spec.rb
 - spec/utils/db_producer_spec.rb
-- spec/utils/executor_spec.rb
+- spec/utils/deadlock_retry_spec.rb
 - spec/utils/lag_reporter_spec.rb
 - spec/utils/platform_schema_validation_spec.rb
-- spec/utils/signal_handler_spec.rb

data/lib/deimos/base_consumer.rb

@@ -1,104 +0,0 @@
-# frozen_string_literal: true
-
-module Deimos
-  # Shared methods for Kafka Consumers
-  class BaseConsumer
-    include SharedConfig
-
-    class << self
-      # @return [Deimos::SchemaBackends::Base]
-      def decoder
-        @decoder ||= Deimos.schema_backend(schema: config[:schema],
-                                           namespace: config[:namespace])
-      end
-
-      # @return [Deimos::SchemaBackends::Base]
-      def key_decoder
-        @key_decoder ||= Deimos.schema_backend(schema: config[:key_schema],
-                                               namespace: config[:namespace])
-      end
-    end
-
-    # Helper method to decode an encoded key.
-    # @param key [String]
-    # @return [Object] the decoded key.
-    def decode_key(key)
-      return nil if key.nil?
-
-      config = self.class.config
-      unless config[:key_configured]
-        raise 'No key config given - if you are not decoding keys, please use '\
-          '`key_config plain: true`'
-      end
-
-      if config[:key_field]
-        self.class.decoder.decode_key(key, config[:key_field])
-      elsif config[:key_schema]
-        self.class.key_decoder.decode(key, schema: config[:key_schema])
-      else # no encoding
-        key
-      end
-    end
-
-  protected
-
-    # @param payload [Hash|String]
-    # @param metadata [Hash]
-    def _with_error_span(payload, metadata)
-      @span = Deimos.config.tracer&.start(
-        'deimos-consumer',
-        resource: self.class.name.gsub('::', '-')
-      )
-      yield
-    rescue StandardError => e
-      _handle_error(e, payload, metadata)
-    ensure
-      Deimos.config.tracer&.finish(@span)
-    end
-
-    def _report_time_delayed(payload, metadata)
-      return if payload.nil? || payload['timestamp'].blank?
-
-      begin
-        time_delayed = Time.now.in_time_zone - payload['timestamp'].to_datetime
-      rescue ArgumentError
-        Deimos.config.logger.info(
-          message: "Error parsing timestamp! #{payload['timestamp']}"
-        )
-        return
-      end
-      Deimos.config.metrics&.histogram('handler', time_delayed, tags: %W(
-                                         time:time_delayed
-                                         topic:#{metadata[:topic]}
-                                       ))
-    end
-
-    # Overrideable method to determine if a given error should be considered
-    # "fatal" and always be reraised.
-    # @param error [Exception]
-    # @param payload [Hash]
-    # @param metadata [Hash]
-    # @return [Boolean]
-    def fatal_error?(_error, _payload, _metadata)
-      false
-    end
-
-    # @param exception [Exception]
-    # @param payload [Hash]
-    # @param metadata [Hash]
-    def _handle_error(exception, payload, metadata)
-      Deimos.config.tracer&.set_error(@span, exception)
-
-      raise if Deimos.config.consumers.reraise_errors ||
-               Deimos.config.consumers.fatal_error&.call(exception, payload, metadata) ||
-               fatal_error?(exception, payload, metadata)
-    end
-
-    # @param _time_taken [Float]
-    # @param _payload [Hash]
-    # @param _metadata [Hash]
-    def _handle_success(_time_taken, _payload, _metadata)
-      raise NotImplementedError
-    end
-  end
-end

data/lib/deimos/utils/executor.rb

@@ -1,124 +0,0 @@
-# frozen_string_literal: true
-
-# rubocop:disable Lint/RescueException
-module Deimos
-  module Utils
-    # Mostly copied from Phobos::Executor. We should DRY this up by putting in a
-    # PR to make it more generic. Might even make sense to move to a separate
-    # gem.
-    class Executor
-      # @return [Array<#start, #stop, #id>]
-      attr_accessor :runners
-
-      # @param runners [Array<#start, #stop, #id>] A list of objects that can be
-      # started or stopped.
-      # @param logger [Logger]
-      # @param sleep_seconds [Integer] Use a fixed time to sleep between
-      # failed runs instead of using an exponential backoff.
-      def initialize(runners, sleep_seconds: nil, logger: Logger.new(STDOUT))
-        @threads = Concurrent::Array.new
-        @runners = runners
-        @logger = logger
-        @sleep_seconds = sleep_seconds
-      end
-
-      # Start the executor.
-      def start
-        @logger.info('Starting executor')
-        @signal_to_stop = false
-        @threads.clear
-        @thread_pool = Concurrent::FixedThreadPool.new(@runners.size)
-
-        @runners.each do |runner|
-          @thread_pool.post do
-            thread = Thread.current
-            thread.abort_on_exception = true
-            @threads << thread
-            run_object(runner)
-          end
-        end
-
-        true
-      end
-
-      # Stop the executor.
-      def stop
-        return if @signal_to_stop
-
-        @logger.info('Stopping executor')
-        @signal_to_stop = true
-        @runners.each(&:stop)
-        @threads.select(&:alive?).each do |thread|
-          begin
-            thread.wakeup
-          rescue StandardError
-            nil
-          end
-        end
-        @thread_pool&.shutdown
-        @thread_pool&.wait_for_termination
-        @logger.info('Executor stopped')
-      end
-
-    private
-
-      # @param exception [Throwable]
-      # @return [Hash]
-      def error_metadata(exception)
-        {
-          exception_class: exception.class.name,
-          exception_message: exception.message,
-          backtrace: exception.backtrace
-        }
-      end
-
-      def run_object(runner)
-        retry_count = 0
-
-        begin
-          @logger.info("Running #{runner.id}")
-          runner.start
-          retry_count = 0 # success - reset retry count
-        rescue Exception => e
-          handle_crashed_runner(runner, e, retry_count)
-          retry_count += 1
-          retry unless @signal_to_stop
-        end
-      rescue Exception => e
-        @logger.error("Failed to run listener (#{e.message}) #{error_metadata(e)}")
-        raise e
-      end
-
-      # @return [ExponentialBackoff]
-      def create_exponential_backoff
-        min = 1
-        max = 60
-        ExponentialBackoff.new(min, max).tap do |backoff|
-          backoff.randomize_factor = rand
-        end
-      end
-
-      # When "runner#start" is interrupted / crashes we assume it's
-      # safe to be called again
-      def handle_crashed_runner(runner, error, retry_count)
-        interval = if @sleep_seconds
-                     @sleep_seconds
-                   else
-                     backoff = create_exponential_backoff
-                     backoff.interval_at(retry_count).round(2)
-                   end
-
-        metadata = {
-          listener_id: runner.id,
-          retry_count: retry_count,
-          waiting_time: interval
-        }.merge(error_metadata(error))
-
-        @logger.error("Runner crashed, waiting #{interval}s (#{error.message}) #{metadata}")
-        sleep(interval)
-      end
-    end
-  end
-end
-
-# rubocop:enable Lint/RescueException