deimos-ruby 1.7.0.pre.beta1 → 1.8.1.pre.beta3

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.7.0.pre.beta1
+  version: 1.8.1.pre.beta3
 platform: ruby
 authors:
 - Daniel Orner
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-05-12 00:00:00.000000000 Z
+date: 2020-08-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf
@@ -53,19 +53,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.7'
 - !ruby/object:Gem::Dependency
-  name: activerecord
+  name: sigurd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '5.2'
-  type: :development
+        version: 0.0.1
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: 0.0.1
 - !ruby/object:Gem::Dependency
   name: activerecord-import
   requirement: !ruby/object:Gem::Requirement
@@ -94,6 +94,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.9'
+- !ruby/object:Gem::Dependency
+  name: database_cleaner
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
 - !ruby/object:Gem::Dependency
   name: ddtrace
   requirement: !ruby/object:Gem::Requirement
@@ -198,20 +212,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.2'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 5.2.4.2
+        version: '6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.2'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 5.2.4.2
+        version: '6'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -254,6 +262,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: rspec-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
@@ -296,7 +318,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
-description: 
+description:
 email:
 - daniel.orner@wishabi.com
 executables:
@@ -323,10 +345,15 @@ files:
 - bin/deimos
 - deimos-ruby.gemspec
 - docker-compose.yml
+- docs/ARCHITECTURE.md
 - docs/CONFIGURATION.md
 - docs/DATABASE_BACKEND.md
 - docs/PULL_REQUEST_TEMPLATE.md
 - lib/deimos.rb
+- lib/deimos/active_record_consume/batch_consumption.rb
+- lib/deimos/active_record_consume/batch_slicer.rb
+- lib/deimos/active_record_consume/message_consumption.rb
+- lib/deimos/active_record_consume/schema_model_converter.rb
 - lib/deimos/active_record_consumer.rb
 - lib/deimos/active_record_producer.rb
 - lib/deimos/backends/base.rb
@@ -334,11 +361,12 @@ files:
 - lib/deimos/backends/kafka.rb
 - lib/deimos/backends/kafka_async.rb
 - lib/deimos/backends/test.rb
-- lib/deimos/base_consumer.rb
 - lib/deimos/batch_consumer.rb
 - lib/deimos/config/configurable.rb
 - lib/deimos/config/configuration.rb
 - lib/deimos/config/phobos_config.rb
+- lib/deimos/consume/batch_consumption.rb
+- lib/deimos/consume/message_consumption.rb
 - lib/deimos/consumer.rb
 - lib/deimos/instrumentation.rb
 - lib/deimos/kafka_message.rb
@@ -368,12 +396,14 @@ files:
 - lib/deimos/tracing/provider.rb
 - lib/deimos/utils/db_poller.rb
 - lib/deimos/utils/db_producer.rb
-- lib/deimos/utils/executor.rb
+- lib/deimos/utils/deadlock_retry.rb
 - lib/deimos/utils/inline_consumer.rb
 - lib/deimos/utils/lag_reporter.rb
-- lib/deimos/utils/platform_schema_validation.rb
-- lib/deimos/utils/signal_handler.rb
+- lib/deimos/utils/schema_controller_mixin.rb
 - lib/deimos/version.rb
+- lib/generators/deimos/active_record/templates/migration.rb.tt
+- lib/generators/deimos/active_record/templates/model.rb.tt
+- lib/generators/deimos/active_record_generator.rb
 - lib/generators/deimos/db_backend/templates/migration
 - lib/generators/deimos/db_backend/templates/rails3_migration
 - lib/generators/deimos/db_backend_generator.rb
@@ -381,6 +411,9 @@ files:
 - lib/generators/deimos/db_poller/templates/rails3_migration
 - lib/generators/deimos/db_poller_generator.rb
 - lib/tasks/deimos.rake
+- spec/active_record_batch_consumer_spec.rb
+- spec/active_record_consume/batch_slicer_spec.rb
+- spec/active_record_consume/schema_model_converter_spec.rb
 - spec/active_record_consumer_spec.rb
 - spec/active_record_producer_spec.rb
 - spec/backends/base_spec.rb
@@ -392,10 +425,13 @@ files:
 - spec/config/configuration_spec.rb
 - spec/consumer_spec.rb
 - spec/deimos_spec.rb
+- spec/generators/active_record_generator_spec.rb
 - spec/handlers/my_batch_consumer.rb
 - spec/handlers/my_consumer.rb
+- spec/kafka_listener_spec.rb
 - spec/kafka_source_spec.rb
 - spec/kafka_topic_info_spec.rb
+- spec/message_spec.rb
 - spec/phobos.bad_db.yml
 - spec/phobos.yml
 - spec/producer_spec.rb
@@ -405,21 +441,29 @@ files:
 - spec/schema_backends/avro_schema_registry_spec.rb
 - spec/schema_backends/avro_validation_spec.rb
 - spec/schema_backends/base_spec.rb
+- spec/schemas/com/my-namespace/Generated.avsc
+- spec/schemas/com/my-namespace/MyNestedSchema.avsc
 - spec/schemas/com/my-namespace/MySchema-key.avsc
 - spec/schemas/com/my-namespace/MySchema.avsc
+- spec/schemas/com/my-namespace/MySchemaCompound-key.avsc
 - spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc
 - spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc
 - spec/schemas/com/my-namespace/MySchemaWithId.avsc
 - spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc
+- spec/schemas/com/my-namespace/Wibble.avsc
 - spec/schemas/com/my-namespace/Widget.avsc
 - spec/schemas/com/my-namespace/WidgetTheSecond.avsc
+- spec/schemas/com/my-namespace/request/Index.avsc
+- spec/schemas/com/my-namespace/request/UpdateRequest.avsc
+- spec/schemas/com/my-namespace/response/Index.avsc
+- spec/schemas/com/my-namespace/response/UpdateResponse.avsc
 - spec/spec_helper.rb
 - spec/utils/db_poller_spec.rb
 - spec/utils/db_producer_spec.rb
-- spec/utils/executor_spec.rb
+- spec/utils/deadlock_retry_spec.rb
 - spec/utils/lag_reporter_spec.rb
 - spec/utils/platform_schema_validation_spec.rb
-- spec/utils/signal_handler_spec.rb
+- spec/utils/schema_controller_mixin_spec.rb
 - support/deimos-solo.png
 - support/deimos-with-name-next.png
 - support/deimos-with-name.png
@@ -428,7 +472,7 @@ homepage: ''
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -443,12 +487,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
-signing_key: 
+rubygems_version: 3.1.3
+signing_key:
 specification_version: 4
 summary: Kafka libraries for Ruby.
 test_files:
+- spec/active_record_batch_consumer_spec.rb
+- spec/active_record_consume/batch_slicer_spec.rb
+- spec/active_record_consume/schema_model_converter_spec.rb
 - spec/active_record_consumer_spec.rb
 - spec/active_record_producer_spec.rb
 - spec/backends/base_spec.rb
@@ -460,10 +506,13 @@ test_files:
 - spec/config/configuration_spec.rb
 - spec/consumer_spec.rb
 - spec/deimos_spec.rb
+- spec/generators/active_record_generator_spec.rb
 - spec/handlers/my_batch_consumer.rb
 - spec/handlers/my_consumer.rb
+- spec/kafka_listener_spec.rb
 - spec/kafka_source_spec.rb
 - spec/kafka_topic_info_spec.rb
+- spec/message_spec.rb
 - spec/phobos.bad_db.yml
 - spec/phobos.yml
 - spec/producer_spec.rb
@@ -473,18 +522,26 @@ test_files:
 - spec/schema_backends/avro_schema_registry_spec.rb
 - spec/schema_backends/avro_validation_spec.rb
 - spec/schema_backends/base_spec.rb
+- spec/schemas/com/my-namespace/Generated.avsc
+- spec/schemas/com/my-namespace/MyNestedSchema.avsc
 - spec/schemas/com/my-namespace/MySchema-key.avsc
 - spec/schemas/com/my-namespace/MySchema.avsc
+- spec/schemas/com/my-namespace/MySchemaCompound-key.avsc
 - spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc
 - spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc
 - spec/schemas/com/my-namespace/MySchemaWithId.avsc
 - spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc
+- spec/schemas/com/my-namespace/Wibble.avsc
 - spec/schemas/com/my-namespace/Widget.avsc
 - spec/schemas/com/my-namespace/WidgetTheSecond.avsc
+- spec/schemas/com/my-namespace/request/Index.avsc
+- spec/schemas/com/my-namespace/request/UpdateRequest.avsc
+- spec/schemas/com/my-namespace/response/Index.avsc
+- spec/schemas/com/my-namespace/response/UpdateResponse.avsc
 - spec/spec_helper.rb
 - spec/utils/db_poller_spec.rb
 - spec/utils/db_producer_spec.rb
-- spec/utils/executor_spec.rb
+- spec/utils/deadlock_retry_spec.rb
 - spec/utils/lag_reporter_spec.rb
 - spec/utils/platform_schema_validation_spec.rb
-- spec/utils/signal_handler_spec.rb
+- spec/utils/schema_controller_mixin_spec.rb

data/lib/deimos/base_consumer.rb

@@ -1,100 +0,0 @@
-# frozen_string_literal: true
-
-module Deimos
-  # Shared methods for Kafka Consumers
-  class BaseConsumer
-    include SharedConfig
-
-    class << self
-      # @return [Deimos::SchemaBackends::Base]
-      def decoder
-        @decoder ||= Deimos.schema_backend(schema: config[:schema],
-                                           namespace: config[:namespace])
-      end
-
-      # @return [Deimos::SchemaBackends::Base]
-      def key_decoder
-        @key_decoder ||= Deimos.schema_backend(schema: config[:key_schema],
-                                               namespace: config[:namespace])
-      end
-    end
-
-    # Helper method to decode an encoded key.
-    # @param key [String]
-    # @return [Object] the decoded key.
-    def decode_key(key)
-      return nil if key.nil?
-
-      config = self.class.config
-      unless config[:key_configured]
-        raise 'No key config given - if you are not decoding keys, please use '\
-          '`key_config plain: true`'
-      end
-
-      if config[:key_field]
-        self.class.decoder.decode_key(key, config[:key_field])
-      elsif config[:key_schema]
-        self.class.key_decoder.decode(key, schema: config[:key_schema])
-      else # no encoding
-        key
-      end
-    end
-
-  protected
-
-    def _with_span
-      @span = Deimos.config.tracer&.start(
-        'deimos-consumer',
-        resource: self.class.name.gsub('::', '-')
-      )
-      yield
-    ensure
-      Deimos.config.tracer&.finish(@span)
-    end
-
-    def _report_time_delayed(payload, metadata)
-      return if payload.nil? || payload['timestamp'].blank?
-
-      begin
-        time_delayed = Time.now.in_time_zone - payload['timestamp'].to_datetime
-      rescue ArgumentError
-        Deimos.config.logger.info(
-          message: "Error parsing timestamp! #{payload['timestamp']}"
-        )
-        return
-      end
-      Deimos.config.metrics&.histogram('handler', time_delayed, tags: %W(
-                                         time:time_delayed
-                                         topic:#{metadata[:topic]}
-                                       ))
-    end
-
-    # Overrideable method to determine if a given error should be considered
-    # "fatal" and always be reraised.
-    # @param error [Exception]
-    # @param payload [Hash]
-    # @param metadata [Hash]
-    # @return [Boolean]
-    def fatal_error?(_error, _payload, _metadata)
-      false
-    end
-
-    # @param exception [Exception]
-    # @param payload [Hash]
-    # @param metadata [Hash]
-    def _handle_error(exception, payload, metadata)
-      Deimos.config.tracer&.set_error(@span, exception)
-
-      raise if Deimos.config.consumers.reraise_errors ||
-               Deimos.config.consumers.fatal_error&.call(exception, payload, metadata) ||
-               fatal_error?(exception, payload, metadata)
-    end
-
-    # @param _time_taken [Float]
-    # @param _payload [Hash]
-    # @param _metadata [Hash]
-    def _handle_success(_time_taken, _payload, _metadata)
-      raise NotImplementedError
-    end
-  end
-end

data/lib/deimos/utils/executor.rb

@@ -1,124 +0,0 @@
-# frozen_string_literal: true
-
-# rubocop:disable Lint/RescueException
-module Deimos
-  module Utils
-    # Mostly copied from Phobos::Executor. We should DRY this up by putting in a
-    # PR to make it more generic. Might even make sense to move to a separate
-    # gem.
-    class Executor
-      # @return [Array<#start, #stop, #id>]
-      attr_accessor :runners
-
-      # @param runners [Array<#start, #stop, #id>] A list of objects that can be
-      # started or stopped.
-      # @param logger [Logger]
-      # @param sleep_seconds [Integer] Use a fixed time to sleep between
-      # failed runs instead of using an exponential backoff.
-      def initialize(runners, sleep_seconds: nil, logger: Logger.new(STDOUT))
-        @threads = Concurrent::Array.new
-        @runners = runners
-        @logger = logger
-        @sleep_seconds = sleep_seconds
-      end
-
-      # Start the executor.
-      def start
-        @logger.info('Starting executor')
-        @signal_to_stop = false
-        @threads.clear
-        @thread_pool = Concurrent::FixedThreadPool.new(@runners.size)
-
-        @runners.each do |runner|
-          @thread_pool.post do
-            thread = Thread.current
-            thread.abort_on_exception = true
-            @threads << thread
-            run_object(runner)
-          end
-        end
-
-        true
-      end
-
-      # Stop the executor.
-      def stop
-        return if @signal_to_stop
-
-        @logger.info('Stopping executor')
-        @signal_to_stop = true
-        @runners.each(&:stop)
-        @threads.select(&:alive?).each do |thread|
-          begin
-            thread.wakeup
-          rescue StandardError
-            nil
-          end
-        end
-        @thread_pool&.shutdown
-        @thread_pool&.wait_for_termination
-        @logger.info('Executor stopped')
-      end
-
-    private
-
-      # @param exception [Throwable]
-      # @return [Hash]
-      def error_metadata(exception)
-        {
-          exception_class: exception.class.name,
-          exception_message: exception.message,
-          backtrace: exception.backtrace
-        }
-      end
-
-      def run_object(runner)
-        retry_count = 0
-
-        begin
-          @logger.info("Running #{runner.id}")
-          runner.start
-          retry_count = 0 # success - reset retry count
-        rescue Exception => e
-          handle_crashed_runner(runner, e, retry_count)
-          retry_count += 1
-          retry unless @signal_to_stop
-        end
-      rescue Exception => e
-        @logger.error("Failed to run executor (#{e.message}) #{error_metadata(e)}")
-        raise e
-      end
-
-      # @return [ExponentialBackoff]
-      def create_exponential_backoff
-        min = 1
-        max = 60
-        ExponentialBackoff.new(min, max).tap do |backoff|
-          backoff.randomize_factor = rand
-        end
-      end
-
-      # When "runner#start" is interrupted / crashes we assume it's
-      # safe to be called again
-      def handle_crashed_runner(runner, error, retry_count)
-        interval = if @sleep_seconds
-                     @sleep_seconds
-                   else
-                     backoff = create_exponential_backoff
-                     backoff.interval_at(retry_count).round(2)
-                   end
-
-        metadata = {
-          listener_id: runner.id,
-          retry_count: retry_count,
-          waiting_time: interval
-        }.merge(error_metadata(error))
-
-        @logger.error("Runner crashed, waiting #{interval}s (#{error.message}) #{metadata}")
-        sleep(interval)
-      end
-    end
-  end
-end
-
-# rubocop:enable Lint/RescueException