deimos-ruby 1.8.6 → 1.10.0

data/lib/deimos/active_record_consume/message_consumption.rb

@@ -58,6 +58,11 @@ module Deimos
         attrs.each do |k, v|
           record.send("#{k}=", v)
         end
+        save_record(record)
+      end
+
+      # @param record [ActiveRecord::Base]
+      def save_record(record)
         record.created_at ||= Time.zone.now if record.respond_to?(:created_at)
         record.updated_at = Time.zone.now if record.respond_to?(:updated_at)
         record.save!

data/lib/deimos/config/configuration.rb

@@ -1,17 +1,17 @@
 # frozen_string_literal: true
 
+require 'fig_tree'
 require_relative 'phobos_config'
-require_relative 'configurable'
 require_relative '../metrics/mock'
 require_relative '../tracing/mock'
 require 'active_support/core_ext/numeric'
 
 # :nodoc:
 module Deimos
-  include Configurable
+  include FigTree
 
   # :nodoc:
-  class Configurable::ConfigStruct
+  class FigTree::ConfigStruct
     include Deimos::PhobosConfig
   end
 
@@ -60,7 +60,7 @@ module Deimos
     end
   end
 
-  # @param kafka_config [Configurable::ConfigStruct]
+  # @param kafka_config [FigTree::ConfigStruct]
   def self.configure_producer_or_consumer(kafka_config)
     klass = kafka_config.class_name.constantize
     klass.class_eval do
@@ -327,19 +327,19 @@ module Deimos
       # These are the phobos "listener" configs. See CONFIGURATION.md for more
       # info.
       setting :group_id
-      setting :max_concurrency
-      setting :start_from_beginning
+      setting :max_concurrency, 1
+      setting :start_from_beginning, true
       setting :max_bytes_per_partition, 500.kilobytes
-      setting :min_bytes
-      setting :max_wait_time
+      setting :min_bytes, 1
+      setting :max_wait_time, 5
       setting :force_encoding
-      setting :delivery
+      setting :delivery, :batch
       setting :backoff
-      setting :session_timeout
-      setting :offset_commit_interval
-      setting :offset_commit_threshold
+      setting :session_timeout, 300
+      setting :offset_commit_interval, 10
+      setting :offset_commit_threshold, 0
       setting :offset_retention_time
-      setting :heartbeat_interval
+      setting :heartbeat_interval, 10
     end
 
     setting_object :db_poller do

data/lib/deimos/kafka_source.rb

@@ -6,6 +6,9 @@ module Deimos
   module KafkaSource
     extend ActiveSupport::Concern
 
+    DEPRECATION_WARNING = 'The kafka_producer interface will be deprecated ' \
+        'in future releases. Please use kafka_producers instead.'
+
     included do
       after_create(:send_kafka_event_on_create)
       after_update(:send_kafka_event_on_update)
@@ -64,19 +67,12 @@ module Deimos
 
       # @return [Array<Deimos::ActiveRecordProducer>] the producers to run.
       def kafka_producers
-        raise NotImplementedError if self.method(:kafka_producer).
-          owner == Deimos::KafkaSource
-
-        [self.kafka_producer]
-      end
-
-      # Deprecated - use #kafka_producers instead.
-      # @return [Deimos::ActiveRecordProducer] the producer to use.
-      def kafka_producer
-        raise NotImplementedError if self.method(:kafka_producers).
-          owner == Deimos::KafkaSource
+        if self.respond_to?(:kafka_producer)
+          Deimos.config.logger.warn(message: DEPRECATION_WARNING)
+          return [self.kafka_producer]
+        end
 
-        self.kafka_producers.first
+        raise NotImplementedError
       end
 
       # This is an internal method, part of the activerecord_import gem. It's

data/lib/deimos/test_helpers.rb

@@ -34,7 +34,7 @@ module Deimos
           end
         end
 
-        config.before(:each) do
+        config.prepend_before(:each) do
           client = double('client').as_null_object
           allow(client).to receive(:time) do |*_args, &block|
             block.call

data/lib/deimos/utils/db_poller.rb

@@ -68,7 +68,7 @@ module Deimos
 
       # Grab the PollInfo or create if it doesn't exist.
       def retrieve_poll_info
-        ActiveRecord::Base.connection.reconnect!
+        ActiveRecord::Base.connection.reconnect! unless ActiveRecord::Base.connection.open_transactions.positive?
         new_time = @config.start_from_beginning ? Time.new(0) : Time.zone.now
         @info = Deimos::PollInfo.find_by_producer(@config.producer_class) ||
                 Deimos::PollInfo.create!(producer: @config.producer_class,

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.8.6'
+  VERSION = '1.10.0'
 end

data/spec/config/configuration_spec.rb

@@ -74,34 +74,34 @@ describe Deimos, 'configuration' do
         {
           topic: 'my_consume_topic',
           group_id: 'my_group_id',
-          max_concurrency: nil,
-          start_from_beginning: nil,
+          max_concurrency: 1,
+          start_from_beginning: true,
           max_bytes_per_partition: 524_288,
-          min_bytes: nil,
-          max_wait_time: nil,
+          min_bytes: 1,
+          max_wait_time: 5,
           force_encoding: nil,
-          delivery: nil,
-          session_timeout: nil,
-          offset_commit_interval: nil,
-          offset_commit_threshold: nil,
+          delivery: 'batch',
+          session_timeout: 300,
+          offset_commit_interval: 10,
+          offset_commit_threshold: 0,
           offset_retention_time: nil,
-          heartbeat_interval: nil,
+          heartbeat_interval: 10,
           handler: 'ConsumerTest::MyConsumer'
         }, {
           topic: 'my_batch_consume_topic',
           group_id: 'my_batch_group_id',
-          max_concurrency: nil,
-          start_from_beginning: nil,
+          max_concurrency: 1,
+          start_from_beginning: true,
           max_bytes_per_partition: 500.kilobytes,
-          min_bytes: nil,
-          max_wait_time: nil,
+          min_bytes: 1,
+          max_wait_time: 5,
           force_encoding: nil,
           delivery: 'inline_batch',
-          session_timeout: nil,
-          offset_commit_interval: nil,
-          offset_commit_threshold: nil,
+          session_timeout: 300,
+          offset_commit_interval: 10,
+          offset_commit_threshold: 0,
           offset_retention_time: nil,
-          heartbeat_interval: nil,
+          heartbeat_interval: 10,
           handler: 'ConsumerTest::MyBatchConsumer'
         }
       ],

data/spec/kafka_source_spec.rb

@@ -308,5 +308,74 @@ module KafkaSourceSpec
                                             }, widgets[1].id)
       end
     end
+
+    context 'with AR models that implement the kafka_producer interface' do
+      before(:each) do
+        # Dummy class we can include the mixin in. Has a backing table created
+        # earlier and has the import hook disabled
+        deprecated_class = Class.new(ActiveRecord::Base) do
+          include Deimos::KafkaSource
+          self.table_name = 'widgets'
+
+          # :nodoc:
+          def self.kafka_config
+            {
+              update: true,
+              delete: true,
+              import: false,
+              create: true
+            }
+          end
+
+          # :nodoc:
+          def self.kafka_producer
+            WidgetProducer
+          end
+        end
+        stub_const('WidgetDeprecated', deprecated_class)
+        WidgetDeprecated.reset_column_information
+      end
+
+      it 'logs a warning and sends the message as usual' do
+        expect(Deimos.config.logger).to receive(:warn).with({ message: WidgetDeprecated::DEPRECATION_WARNING })
+        widget = WidgetDeprecated.create(widget_id: 1, name: 'Widget 1')
+        expect('my-topic').to have_sent({
+                                          widget_id: 1,
+                                          name: 'Widget 1',
+                                          id: widget.id,
+                                          created_at: anything,
+                                          updated_at: anything
+                                        }, widget.id)
+      end
+    end
+
+    context 'with AR models that do not implement any producer interface' do
+      before(:each) do
+        # Dummy class we can include the mixin in. Has a backing table created
+        # earlier and has the import hook disabled
+        buggy_class = Class.new(ActiveRecord::Base) do
+          include Deimos::KafkaSource
+          self.table_name = 'widgets'
+
+          # :nodoc:
+          def self.kafka_config
+            {
+              update: true,
+              delete: true,
+              import: false,
+              create: true
+            }
+          end
+        end
+        stub_const('WidgetBuggy', buggy_class)
+        WidgetBuggy.reset_column_information
+      end
+
+      it 'raises a NotImplementedError exception' do
+        expect {
+          WidgetBuggy.create(widget_id: 1, name: 'Widget 1')
+        }.to raise_error(NotImplementedError)
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.8.6
+  version: 1.10.0
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-01-14 00:00:00.000000000 Z
+date: 2021-03-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf
@@ -28,44 +28,64 @@ dependencies:
   name: phobos
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.9'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.9'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: ruby-kafka
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: sigurd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.0.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.0.1
+- !ruby/object:Gem::Dependency
+  name: fig_tree
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.2
 - !ruby/object:Gem::Dependency
   name: activerecord-import
   requirement: !ruby/object:Gem::Requirement
@@ -364,7 +384,6 @@ files:
 - lib/deimos/backends/kafka_async.rb
 - lib/deimos/backends/test.rb
 - lib/deimos/batch_consumer.rb
-- lib/deimos/config/configurable.rb
 - lib/deimos/config/configuration.rb
 - lib/deimos/config/phobos_config.rb
 - lib/deimos/consume/batch_consumption.rb
@@ -380,7 +399,6 @@ files:
 - lib/deimos/metrics/provider.rb
 - lib/deimos/monkey_patches/phobos_cli.rb
 - lib/deimos/monkey_patches/phobos_producer.rb
-- lib/deimos/monkey_patches/ruby_kafka_heartbeat.rb
 - lib/deimos/poll_info.rb
 - lib/deimos/producer.rb
 - lib/deimos/railtie.rb
@@ -423,7 +441,6 @@ files:
 - spec/backends/kafka_async_spec.rb
 - spec/backends/kafka_spec.rb
 - spec/batch_consumer_spec.rb
-- spec/config/configurable_spec.rb
 - spec/config/configuration_spec.rb
 - spec/consumer_spec.rb
 - spec/deimos_spec.rb
@@ -507,7 +524,6 @@ test_files:
 - spec/backends/kafka_async_spec.rb
 - spec/backends/kafka_spec.rb
 - spec/batch_consumer_spec.rb
-- spec/config/configurable_spec.rb
 - spec/config/configuration_spec.rb
 - spec/consumer_spec.rb
 - spec/deimos_spec.rb

data/lib/deimos/config/configurable.rb

@@ -1,278 +0,0 @@
-# frozen_string_literal: true
-
-require 'active_support/concern'
-require 'active_support/callbacks'
-
-module Deimos
-  # Module to allow configuration. Loosely based off of the dry-configuration
-  # gem but with several advantages:
-  # - Works with Ruby 2.3.
-  # - More succinct syntax using method_missing so you do not need to write
-  #   "config.whatever" and can just write "whatever".
-  # - Use nested blocks:
-  #   Deimos.configure do
-  #     config.kafka.ssl do
-  #       enabled true
-  #       ca_cert_file 'my_file'
-  #     end
-  #     config.kafka do
-  #       ssl do
-  #         enabled true
-  #       end
-  #     end
-  #   end
-  # - Allows for arrays of configurations:
-  #   Deimos.configure do |config|
-  #     config.producer do
-  #       class_name 'MyProducer'
-  #       topic 'MyTopic'
-  #     end
-  #   end
-  # - Allows to call `configure` multiple times without crashing.
-  # - Allows to lazy-set default values by passing a proc as a default:
-  #  Deimos.define_settings do |config|
-  #   setting :my_val, default_proc: proc { MyDefault.calculated_value }
-  #  end
-  # - Support for setting up and automatically calling deprecated configurations.
-  # - Support for configuration callbacks.
-  module Configurable
-    extend ActiveSupport::Concern
-
-    ConfigValue = Struct.new(:value, :default_value, :default_proc, :deprecation) do
-
-      # Reset value back to default.
-      def reset!
-        if self.value.is_a?(ConfigStruct)
-          self.value.reset!
-        else
-          self.value = self.default_value
-        end
-      end
-
-      # :nodoc:
-      def clone_and_reset
-        setting = ConfigValue.new(self.value, self.default_value,
-                                  self.default_proc, self.deprecation)
-        setting.reset!
-        setting
-      end
-
-    end
-
-    # Class that defines and keeps the configuration values.
-    class ConfigStruct
-      include ActiveSupport::Callbacks
-
-      define_callbacks :configure
-
-      # @param name [String]
-      def initialize(name)
-        @name = name
-        @settings = {}
-        @setting_objects = {}
-        @setting_templates = {}
-      end
-
-      # Reset config back to default values.
-      def reset!
-        @setting_objects = @setting_templates.map { |k, _| [k, []] }.to_h
-        @settings.values.each(&:reset!)
-      end
-
-      # Mark a configuration as deprecated and replaced with the new config.
-      # @param old_config [String]
-      # @param new_config [String]
-      def deprecate(old_config, new_config)
-        @settings[old_config.to_sym] ||= ConfigValue.new
-        @settings[old_config.to_sym].deprecation = new_config
-      end
-
-      # :nodoc:
-      def inspect
-        "#{@name}: #{@settings.inspect} #{@setting_objects.inspect}"
-      end
-
-      # @return [Hash]
-      def to_h
-        @settings.map { |k, v| [k, v.value] }.to_h
-      end
-
-      # :nodoc:
-      def clone_and_reset
-        new_config = self.clone
-        new_config.setting_objects = new_config.setting_objects.clone
-        new_config.settings = new_config.settings.map { |k, v| [k, v.clone_and_reset] }.to_h
-        new_config
-      end
-
-      # Define a setting template for an array of objects via a block:
-      #   setting_object :producer do
-      #     setting :topic
-      #     setting :class_name
-      #   end
-      # This will create the `producer` method to define these values as well
-      # as the `producer_objects` method to retrieve them.
-      # @param name [Symbol]
-      def setting_object(name, &block)
-        new_config = ConfigStruct.new("#{@name}.#{name}")
-        @setting_objects[name] = []
-        @setting_templates[name] = new_config
-        new_config.instance_eval(&block)
-      end
-
-      # Define a setting with the given name.
-      # @param name [Symbol]
-      # @param default_value [Object]
-      # @param default_proc [Proc]
-      def setting(name, default_value=nil, default_proc: nil, &block)
-        if block_given?
-          # Create a nested setting
-          setting_config = @settings[name]&.value || ConfigStruct.new("#{@name}.#{name}")
-          setting = ConfigValue.new
-          setting.value = setting_config
-          @settings[name] = setting
-          setting_config.instance_eval(&block)
-        else
-          setting = ConfigValue.new
-          setting.default_proc = default_proc
-          setting.default_value = default_value
-          setting.reset!
-          @settings[name] = setting
-        end
-      end
-
-      # :nodoc:
-      def respond_to_missing?(method, include_all=true)
-        method = method.to_s.sub(/=$/, '')
-        method.ends_with?('objects') ||
-          @setting_templates.key?(method.to_sym) ||
-          @settings.key?(method.to_sym) ||
-          super
-      end
-
-      # :nodoc:
-      def method_missing(method, *args, &block)
-        config_key = method.to_s.sub(/=$/, '').to_sym
-
-        # Return the list of setting objects with the given name
-        if config_key.to_s.end_with?('objects')
-          return _setting_object_method(config_key)
-        end
-
-        # Define a new setting object with the given name
-        if @setting_templates.key?(config_key) && block_given?
-          return _new_setting_object_method(config_key, &block)
-        end
-
-        setting = @settings[config_key]
-
-        if setting&.deprecation
-          return _deprecated_config_method(method, *args)
-        end
-
-        return super unless setting
-
-        if block_given?
-          return _block_config_method(config_key, &block)
-        end
-
-        _default_config_method(config_key, *args)
-      end
-
-    protected
-
-      # Only for the clone method
-      attr_accessor :settings, :setting_objects
-
-    private
-
-      def _deprecated_config_method(method, *args)
-        config_key = method.to_s.sub(/=$/, '').to_sym
-        new_config = @settings[config_key].deprecation
-        equals = method.to_s.end_with?('=') ? '=' : ''
-        ActiveSupport::Deprecation.warn("config.#{config_key}#{equals} is deprecated - use config.#{new_config}#{equals}")
-        obj = self
-        messages = new_config.split('.')
-        messages[0..-2].each do |message|
-          obj = obj.send(message)
-        end
-        if args.length.positive?
-          obj.send(messages[-1], args[0])
-        else
-          obj.send(messages[-1])
-        end
-      end
-
-      # Get or set a value.
-      def _default_config_method(config_key, *args)
-        if args.length.positive?
-          # Set the value
-          @settings[config_key].value = args[0]
-        else
-          # Get the value
-          setting = @settings[config_key]
-          if setting.default_proc && setting.value.nil?
-            setting.value = setting.default_proc.call
-          end
-          setting.value
-        end
-      end
-
-      # Define a new setting object and use the passed block to define values.
-      def _new_setting_object_method(config_key, &block)
-        new_config = @setting_templates[config_key].clone_and_reset
-        new_config.instance_eval(&block)
-        @setting_objects[config_key] << new_config
-      end
-
-      # Return a setting object.
-      def _setting_object_method(config_key)
-        key = config_key.to_s.sub(/_objects$/, '').to_sym
-        @setting_objects[key]
-      end
-
-      # Define new values inside a block.
-      def _block_config_method(config_key, &block)
-        unless @settings[config_key].value.is_a?(ConfigStruct)
-          raise "Block called for #{config_key} but it is not a nested config!"
-        end
-
-        @settings[config_key].value.instance_eval(&block)
-      end
-    end
-
-    # :nodoc:
-    module ClassMethods
-      # Define and redefine settings.
-      def define_settings(&block)
-        config.instance_eval(&block)
-      end
-
-      # Configure the settings with values.
-      def configure(&block)
-        if defined?(Rake) && defined?(Rake.application)
-          tasks = Rake.application.top_level_tasks
-          if tasks.any? { |t| %w(assets webpacker yarn).include?(t.split(':').first) }
-            puts 'Skipping Deimos configuration since we are in JS/CSS compilation'
-            return
-          end
-        end
-        config.run_callbacks(:configure) do
-          config.instance_eval(&block)
-        end
-      end
-
-      # @return [ConfigStruct]
-      def config
-        @config ||= ConfigStruct.new('config')
-      end
-
-      # Pass a block to run after configuration is done.
-      def after_configure(&block)
-        mod = self
-        config.class.set_callback(:configure, :after,
-                                  proc { mod.instance_eval(&block) })
-      end
-    end
-  end
-end