deimos-ruby 1.8.5 → 1.9.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fb39b256f4c8fe4df29db4eb3605b3131d32910b09117d6bed254a06a887d615
-  data.tar.gz: 0d326e5d41ccc69b1da7670ee7a8b57655aae6f434b11365dd33b3297f75f488
+  metadata.gz: 32eb1834570b6a1ea7d61274bf35be7365f16f1bb818c5c1148459bbc5a0b87a
+  data.tar.gz: eb1ea3e922b947c631f05c0e1a41ce65a3c7a366b42e9b5884de499fff14817b
 SHA512:
-  metadata.gz: 3e1f504e2b62f5e6a530ab79ced9e6116954131efe65885fc86fe3148d2fa019f8c6ddc803b723919ba3240dbbbddfa6747d8e8cb4a3dd36bfb534c87fb22870
-  data.tar.gz: efbaa2ca661b9dffbd815efea85da62af3486a4a4770f9b3601a4f1722dd1a995ea61aa7f3e79eb1339b7ecf79ad8532a2a04c6e1e52371987ac757d63a5cb3a
+  metadata.gz: f9d27f061f4ddcc3db165c08aaa09c02d80471cfa8b815cb2fc91909a6992615a49697870f517e70fae958d8bd8fa022b2369310f839c5bcba60a415c9a95750
+  data.tar.gz: '0578c18592f11f646ae89b44a12c712ef86403b282ce9624c7e3127f5d54ba9f20faa634fd463e03b1287dd84d4bfddbecc2ee20e503a76b2f83a700c491b72e'

data/CHANGELOG.md

@@ -7,6 +7,36 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+
+## 1.9.2 - 2021-01-29
+
+- ### Fixes :wrench:
+
+- Fix for `uninitialized constant ActiveSupport::Autoload` in certain circumstances
+- Removed unnecessary monkey patch which was crashing on newer versions of ruby-kafka
+
+## 1.9.0 - 2021-01-28
+
+- ### Roadmap :car:
+
+- Bumped the version of ruby-kafka to latest
+
+- ### Fixes :wrench:
+
+- Prevents DB Poller from reconnecting to DB if there is an open transaction
+- Replaces `before` by `prepend_before` for more consistent test setups.
+- Adds validation in the `kafka_producers` method (fixes [#90](https://github.com/flipp-oss/deimos/issues/90))
+
+## 1.8.7 - 2021-01-14
+
+- ### Roadmap :car:
+- Update Phobos version to allow version 1.9 or 2.x.
+
+## 1.8.6 - 2021-01-14
+
+- ### Fixes :wrench:
+- Fix for configuration bug with Ruby 3.0 (** instead of passing hash)
+
 ## 1.8.5 - 2021-01-13
 
 - ### Fixes :wrench:

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    deimos-ruby (1.8.5)
+    deimos-ruby (1.9.1)
       avro_turf (~> 0.11)
-      phobos (~> 1.9)
-      ruby-kafka (~> 0.7)
+      phobos (>= 1.9, < 3.0)
+      ruby-kafka (< 2)
       sigurd (= 0.0.1)
 
 GEM
@@ -71,7 +71,7 @@ GEM
       minitest (>= 5.1)
       tzinfo (~> 2.0)
       zeitwerk (~> 2.3)
-    ast (2.4.1)
+    ast (2.4.2)
     avro (1.9.2)
       multi_json
     avro_turf (0.11.0)
@@ -79,21 +79,21 @@ GEM
       excon (~> 0.45)
     builder (3.2.4)
     coderay (1.1.3)
-    concurrent-ruby (1.1.6)
-    concurrent-ruby-ext (1.1.6)
-      concurrent-ruby (= 1.1.6)
+    concurrent-ruby (1.1.8)
+    concurrent-ruby-ext (1.1.8)
+      concurrent-ruby (= 1.1.8)
     crass (1.0.6)
     database_cleaner (1.8.5)
-    ddtrace (0.37.0)
+    ddtrace (0.45.0)
       msgpack
     diff-lcs (1.4.4)
     digest-crc (0.6.3)
       rake (>= 12.0.0, < 14.0.0)
-    dogstatsd-ruby (4.8.1)
+    dogstatsd-ruby (4.8.2)
     erubi (1.10.0)
     excon (0.78.1)
     exponential-backoff (0.0.4)
-    ffi (1.13.1)
+    ffi (1.14.2)
     formatador (0.2.5)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
@@ -111,22 +111,22 @@ GEM
       guard (~> 2.1)
       guard-compat (~> 1.1)
       rspec (>= 2.99.0, < 4.0)
-    guard-rubocop (1.3.0)
+    guard-rubocop (1.4.0)
       guard (~> 2.0)
-      rubocop (~> 0.20)
+      rubocop (< 2.0)
     i18n (1.8.7)
       concurrent-ruby (~> 1.0)
-    listen (3.2.1)
+    listen (3.4.1)
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
     little-plugger (1.1.4)
     logging (2.3.0)
       little-plugger (~> 1.1)
       multi_json (~> 1.14)
-    loofah (2.8.0)
+    loofah (2.9.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
-    lumberjack (1.2.6)
+    lumberjack (1.2.8)
     mail (2.7.1)
       mini_mime (>= 0.1.1)
     marcel (0.3.3)
@@ -136,7 +136,7 @@ GEM
     mini_mime (1.0.2)
     mini_portile2 (2.5.0)
     minitest (5.14.3)
-    msgpack (1.3.3)
+    msgpack (1.4.1)
     multi_json (1.15.0)
     mysql2 (0.5.3)
     nenv (0.3.0)
@@ -147,11 +147,11 @@ GEM
     notiffany (0.1.3)
       nenv (~> 0.1)
       shellany (~> 0.0)
-    parallel (1.19.2)
-    parser (2.7.1.4)
+    parallel (1.20.1)
+    parser (3.0.0.0)
       ast (~> 2.4.1)
     pg (1.2.3)
-    phobos (1.9.0)
+    phobos (2.0.2)
       activesupport (>= 3.0.0)
       concurrent-ruby (>= 1.0.2)
       concurrent-ruby-ext (>= 1.0.2)
@@ -197,29 +197,29 @@ GEM
     rb-fsevent (0.10.4)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
-    regexp_parser (1.7.1)
+    regexp_parser (2.0.3)
     rexml (3.2.4)
-    rspec (3.9.0)
-      rspec-core (~> 3.9.0)
-      rspec-expectations (~> 3.9.0)
-      rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.2)
-      rspec-support (~> 3.9.3)
-    rspec-expectations (3.9.2)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.1)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-rails (4.0.1)
+      rspec-support (~> 3.10.0)
+    rspec-rails (4.0.2)
       actionpack (>= 4.2)
       activesupport (>= 4.2)
       railties (>= 4.2)
-      rspec-core (~> 3.9)
-      rspec-expectations (~> 3.9)
-      rspec-mocks (~> 3.9)
-      rspec-support (~> 3.9)
-    rspec-support (3.9.3)
+      rspec-core (~> 3.10)
+      rspec-expectations (~> 3.10)
+      rspec-mocks (~> 3.10)
+      rspec-support (~> 3.10)
+    rspec-support (3.10.1)
     rspec_junit_formatter (0.4.1)
       rspec-core (>= 2, < 4, != 2.12.0)
     rubocop (0.88.0)
@@ -231,13 +231,13 @@ GEM
       rubocop-ast (>= 0.1.0, < 1.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 2.0)
-    rubocop-ast (0.2.0)
-      parser (>= 2.7.0.1)
+    rubocop-ast (0.8.0)
+      parser (>= 2.7.1.5)
     rubocop-rspec (1.42.0)
       rubocop (>= 0.87.0)
-    ruby-kafka (0.7.10)
+    ruby-kafka (1.3.0)
       digest-crc
-    ruby-progressbar (1.10.1)
+    ruby-progressbar (1.11.0)
     shellany (0.0.1)
     sigurd (0.0.1)
       concurrent-ruby (~> 1)
@@ -250,7 +250,7 @@ GEM
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)
     sqlite3 (1.4.2)
-    thor (1.0.1)
+    thor (1.1.0)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
     unicode-display_width (1.7.0)

data/deimos-ruby.gemspec

@@ -19,8 +19,8 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_runtime_dependency('avro_turf', '~> 0.11')
-  spec.add_runtime_dependency('phobos', '~> 1.9')
-  spec.add_runtime_dependency('ruby-kafka', '~> 0.7')
+  spec.add_runtime_dependency('phobos', '>= 1.9', '< 3.0')
+  spec.add_runtime_dependency('ruby-kafka', '< 2')
   spec.add_runtime_dependency('sigurd', '0.0.1')
 
   spec.add_development_dependency('activerecord-import')

data/lib/deimos.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'active_support'
+
 require 'phobos'
 require 'deimos/version'
 require 'deimos/config/configuration'
@@ -18,7 +20,6 @@ require 'deimos/backends/test'
 
 require 'deimos/schema_backends/base'
 
-require 'deimos/monkey_patches/ruby_kafka_heartbeat'
 require 'deimos/monkey_patches/phobos_producer'
 require 'deimos/monkey_patches/phobos_cli'
 

data/lib/deimos/config/configuration.rb

@@ -67,7 +67,7 @@ module Deimos
       topic(kafka_config.topic) if kafka_config.topic.present? && klass.respond_to?(:topic)
       schema(kafka_config.schema) if kafka_config.schema.present?
       namespace(kafka_config.namespace) if kafka_config.namespace.present?
-      key_config(kafka_config.key_config) if kafka_config.key_config.present?
+      key_config(**kafka_config.key_config) if kafka_config.key_config.present?
     end
   end
 

data/lib/deimos/kafka_source.rb

@@ -6,6 +6,9 @@ module Deimos
   module KafkaSource
     extend ActiveSupport::Concern
 
+    DEPRECATION_WARNING = 'The kafka_producer interface will be deprecated ' \
+        'in future releases. Please use kafka_producers instead.'
+
     included do
       after_create(:send_kafka_event_on_create)
       after_update(:send_kafka_event_on_update)
@@ -64,19 +67,12 @@ module Deimos
 
       # @return [Array<Deimos::ActiveRecordProducer>] the producers to run.
       def kafka_producers
-        raise NotImplementedError if self.method(:kafka_producer).
-          owner == Deimos::KafkaSource
-
-        [self.kafka_producer]
-      end
-
-      # Deprecated - use #kafka_producers instead.
-      # @return [Deimos::ActiveRecordProducer] the producer to use.
-      def kafka_producer
-        raise NotImplementedError if self.method(:kafka_producers).
-          owner == Deimos::KafkaSource
+        if self.respond_to?(:kafka_producer)
+          Deimos.config.logger.warn(message: DEPRECATION_WARNING)
+          return [self.kafka_producer]
+        end
 
-        self.kafka_producers.first
+        raise NotImplementedError
       end
 
       # This is an internal method, part of the activerecord_import gem. It's

data/lib/deimos/test_helpers.rb

@@ -34,7 +34,7 @@ module Deimos
           end
         end
 
-        config.before(:each) do
+        config.prepend_before(:each) do
           client = double('client').as_null_object
           allow(client).to receive(:time) do |*_args, &block|
             block.call

data/lib/deimos/utils/db_poller.rb

@@ -68,7 +68,7 @@ module Deimos
 
       # Grab the PollInfo or create if it doesn't exist.
       def retrieve_poll_info
-        ActiveRecord::Base.connection.reconnect!
+        ActiveRecord::Base.connection.reconnect! unless ActiveRecord::Base.connection.open_transactions.positive?
         new_time = @config.start_from_beginning ? Time.new(0) : Time.zone.now
         @info = Deimos::PollInfo.find_by_producer(@config.producer_class) ||
                 Deimos::PollInfo.create!(producer: @config.producer_class,

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.8.5'
+  VERSION = '1.9.2'
 end

data/spec/kafka_source_spec.rb

@@ -308,5 +308,74 @@ module KafkaSourceSpec
                                             }, widgets[1].id)
       end
     end
+
+    context 'with AR models that implement the kafka_producer interface' do
+      before(:each) do
+        # Dummy class we can include the mixin in. Has a backing table created
+        # earlier and has the import hook disabled
+        deprecated_class = Class.new(ActiveRecord::Base) do
+          include Deimos::KafkaSource
+          self.table_name = 'widgets'
+
+          # :nodoc:
+          def self.kafka_config
+            {
+              update: true,
+              delete: true,
+              import: false,
+              create: true
+            }
+          end
+
+          # :nodoc:
+          def self.kafka_producer
+            WidgetProducer
+          end
+        end
+        stub_const('WidgetDeprecated', deprecated_class)
+        WidgetDeprecated.reset_column_information
+      end
+
+      it 'logs a warning and sends the message as usual' do
+        expect(Deimos.config.logger).to receive(:warn).with({ message: WidgetDeprecated::DEPRECATION_WARNING })
+        widget = WidgetDeprecated.create(widget_id: 1, name: 'Widget 1')
+        expect('my-topic').to have_sent({
+                                          widget_id: 1,
+                                          name: 'Widget 1',
+                                          id: widget.id,
+                                          created_at: anything,
+                                          updated_at: anything
+                                        }, widget.id)
+      end
+    end
+
+    context 'with AR models that do not implement any producer interface' do
+      before(:each) do
+        # Dummy class we can include the mixin in. Has a backing table created
+        # earlier and has the import hook disabled
+        buggy_class = Class.new(ActiveRecord::Base) do
+          include Deimos::KafkaSource
+          self.table_name = 'widgets'
+
+          # :nodoc:
+          def self.kafka_config
+            {
+              update: true,
+              delete: true,
+              import: false,
+              create: true
+            }
+          end
+        end
+        stub_const('WidgetBuggy', buggy_class)
+        WidgetBuggy.reset_column_information
+      end
+
+      it 'raises a NotImplementedError exception' do
+        expect {
+          WidgetBuggy.create(widget_id: 1, name: 'Widget 1')
+        }.to raise_error(NotImplementedError)
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.8.5
+  version: 1.9.2
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-01-13 00:00:00.000000000 Z
+date: 2021-01-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf
@@ -28,30 +28,36 @@ dependencies:
   name: phobos
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.9'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.9'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: ruby-kafka
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: sigurd
   requirement: !ruby/object:Gem::Requirement
@@ -380,7 +386,6 @@ files:
 - lib/deimos/metrics/provider.rb
 - lib/deimos/monkey_patches/phobos_cli.rb
 - lib/deimos/monkey_patches/phobos_producer.rb
-- lib/deimos/monkey_patches/ruby_kafka_heartbeat.rb
 - lib/deimos/poll_info.rb
 - lib/deimos/producer.rb
 - lib/deimos/railtie.rb

data/lib/deimos/monkey_patches/ruby_kafka_heartbeat.rb

@@ -1,85 +0,0 @@
-# frozen_string_literal: true
-
-module Kafka
-  class Heartbeat
-    def initialize(group:, interval:, instrumenter:)
-      @group = group
-      @interval = interval
-      @last_heartbeat = Time.now
-      @instrumenter = instrumenter
-    end
-
-    def trigger!
-      @instrumenter.instrument('heartbeat.consumer',
-                               group_id: @group.group_id,
-                               topic_partitions: @group.assigned_partitions) do
-                                 @group.heartbeat
-                                 @last_heartbeat = Time.now
-                               end
-    end
-  end
-
-  class Client
-    def consumer(
-        group_id:,
-        session_timeout: 30,
-        offset_commit_interval: 10,
-        offset_commit_threshold: 0,
-        heartbeat_interval: 10,
-        offset_retention_time: nil,
-        fetcher_max_queue_size: 100
-      )
-      cluster = initialize_cluster
-
-      instrumenter = DecoratingInstrumenter.new(@instrumenter,
-                                                group_id: group_id)
-
-      # The Kafka protocol expects the retention time to be in ms.
-      retention_time = (offset_retention_time && offset_retention_time * 1_000) || -1
-
-      group = ConsumerGroup.new(
-        cluster: cluster,
-        logger: @logger,
-        group_id: group_id,
-        session_timeout: session_timeout,
-        retention_time: retention_time,
-        instrumenter: instrumenter
-      )
-
-      fetcher = Fetcher.new(
-        cluster: initialize_cluster,
-        group: group,
-        logger: @logger,
-        instrumenter: instrumenter,
-        max_queue_size: fetcher_max_queue_size
-      )
-
-      offset_manager = OffsetManager.new(
-        cluster: cluster,
-        group: group,
-        fetcher: fetcher,
-        logger: @logger,
-        commit_interval: offset_commit_interval,
-        commit_threshold: offset_commit_threshold,
-        offset_retention_time: offset_retention_time
-      )
-
-      heartbeat = Heartbeat.new(
-        group: group,
-        interval: heartbeat_interval,
-        instrumenter: instrumenter
-      )
-
-      Consumer.new(
-        cluster: cluster,
-        logger: @logger,
-        instrumenter: instrumenter,
-        group: group,
-        offset_manager: offset_manager,
-        fetcher: fetcher,
-        session_timeout: session_timeout,
-        heartbeat: heartbeat
-      )
-    end
-  end
-end