deimos-ruby 2.0.0.pre.alpha2 → 2.0.0.pre.alpha4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c7563872db1d29eb0e6873eec799fe173ceb7484ec8e728063e49cacd3dc3083
-  data.tar.gz: d768d9026137b265887c3e0a063c9737974338de2a460c2c747440ba6d2ea94c
+  metadata.gz: 13ff4c1182fe82aa5e41b6981480e463588c64803b060d1ea68aeeded001b945
+  data.tar.gz: 3cd1ff2eac1aa981aeef0c15d399850ffbfb13a71a3315476f62494eaea16f09
 SHA512:
-  metadata.gz: 76c9cb88a3e130eedcdaa45e4ad5cdf9cbed0bdf3c61e68bde81358785f250429ad1c1ffbaaaa520e739f0e0ef3c16fd8a0f5422e706c3178267600456bf9930
-  data.tar.gz: b0a531fdbd9b1ec9276f44783418e8df5fc358c6c3c2e1e51f6265b6e664cdf4fb83ad33d631f2a3ebb465aba52724d8941a7e97fbf49632f9299bd9249d7928
+  metadata.gz: '0389849fa59d727f6b44d850874e4b6a694e856b9dbfa1624282b77f6113ae31a208f3f41dae677b4563add662b7232462575059b05867298fd40440255d5e3d'
+  data.tar.gz: f03cd3234d22da82ae55976478023b90353823cceea9440aa805cc8994a527867ca1af074c6df2eb99a96f9a0efaa14a8da5bd10a00f5d0bf01c0bf746687333

data/Gemfile

@@ -4,3 +4,4 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in boilerplate.gemspec
 gemspec
+gem 'karafka', git: 'https://github.com/dorner/karafka', branch: 'redraw'

data/docs/CONFIGURATION.md

@@ -86,21 +86,22 @@ Deimos.configure do
 end
 ```
 
-| Config name              | Default       | Description                                                                                                                           |
-|--------------------------|---------------|---------------------------------------------------------------------------------------------------------------------------------------|
-| producer_class           | nil           | ActiveRecordProducer class to use for sending messages.                                                                               |
-| mode                     | :time_based   | Whether to use time-based polling or state-based polling.                                                                             |
-| run_every                | 60            | Amount of time in seconds to wait between runs.                                                                                       |
+| Config name              | Default      | Description                                                                                                                           |
+|--------------------------|--------------|---------------------------------------------------------------------------------------------------------------------------------------|
+| producer_class           | nil          | ActiveRecordProducer class to use for sending messages.                                                                               |
+| producer_classes         | []           | Array of ActiveRecordProducer classes to use for sending messages. You can use this instead of `producer_class`.                      |
+| mode                     | :time_based  | Whether to use time-based polling or state-based polling.                                                                             |
+| run_every                | 60           | Amount of time in seconds to wait between runs.                                                                                       |
 | timestamp_column         | `:updated_at` | Name of the column to query. Remember to add an index to this column!                                                                 |
-| delay_time               | 2             | Amount of time in seconds to wait before picking up records, to allow for transactions to finish.                                     |
-| retries                  | 1             | The number of times to retry for a *non-Kafka* error.                                                                                 |
-| full_table               | false         | If set to true, do a full table dump to Kafka each run. Good for very small tables. Time-based only.                                  |
-| start_from_beginning     | true          | If false, start from the current time instead of the beginning of time if this is the first time running the poller. Time-based only. |
-| state_column             | nil           | If set, this represents the DB column to use to update publishing status. State-based only.                                           |
-| publish_timestamp_column | nil           | If set, this represents the DB column to use to update when publishing is done. State-based only.                                     |
-| published_state          | nil           | If set, the poller will update the `state_column` to this value when publishing succeeds. State-based only.                           |
-| failed_state             | nil           | If set, the poller will update the `state_column` to this value when publishing fails. State-based only.                              |
-| poller_class             | nil           | Poller subclass name to use for publishing to multiple kafka topics from a single poller.                                             |
+| delay_time               | 2            | Amount of time in seconds to wait before picking up records, to allow for transactions to finish.                                     |
+| retries                  | 1            | The number of times to retry for a *non-Kafka* error.                                                                                 |
+| full_table               | false        | If set to true, do a full table dump to Kafka each run. Good for very small tables. Time-based only.                                  |
+| start_from_beginning     | true         | If false, start from the current time instead of the beginning of time if this is the first time running the poller. Time-based only. |
+| state_column             | nil          | If set, this represents the DB column to use to update publishing status. State-based only.                                           |
+| publish_timestamp_column | nil          | If set, this represents the DB column to use to update when publishing is done. State-based only.                                     |
+| published_state          | nil          | If set, the poller will update the `state_column` to this value when publishing succeeds. State-based only.                           |
+| failed_state             | nil          | If set, the poller will update the `state_column` to this value when publishing fails. State-based only.                              |
+| poller_class             | nil          | Poller subclass name to use for publishing to multiple kafka topics from a single poller.                                             |
 
 ## Karafka Routing
 

data/docs/UPGRADING.md

@@ -216,6 +216,7 @@ The following events have been **renamed**:
 
 ### Additional breaking changes
 * `key_config` now defaults to `{none: true}` instead of erroring out if not set.
+* `reraise_errors` now defaults to true if the Rails env is set to `test`, and false otherwise.
 * `fatal_error?` now receives a Karafka `messages` object instead of a payload hash or array of hashes.
 * `watched_attributes` has been moved from the corresponding ActiveRecord class to the ActiveRecordProducer class. The object being watched is passed into the method.
 * Removed `TestHelpers.full_integration_test!` and `kafka_test!` as Karafka does not currently support these use cases. If we need them back, we will need to put in changes to the testing library to support them.

data/lib/deimos/active_record_producer.rb

@@ -53,7 +53,7 @@ module Deimos
       end
 
       def config
-        Deimos.karafka_configs.find { |t| t.producer_class == self }
+        Deimos.karafka_configs.find { |t| t.producer_classes.include?(self) }
       end
 
       def encoder

data/lib/deimos/ext/consumer_route.rb

@@ -19,10 +19,11 @@ module Deimos
             bulk_import_id_column: :bulk_import_id,
             replace_associations: true,
             each_message: false,
+            reraise_errors: Rails.env.test?,
             bulk_import_id_generator: proc { SecureRandom.uuid },
             fatal_error: proc { false }
           )
-          if args.any?
+          if args.size.positive?
             @deimos_config.public_send("#{field}=", args[0])
           end
           @deimos_config[field]

data/lib/deimos/ext/producer_route.rb

@@ -1,18 +1,26 @@
 module Deimos
   class ProducerRoute < Karafka::Routing::Features::Base
-    FIELDS = %i(producer_class payload_log disabled)
+    FIELDS = %i(producer_classes payload_log disabled)
 
-    Config = Struct.new(*FIELDS, keyword_init: true)
+    Config = Struct.new(*FIELDS, keyword_init: true) do
+      def producer_class=(val)
+        self.producer_classes = [val]
+      end
+
+      def producer_class
+        self.producer_classes.first
+      end
+    end
     module Topic
-      FIELDS.each do |field|
+      (FIELDS + [:producer_class]).each do |field|
         define_method(field) do |*args|
-          active(false) if field == :producer_class
+          active(false) if %i(producer_class producer_classes).include?(field)
           @deimos_producer_config ||= Config.new
           if args.any?
             @deimos_producer_config.public_send("#{field}=", args[0])
             _deimos_setup_transcoders if schema && namespace
           end
-          @deimos_producer_config[field]
+          @deimos_producer_config.send(field)
         end
       end
     end

data/lib/deimos/logging.rb

@@ -3,7 +3,11 @@ module Deimos
     class << self
 
       def log_add(method, msg)
-        Karafka.logger.tagged('Deimos') do |logger|
+        if Karafka.logger.respond_to?(:tagged)
+          Karafka.logger.tagged('Deimos') do |logger|
+            logger.send(method, msg.to_json)
+          end
+        else
           logger.send(method, msg.to_json)
         end
 

data/lib/deimos/producer.rb

@@ -122,7 +122,7 @@ module Deimos
       end
 
       def karafka_config
-        Deimos.karafka_configs.find { |topic| topic.producer_class == self }
+        Deimos.karafka_configs.find { |topic| topic.producer_classes.include?(self) }
       end
 
       def topic

data/lib/deimos/test_helpers.rb

@@ -98,6 +98,8 @@ module Deimos
         message_key = Deimos::TestHelpers.normalize_message(key)
         hash_matcher = RSpec::Matchers::BuiltIn::Match.new(message)
         Deimos::TestHelpers.sent_messages.any? do |m|
+          message.delete(:payload_key) if message.respond_to?(:[]) && message[:payload_key].nil?
+          m[:payload].delete(:payload_key) if m.respond_to?(:[]) && m[:payload]&.respond_to?(:[]) && m[:payload][:payload_key].nil?
           hash_matcher.send(:match, message, m[:payload]) &&
             topic == m[:topic] &&
             (key.present? ? message_key == m[:key] : true) &&
@@ -171,6 +173,7 @@ module Deimos
       unless call_original.nil?
         puts "test_consume_batch(call_original: true) is deprecated and will be removed in the future. You can remove the call_original parameter."
       end
+      karafka.consumer_messages.clear
       consumer = nil
       topic_name = nil
       if handler_class_or_topic.is_a?(String)
@@ -183,15 +186,22 @@ module Deimos
 
       Deimos.karafka_config_for(topic: topic_name).each_message(single)
 
-       payloads.each_with_index do |payload, i|
-         karafka.produce(payload, {key: keys[i], partition_key: partition_keys[i], topic: consumer.topic.name})
-       end
-       if block_given?
-         allow_any_instance_of(consumer_class).to receive(:consume_batch) do
-           yield
-         end
+      # don't record messages sent with test_consume_batch
+      original_messages = karafka.produced_messages.dup
+      payloads.each_with_index do |payload, i|
+       karafka.produce(payload, {key: keys[i], partition_key: partition_keys[i], topic: consumer.topic.name})
+      end
+      if block_given?
+       allow_any_instance_of(consumer_class).to receive(:consume_batch) do
+         yield
        end
-       consumer.consume
       end
+
+      # sent_messages should only include messages sent by application code, not this method
+      karafka.produced_messages.clear
+      karafka.produced_messages.concat(original_messages)
+
+      consumer.consume
+    end
   end
 end

data/lib/deimos/transcoder.rb

@@ -38,9 +38,11 @@ module Deimos
       return nil if key.nil? || self.key_field.nil?
 
       decoded_key = self.backend.decode_key(key, self.key_field)
-      return decoded_key unless @use_schema_classes
+      return decoded_key if self.key_field || !@use_schema_classes
 
-      Utils::SchemaClass.instance(decoded_key,
+      schema_key = decoded_key.is_a?(Hash) ? decoded_key : { self.key_field => decoded_key }
+
+      Utils::SchemaClass.instance(schema_key,
                                   "#{@schema}_key",
                                   @namespace)
     end

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '2.0.0-alpha2'
+  VERSION = '2.0.0-alpha4'
 end

data/lib/deimos.rb

@@ -162,7 +162,7 @@ module Deimos
       if topic
         karafka_configs.find { |t| t.name == topic}
       elsif producer
-        karafka_configs.find { |t| t.producer_class == producer}
+        karafka_configs.find { |t| t.producer_classes.include?(producer)}
       end
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 2.0.0.pre.alpha2
+  version: 2.0.0.pre.alpha4
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-11-29 00:00:00.000000000 Z
+date: 2024-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf