deimos-ruby 1.24.2 → 2.0.0.pre.alpha1

data/lib/deimos/active_record_producer.rb

@@ -18,9 +18,11 @@ module Deimos
       # a record object, refetch the record to pass into the `generate_payload`
       # method.
       # @return [void]
-      def record_class(klass, refetch: true)
-        config[:record_class] = klass
-        config[:refetch_record] = refetch
+      def record_class(klass=nil, refetch: true)
+        return @record_class if klass.nil?
+
+        @record_class = klass
+        @refetch_record = refetch
       end
 
       # @param record [ActiveRecord::Base]
@@ -34,14 +36,14 @@ module Deimos
       # @param force_send [Boolean]
       # @return [void]
       def send_events(records, force_send: false)
-        primary_key = config[:record_class]&.primary_key
+        primary_key = @record_class&.primary_key
         messages = records.map do |record|
           if record.respond_to?(:attributes)
             attrs = record.attributes.with_indifferent_access
           else
             attrs = record.with_indifferent_access
-            if config[:refetch_record] && attrs[primary_key]
-              record = config[:record_class].find(attrs[primary_key])
+            if @refetch_record && attrs[primary_key]
+              record = @record_class.find(attrs[primary_key])
             end
           end
           generate_payload(attrs, record).with_indifferent_access
@@ -50,6 +52,15 @@ module Deimos
         self.post_process(records)
       end
 
+      def config
+        Deimos.karafka_configs.find { |t| t.producer_class == self }
+      end
+
+      def encoder
+        raise "No schema or namespace configured for #{self.name}" if config.nil?
+        config.deserializers[:payload].backend
+      end
+
       # Generate the payload, given a list of attributes or a record..
       # Can be overridden or added to by subclasses.
       # @param attributes [Hash]
@@ -62,9 +73,9 @@ module Deimos
         payload.delete_if do |k, _|
           k.to_sym != :payload_key && !fields.map(&:name).include?(k)
         end
-        return payload unless Utils::SchemaClass.use?(config.to_h)
+        return payload unless self.config.use_schema_classes
 
-        Utils::SchemaClass.instance(payload, config[:schema], config[:namespace])
+        Utils::SchemaClass.instance(payload, encoder.schema, encoder.namespace)
       end
 
       # Query to use when polling the database with the DbPoller. Add
@@ -76,7 +87,7 @@ module Deimos
       # than this value).
       # @return [ActiveRecord::Relation]
       def poll_query(time_from:, time_to:, column_name: :updated_at, min_id:)
-        klass = config[:record_class]
+        klass = @record_class
         table = ActiveRecord::Base.connection.quote_table_name(klass.table_name)
         column = ActiveRecord::Base.connection.quote_column_name(column_name)
         primary = ActiveRecord::Base.connection.quote_column_name(klass.primary_key)
@@ -95,6 +106,14 @@ module Deimos
       def post_process(_records)
       end
 
+      # Override this in active record producers to add
+      # non-schema fields to check for updates
+      # @param _record [ActiveRecord::Base]
+      # @return [Array<String>] fields to check for updates
+      def watched_attributes(_record)
+        self.encoder.schema_fields.map(&:name)
+      end
+
     end
   end
 end

data/lib/deimos/backends/base.rb

@@ -6,10 +6,11 @@ module Deimos
     class Base
       class << self
         # @param producer_class [Class<Deimos::Producer>]
-        # @param messages [Array<Deimos::Message>]
+        # @param messages [Array<Hash>]
         # @return [void]
         def publish(producer_class:, messages:)
-          Deimos.config.logger.info(log_message(messages))
+          message = ::Deimos::Logging.messages_log_text(producer_class.karafka_config.payload_log, messages)
+          Deimos::Logging.log_info({message: 'Publishing Messages:'}.merge(message))
           execute(producer_class: producer_class, messages: messages)
         end
 
@@ -17,43 +18,11 @@ module Deimos
         # @param messages [Array<Deimos::Message>]
         # @return [void]
         def execute(producer_class:, messages:)
-          raise NotImplementedError
+          raise MissingImplementationError
         end
 
       private
 
-        def log_message(messages)
-          log_message = {
-            message: 'Publishing messages',
-            topic: messages.first&.topic
-          }
-
-          case Deimos.config.payload_log
-          when :keys
-            log_message.merge!(
-              payload_keys: messages.map(&:key)
-            )
-          when :count
-            log_message.merge!(
-              payloads_count: messages.count
-            )
-          when :headers
-            log_message.merge!(
-              payload_headers: messages.map(&:headers)
-            )
-          else
-            log_message.merge!(
-              payloads: messages.map do |message|
-                {
-                  payload: message.payload,
-                  key: message.key
-                }
-              end
-            )
-          end
-
-          log_message
-        end
       end
     end
   end

data/lib/deimos/backends/kafka.rb

@@ -4,30 +4,14 @@ module Deimos
   module Backends
     # Default backend to produce to Kafka.
     class Kafka < Base
-      include Phobos::Producer
-
-      # Shut down the producer if necessary.
-      # @return [void]
-      def self.shutdown_producer
-        producer.sync_producer_shutdown if producer.respond_to?(:sync_producer_shutdown)
-        producer.kafka_client&.close
-      end
-
       # :nodoc:
       def self.execute(producer_class:, messages:)
-        Deimos.instrument(
-          'produce',
-          producer: producer_class,
-          topic: producer_class.topic,
-          payloads: messages.map(&:payload)
-        ) do
-          producer.publish_list(messages.map(&:encoded_hash))
-          Deimos.config.metrics&.increment(
-            'publish',
-            tags: %W(status:success topic:#{producer_class.topic}),
-            by: messages.size
-          )
-        end
+        Karafka.producer.produce_many_sync(messages)
+        Deimos.config.metrics&.increment(
+          'publish',
+          tags: %W(status:success topic:#{messages.first[:topic]}),
+          by: messages.size
+        )
       end
     end
   end

data/lib/deimos/backends/kafka_async.rb

@@ -4,30 +4,14 @@ module Deimos
   module Backends
     # Backend which produces to Kafka via an async producer.
     class KafkaAsync < Base
-      include Phobos::Producer
-
-      # Shut down the producer cleanly.
-      # @return [void]
-      def self.shutdown_producer
-        producer.async_producer_shutdown
-        producer.kafka_client&.close
-      end
-
       # :nodoc:
       def self.execute(producer_class:, messages:)
-        Deimos.instrument(
-          'produce',
-          producer: producer_class,
-          topic: producer_class.topic,
-          payloads: messages.map(&:payload)
-        ) do
-          producer.async_publish_list(messages.map(&:encoded_hash))
-          Deimos.config.metrics&.increment(
-            'publish',
-            tags: %W(status:success topic:#{producer_class.topic}),
-            by: messages.size
-          )
-        end
+        Karafka.producer.produce_many_async(messages)
+        Deimos.config.metrics&.increment(
+          'publish',
+          tags: %W(status:success topic:#{messages.first[:topic]}),
+          by: messages.size
+        )
       end
     end
   end

data/lib/deimos/backends/{db.rb → outbox.rb}

@@ -6,22 +6,23 @@ module Deimos
   module Backends
     # Backend which saves messages to the database instead of immediately
     # sending them.
-    class Db < Base
+    class Outbox < Base
       class << self
         # :nodoc:
         def execute(producer_class:, messages:)
           records = messages.map do |m|
+            Deimos::ProducerMiddleware.call(m)
             message = Deimos::KafkaMessage.new(
-              message: m.encoded_payload ? m.encoded_payload.to_s.b : nil,
-              topic: m.topic,
+              message: m[:payload] ? m[:payload].to_s.b : nil,
+              topic: m[:topic],
               partition_key: partition_key_for(m)
             )
-            message.key = m.encoded_key.to_s.b unless producer_class.config[:no_keys]
+            message.key = m[:key].to_s.b if m[:key]
             message
           end
           Deimos::KafkaMessage.import(records)
           Deimos.config.metrics&.increment(
-            'db_producer.insert',
+            'outbox.insert',
             tags: %W(topic:#{producer_class.topic}),
             by: records.size
           )
@@ -30,10 +31,13 @@ module Deimos
         # @param message [Deimos::Message]
         # @return [String] the partition key to use for this message
         def partition_key_for(message)
-          return message.partition_key if message.partition_key.present?
-          return message.key unless message.key.is_a?(Hash)
-
-          message.key.to_yaml
+          if message[:partition_key].present?
+            message[:partition_key]
+          elsif message[:key].present?
+            message[:key].to_s.b
+          else
+            nil
+          end
         end
       end
     end