deimos-ruby 1.24.3 → 2.0.0.pre.alpha1

data/lib/deimos/consume/batch_consumption.rb

@@ -7,151 +7,51 @@ module Deimos
     # of messages to be handled at once
     module BatchConsumption
       extend ActiveSupport::Concern
-      include Phobos::BatchHandler
 
-      # @param batch [Array<String>]
-      # @param metadata [Hash]
-      # @return [void]
-      def around_consume_batch(batch, metadata)
-        payloads = []
-        _with_span do
-          benchmark = Benchmark.measure do
-            if self.class.config[:key_configured]
-              metadata[:keys] = batch.map do |message|
-                decode_key(message.key)
-              end
-            end
-            metadata[:first_offset] = batch.first&.offset
-
-            payloads = batch.map do |message|
-              decode_message(message.payload)
-            end
-            _received_batch(payloads, metadata)
-            yield(payloads, metadata)
-          end
-          _handle_batch_success(benchmark.real, payloads, metadata)
-        end
-      rescue StandardError => e
-        _handle_batch_error(e, payloads, metadata)
-      end
-
-      # Consume a batch of incoming messages.
-      # @param _payloads [Array<Phobos::BatchMessage>]
-      # @param _metadata [Hash]
-      # @return [void]
-      def consume_batch(_payloads, _metadata)
-        raise NotImplementedError
+      def consume_batch
+        raise MissingImplementationError
       end
 
     protected
 
-      # @!visibility private
-      def _received_batch(payloads, metadata)
-        Deimos.config.logger.info(
-          message: 'Got Kafka batch event',
-          message_ids: _payload_identifiers(payloads, metadata),
-          metadata: metadata.except(:keys)
-        )
-        Deimos.config.logger.debug(
-          message: 'Kafka batch event payloads',
-          payloads: payloads
-        )
-        Deimos.config.metrics&.increment(
-          'handler',
-          tags: %W(
-            status:batch_received
-            topic:#{metadata[:topic]}
-          ))
-        Deimos.config.metrics&.increment(
-          'handler',
-          by: metadata[:batch_size],
-          tags: %W(
-            status:received
-            topic:#{metadata[:topic]}
-          ))
-        if payloads.present?
-          payloads.each { |payload| _report_time_delayed(payload, metadata) }
+      def _consume_batch
+        _with_span do
+          begin
+            benchmark = Benchmark.measure do
+              consume_batch
+            end
+            _handle_batch_success(benchmark.real)
+          rescue StandardError => e
+            _handle_batch_error(e)
+          end
         end
       end
 
       # @!visibility private
       # @param exception [Throwable]
-      # @param payloads [Array<Hash>]
-      # @param metadata [Hash]
-      def _handle_batch_error(exception, payloads, metadata)
-        Deimos.config.metrics&.increment(
-          'handler',
-          tags: %W(
-            status:batch_error
-            topic:#{metadata[:topic]}
-          ))
-        Deimos.config.logger.warn(
+      def _handle_batch_error(exception)
+        Deimos::Logging.log_warn(
           message: 'Error consuming message batch',
           handler: self.class.name,
-          metadata: metadata.except(:keys),
-          message_ids: _payload_identifiers(payloads, metadata),
+          metadata: Deimos::Logging.metadata_log_text(messages.metadata),
+          messages: Deimos::Logging.messages_log_text(self.topic.payload_log, messages),
           error_message: exception.message,
           error: exception.backtrace
         )
-        _error(exception, payloads, metadata)
+        _error(exception, messages)
       end
 
       # @!visibility private
       # @param time_taken [Float]
-      # @param payloads [Array<Hash>]
-      # @param metadata [Hash]
-      def _handle_batch_success(time_taken, payloads, metadata)
-        Deimos.config.metrics&.histogram('handler',
-                                         time_taken,
-                                         tags: %W(
-                                           time:consume_batch
-                                           topic:#{metadata[:topic]}
-                                         ))
-        Deimos.config.metrics&.increment(
-          'handler',
-          tags: %W(
-            status:batch_success
-            topic:#{metadata[:topic]}
-          ))
-        Deimos.config.metrics&.increment(
-          'handler',
-          by: metadata[:batch_size],
-          tags: %W(
-            status:success
-            topic:#{metadata[:topic]}
-          ))
-        Deimos.config.logger.info(
-          message: 'Finished processing Kafka batch event',
-          message_ids: _payload_identifiers(payloads, metadata),
-          time_elapsed: time_taken,
-          metadata: metadata.except(:keys)
-        )
+      def _handle_batch_success(time_taken)
+        Deimos::Logging.log_info(
+          {
+            message: 'Finished processing Kafka batch event',
+            time_elapsed: time_taken,
+            metadata: Deimos::Logging.metadata_log_text(messages.metadata)
+          }.merge(Deimos::Logging.messages_log_text(self.topic.payload_log, messages)))
       end
 
-      # @!visibility private
-      # Get payload identifiers (key and message_id if present) for logging.
-      # @param payloads [Array<Hash>]
-      # @param metadata [Hash]
-      # @return [Array<Array>] the identifiers.
-      def _payload_identifiers(payloads, metadata)
-        message_ids = payloads&.map do |payload|
-          if payload.is_a?(Hash) && payload.key?('message_id')
-            payload['message_id']
-          end
-        end
-
-        # Payloads may be nil if preprocessing failed
-        messages = payloads || metadata[:keys] || []
-
-        messages.zip(metadata[:keys] || [], message_ids || []).map do |_, k, m_id|
-          ids = {}
-
-          ids[:key] = k if k.present?
-          ids[:message_id] = m_id if m_id.present?
-
-          ids
-        end
-      end
     end
   end
 end

data/lib/deimos/consume/message_consumption.rb

@@ -6,90 +6,63 @@ module Deimos
     # are invoked for every individual message.
     module MessageConsumption
       extend ActiveSupport::Concern
-      include Phobos::Handler
-
-      # @param payload [String]
-      # @param metadata [Hash]
-      # @return [void]
-      def around_consume(payload, metadata)
-        decoded_payload = payload.nil? ? nil : payload.dup
-        new_metadata = metadata.dup
-        benchmark = Benchmark.measure do
-          _with_span do
-            new_metadata[:key] = decode_key(metadata[:key]) if self.class.config[:key_configured]
-            decoded_payload = decode_message(payload)
-            _received_message(decoded_payload, new_metadata)
-            yield(decoded_payload, new_metadata)
-          end
-        end
-        _handle_success(benchmark.real, decoded_payload, new_metadata)
-      rescue StandardError => e
-        _handle_error(e, decoded_payload, new_metadata)
-      end
 
       # Consume incoming messages.
-      # @param _payload [String]
-      # @param _metadata [Hash]
+      # @param _message [Karafka::Messages::Message]
       # @return [void]
-      def consume(_payload, _metadata)
-        raise NotImplementedError
+      def consume_message(_message)
+        raise MissingImplementationError
       end
 
     private
 
-      def _received_message(payload, metadata)
-        Deimos.config.logger.info(
+      def _consume_messages
+        messages.each do |message|
+          begin
+            _with_span do
+              _received_message(message)
+              benchmark = Benchmark.measure do
+                consume_message(message)
+              end
+              _handle_success(message, benchmark.real)
+            rescue StandardError => e
+              _handle_message_error(e, message)
+            end
+          end
+        end
+      end
+
+      def _received_message(message)
+        Deimos::Logging.log_info(
           message: 'Got Kafka event',
-          payload: payload,
-          metadata: metadata
+          payload: message.payload,
+          metadata: Deimos::Logging.metadata_log_text(message.metadata)
         )
-        Deimos.config.metrics&.increment('handler', tags: %W(
-                                           status:received
-                                           topic:#{metadata[:topic]}
-                                         ))
-        _report_time_delayed(payload, metadata)
       end
 
       # @param exception [Throwable]
-      # @param payload [Hash]
-      # @param metadata [Hash]
-      def _handle_error(exception, payload, metadata)
-        Deimos.config.metrics&.increment(
-          'handler',
-          tags: %W(
-            status:error
-            topic:#{metadata[:topic]}
-          )
-        )
-        Deimos.config.logger.warn(
+      # @param message [Karafka::Messages::Message]
+      def _handle_message_error(exception, message)
+        Deimos::Logging.log_warn(
           message: 'Error consuming message',
           handler: self.class.name,
-          metadata: metadata,
-          data: payload,
+          metadata: Deimos::Logging.metadata_log_text(message.metadata),
+          key: message.key,
+          data: message.payload,
           error_message: exception.message,
           error: exception.backtrace
         )
 
-        _error(exception, payload, metadata)
+        _error(exception, Karafka::Messages::Messages.new([message], messages.metadata))
       end
 
-      # @param time_taken [Float]
-      # @param payload [Hash]
-      # @param metadata [Hash]
-      def _handle_success(time_taken, payload, metadata)
-        Deimos.config.metrics&.histogram('handler', time_taken, tags: %W(
-                                           time:consume
-                                           topic:#{metadata[:topic]}
-                                         ))
-        Deimos.config.metrics&.increment('handler', tags: %W(
-                                           status:success
-                                           topic:#{metadata[:topic]}
-                                         ))
-        Deimos.config.logger.info(
+      def _handle_success(message, benchmark)
+        mark_as_consumed(message)
+        Deimos::Logging.log_info(
           message: 'Finished processing Kafka event',
-          payload: payload,
-          time_elapsed: time_taken,
-          metadata: metadata
+          payload: message.payload,
+          time_elapsed: benchmark,
+          metadata: Deimos::Logging.metadata_log_text(message.metadata)
         )
       end
     end

data/lib/deimos/consumer.rb

@@ -7,62 +7,21 @@ require 'deimos/consume/message_consumption'
 # Note: According to the docs, instances of your handler will be created
 # for every incoming message/batch. This class should be lightweight.
 module Deimos
-  # Basic consumer class. Inherit from this class and override either consume
-  # or consume_batch, depending on the delivery mode of your listener.
-  # `consume` -> use `delivery :message` or `delivery :batch`
-  # `consume_batch` -> use `delivery :inline_batch`
-  class Consumer
+  # Basic consumer class. Inherit from this class and override either consume_message
+  # or consume_batch, depending on the `:batch` config setting.
+  class Consumer < Karafka::BaseConsumer
     include Consume::MessageConsumption
     include Consume::BatchConsumption
     include SharedConfig
 
-    class << self
-      # @return [Deimos::SchemaBackends::Base]
-      def decoder
-        @decoder ||= Deimos.schema_backend(schema: config[:schema],
-                                           namespace: config[:namespace])
-      end
-
-      # @return [Deimos::SchemaBackends::Base]
-      def key_decoder
-        @key_decoder ||= Deimos.schema_backend(schema: config[:key_schema],
-                                               namespace: config[:namespace])
-      end
-    end
-
-    # Helper method to decode an encoded key.
-    # @param key [String]
-    # @return [Object] the decoded key.
-    def decode_key(key)
-      return nil if key.nil?
-
-      config = self.class.config
-      unless config[:key_configured]
-        raise 'No key config given - if you are not decoding keys, please use '\
-          '`key_config plain: true`'
-      end
-
-      if config[:key_field]
-        self.class.decoder.decode_key(key, config[:key_field])
-      elsif config[:key_schema]
-        self.class.key_decoder.decode(key, schema: config[:key_schema])
-      else # no encoding
-        key
+    def consume
+      if self.topic.each_message
+        _consume_messages
+      else
+        _consume_batch
       end
     end
 
-    # Helper method to decode an encoded message.
-    # @param payload [Object]
-    # @return [Object] the decoded message.
-    def decode_message(payload)
-      decoded_payload = payload.nil? ? nil : self.class.decoder.decode(payload)
-      return decoded_payload unless Utils::SchemaClass.use?(self.class.config.to_h)
-
-      Utils::SchemaClass.instance(decoded_payload,
-                                  self.class.config[:schema],
-                                  self.class.config[:namespace])
-    end
-
   private
 
     def _with_span
@@ -75,42 +34,24 @@ module Deimos
       Deimos.config.tracer&.finish(@span)
     end
 
-    def _report_time_delayed(payload, metadata)
-      return if payload.nil? || payload['timestamp'].blank?
-
-      begin
-        time_delayed = Time.now.in_time_zone - payload['timestamp'].to_datetime
-      rescue ArgumentError
-        Deimos.config.logger.info(
-          message: "Error parsing timestamp! #{payload['timestamp']}"
-        )
-        return
-      end
-      Deimos.config.metrics&.histogram('handler', time_delayed, tags: %W(
-                                         time:time_delayed
-                                         topic:#{metadata[:topic]}
-                                       ))
-    end
-
     # Overrideable method to determine if a given error should be considered
     # "fatal" and always be reraised.
     # @param _error [Exception]
-    # @param _payload [Hash]
-    # @param _metadata [Hash]
+    # @param _messages [Array<Karafka::Message>]
     # @return [Boolean]
-    def fatal_error?(_error, _payload, _metadata)
+    def fatal_error?(_error, _messages)
       false
     end
 
     # @param exception [Exception]
-    # @param payload [Hash]
-    # @param metadata [Hash]
-    def _error(exception, payload, metadata)
+    # @param messages [Array<Karafka::Message>]
+    def _error(exception, messages)
       Deimos.config.tracer&.set_error(@span, exception)
 
-      raise if Deimos.config.consumers.reraise_errors ||
-               Deimos.config.consumers.fatal_error&.call(exception, payload, metadata) ||
-               fatal_error?(exception, payload, metadata)
+      raise if self.topic.reraise_errors ||
+               Deimos.config.consumers.fatal_error&.call(exception, messages) ||
+               fatal_error?(exception, messages)
     end
+
   end
 end

data/lib/deimos/ext/consumer_route.rb

@@ -0,0 +1,35 @@
+module Deimos
+  class ConsumerRoute < Karafka::Routing::Features::Base
+    module Topic
+
+      FIELDS = %i(max_db_batch_size
+                  bulk_import_id_column
+                  replace_associations
+                  bulk_import_id_generator
+                  each_message
+                  reraise_errors
+                  fatal_error
+                  save_associations_first
+      )
+      Config = Struct.new(*FIELDS, keyword_init: true)
+
+      FIELDS.each do |field|
+        define_method(field) do |*args|
+          @deimos_config ||= Config.new(
+            bulk_import_id_column: :bulk_import_id,
+            replace_associations: true,
+            each_message: false,
+            bulk_import_id_generator: proc { SecureRandom.uuid },
+            fatal_error: proc { false }
+          )
+          if args.any?
+            @deimos_config.public_send("#{field}=", args[0])
+          end
+          @deimos_config[field]
+        end
+      end
+    end
+  end
+end
+
+Deimos::ConsumerRoute.activate

data/lib/deimos/ext/producer_middleware.rb

@@ -0,0 +1,94 @@
+module Deimos
+
+  module ProducerMiddleware
+    class << self
+
+      def call(message)
+        Karafka.monitor.instrument(
+          'deimos.encode_message',
+          producer: self,
+          message: message
+        ) do
+          config = Deimos.karafka_config_for(topic: message[:topic])
+          return message if config.nil?
+          return if message[:payload] && !message[:payload].is_a?(Hash) && !message[:payload].is_a?(SchemaClass::Record)
+
+          m = Deimos::Message.new(message[:payload].to_h,
+                                  headers: message[:headers],
+                                  partition_key: message[:partition_key])
+          _process_message(m, message, config)
+          message[:payload] = m.encoded_payload
+          message[:key] = m.encoded_key
+          message[:partition_key] = if m.partition_key
+                                      m.partition_key.to_s
+                                    elsif m.key
+                                      m.key.to_s
+                                    else
+                                      nil
+                                    end
+          message[:topic] = "#{Deimos.config.producers.topic_prefix}#{config.name}"
+
+          validate_key_config(config, message)
+
+          message
+        end
+      end
+
+      def validate_key_config(config, message)
+        if message[:key].nil? && config.deserializers[:key].is_a?(Deimos::Transcoder)
+           raise 'No key given but a key is required! Use `key_config none: true` to avoid using keys.'
+        end
+      end
+
+      # @param message [Deimos::Message]
+      # @param karafka_message [Hash]
+      # @param config [Deimos::ProducerConfig]
+      def _process_message(message, karafka_message, config)
+        encoder = config.deserializers[:payload].backend
+        key_transcoder = config.deserializers[:key]
+        # this violates the Law of Demeter but it has to happen in a very
+        # specific order and requires a bunch of methods on the producer
+        # to work correctly.
+        message.add_fields(encoder.schema_fields.map(&:name))
+        message.key = karafka_message[:key] || _retrieve_key(message.payload, key_transcoder)
+        # need to do this before _coerce_fields because that might result
+        # in an empty payload which is an *error* whereas this is intended.
+        message.payload = nil if message.payload.blank?
+        message.coerce_fields(encoder)
+        message.encoded_key = _encode_key(message.key, config)
+        message.topic = config.name
+        message.encoded_payload = if message.payload.nil?
+                                    nil
+                                  else
+                                    encoder.encode(message.payload,
+                                                   topic: "#{Deimos.config.producers.topic_prefix}#{config.name}-value")
+                                  end
+      end
+
+      # @param key [Object]
+      # @param config [ProducerConfig]
+      # @return [String|Object]
+      def _encode_key(key, config)
+        return nil if key.nil?
+
+        if config.deserializers[:key].respond_to?(:encode_key)
+          config.deserializers[:key].encode_key(key)
+        elsif key
+          config.deserializers[:payload].encode(key)
+        else
+          key
+        end
+      end
+
+      # @param payload [Hash]
+      # @param key_transcoder [Deimos::Transcoder]
+      # @return [String]
+      def _retrieve_key(payload, key_transcoder)
+        key = payload.delete(:payload_key)
+        return key if key || !key_transcoder.respond_to?(:key_field)
+
+        key_transcoder.key_field ? payload[key_transcoder.key_field] : nil
+      end
+    end
+  end
+end

data/lib/deimos/ext/producer_route.rb

@@ -0,0 +1,22 @@
+module Deimos
+  class ProducerRoute < Karafka::Routing::Features::Base
+    FIELDS = %i(producer_class payload_log disabled)
+
+    Config = Struct.new(*FIELDS, keyword_init: true)
+    module Topic
+      FIELDS.each do |field|
+        define_method(field) do |*args|
+          active(false) if field == :producer_class
+          @deimos_producer_config ||= Config.new
+          if args.any?
+            @deimos_producer_config.public_send("#{field}=", args[0])
+            _deimos_setup_transcoders if schema && namespace
+          end
+          @deimos_producer_config[field]
+        end
+      end
+    end
+  end
+end
+
+Deimos::ProducerRoute.activate

data/lib/deimos/ext/redraw.rb

@@ -0,0 +1,29 @@
+# This is for temporary testing until https://github.com/karafka/karafka/pull/2347 is merged and released.
+
+module Karafka
+  module Routing
+    class Builder < Array
+      alias array_clear clear
+
+      def clear
+        @mutex.synchronize do
+          @defaults = EMPTY_DEFAULTS
+          @draws.clear
+          array_clear
+        end
+      end
+
+      def redraw(&block)
+        @mutex.synchronize do
+          @draws.clear
+          array_clear
+        end
+        draw(&block)
+      end
+
+    end
+  end
+end
+
+require 'karafka'
+require 'karafka/routing/builder'

data/lib/deimos/ext/routing_defaults.rb

@@ -0,0 +1,72 @@
+# This monkey patch was provided by Maciej, the maintainer of Karafka. This allows
+# configs to override each other on a more granular basis rather than each `configure` call
+# blowing away all fields. It also supports multiple default blocks.
+#
+# Unfortunately this can't be merged into Karafka as of now because it will be a major breaking
+# change. As a compromise, it has been added to the test coverage of Karafka to ensure that
+# other changes don't break this.
+# https://github.com/karafka/karafka/issues/2344
+class Matcher
+  def initialize
+    @applications = []
+  end
+
+  def replay_on(topic_node)
+    @applications.each do |method, kwargs|
+      if kwargs.is_a?(Hash)
+        ref = topic_node.public_send(method)
+
+        kwargs.each do |arg, val|
+          if ref.respond_to?("#{arg}=")
+            ref.public_send("#{arg}=", val)
+          else
+            if ref.respond_to?(:details)
+              ref.details.merge!(kwargs)
+            elsif ref.is_a?(Hash)
+              ref.merge!(kwargs)
+            else
+              raise 'No idea if such case exists, if so, similar handling as config'
+            end
+          end
+        end
+      end
+
+      if kwargs.is_a?(Array) && kwargs.size == 1
+        if topic_node.respond_to?("#{method}=")
+          topic_node.public_send(:"#{method}=", kwargs.first)
+        else
+          topic_node.public_send(method, *kwargs)
+        end
+      end
+    end
+  end
+
+  def method_missing(m, *args, **kwargs)
+    if args.empty?
+      @applications << [m, kwargs]
+    else
+      @applications << [m, args]
+    end
+  end
+end
+
+DEFAULTS = Matcher.new
+
+module Builder
+  def defaults(&block)
+    DEFAULTS.instance_eval(&block) if block
+  end
+end
+
+module ConsumerGroup
+  def topic=(name, &block)
+    k = Matcher.new
+    t = super(name)
+    k.instance_eval(&block) if block
+    DEFAULTS.replay_on(t)
+    k.replay_on(t)
+  end
+end
+
+Karafka::Routing::Builder.prepend Builder
+Karafka::Routing::ConsumerGroup.prepend ConsumerGroup