deimos-ruby 1.24.2 → 2.0.0.pre.alpha1

data/lib/deimos/ext/schema_route.rb

@@ -0,0 +1,70 @@
+require "deimos/transcoder"
+require "deimos/ext/producer_middleware"
+require "deimos/schema_backends/plain"
+
+module Deimos
+  class SchemaRoute < Karafka::Routing::Features::Base
+
+    module Topic
+      {
+        schema: nil,
+        namespace: nil,
+        key_config: {none: true},
+        use_schema_classes: Deimos.config.schema.use_schema_classes
+      }.each do |field, default|
+        define_method(field) do |*args|
+          @_deimos_config ||= {}
+          @_deimos_config[:schema] ||= {}
+          if args.any?
+            @_deimos_config[:schema][field] = args[0]
+            _deimos_setup_transcoders if schema && namespace
+          end
+          @_deimos_config[:schema][field] || default
+        end
+      end
+      def _deimos_setup_transcoders
+        payload = Transcoder.new(
+            schema: schema,
+            namespace: namespace,
+            use_schema_classes: use_schema_classes,
+            topic: name
+          )
+
+        key = nil
+
+        if key_config[:plain]
+          key = Transcoder.new(
+            schema: schema,
+            namespace: namespace,
+            use_schema_classes: use_schema_classes,
+            topic: name
+          )
+          key.backend = Deimos::SchemaBackends::Plain.new(schema: nil, namespace: nil)
+        elsif !key_config[:none]
+          if key_config[:field]
+            key = Transcoder.new(
+              schema: schema,
+              namespace: namespace,
+              use_schema_classes: use_schema_classes,
+              key_field: key_config[:field].to_s,
+              topic: name
+            )
+          elsif key_config[:schema]
+            key = Transcoder.new(
+              schema: key_config[:schema] || schema,
+              namespace: namespace,
+              use_schema_classes: use_schema_classes,
+              topic: self.name
+            )
+          else
+            raise 'No key config given - if you are not encoding keys, please use `key_config plain: true`'
+          end
+        end
+        deserializers.payload = payload
+        deserializers.key = key if key
+      end
+    end
+  end
+end
+
+Deimos::SchemaRoute.activate

data/lib/deimos/kafka_message.rb

@@ -49,8 +49,7 @@ module Deimos
       end
     end
 
-    # @return [Hash]
-    def phobos_message
+    def karafka_message
       {
         payload: self.message,
         partition_key: self.partition_key,
@@ -58,5 +57,6 @@ module Deimos
         topic: self.topic
       }
     end
+
   end
 end

data/lib/deimos/kafka_source.rb

@@ -31,7 +31,7 @@ module Deimos
       return unless self.class.kafka_config[:update]
 
       producers = self.class.kafka_producers
-      fields = producers.flat_map(&:watched_attributes).uniq
+      fields = producers.flat_map { |p| p.watched_attributes(self) }.uniq
       fields -= ['updated_at']
       # Only send an event if a field we care about was changed.
       any_changes = fields.any? do |field|
@@ -71,12 +71,7 @@ module Deimos
 
       # @return [Array<Deimos::ActiveRecordProducer>] the producers to run.
       def kafka_producers
-        if self.respond_to?(:kafka_producer)
-          Deimos.config.logger.warn(message: DEPRECATION_WARNING)
-          return [self.kafka_producer]
-        end
-
-        raise NotImplementedError
+        raise MissingImplementationError
       end
 
       # This is an internal method, part of the activerecord_import gem. It's

data/lib/deimos/kafka_topic_info.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Deimos
-  # Record that keeps track of which topics are being worked on by DbProducers.
+  # Record that keeps track of which topics are being worked on by OutboxProducers.
   class KafkaTopicInfo < ActiveRecord::Base
     self.table_name = 'kafka_topic_info'
 

data/lib/deimos/logging.rb

@@ -0,0 +1,71 @@
+module Deimos
+  module Logging
+    class << self
+
+      def log_add(method, msg)
+        Karafka.logger.tagged('Deimos') do |logger|
+          logger.send(method, msg.to_json)
+        end
+
+      end
+
+      def log_info(*args)
+        log_add(:info, *args)
+      end
+
+      def log_debug(*args)
+        log_add(:debug, *args)
+      end
+
+      def log_error(*args)
+        log_add(:error, *args)
+      end
+
+      def log_warn(*args)
+        log_add(:warn, *args)
+      end
+
+      def metadata_log_text(metadata)
+        metadata.to_h.slice(:timestamp, :offset, :first_offset, :last_offset, :partition, :topic, :size)
+      end
+
+      def _payloads(messages)
+
+      end
+
+      def messages_log_text(payload_log, messages)
+        log_message = {}
+
+        case payload_log
+        when :keys
+          keys = messages.map do |m|
+            m.respond_to?(:payload) ? m.key || m.payload['message_id'] : m[:key] || m[:payload_key] || m[:payload]['message_id']
+          end
+          log_message.merge!(
+            payload_keys: keys
+          )
+        when :count
+          log_message.merge!(
+            payloads_count: messages.count
+          )
+        when :headers
+          log_message.merge!(
+            payload_headers: messages.map { |m| m.respond_to?(:headers) ? m.headers : m[:headers] }
+          )
+        else
+          log_message.merge!(
+            payloads: messages.map do |m|
+              {
+                payload: m.respond_to?(:payload) ? m.payload : m[:payload],
+                key: m.respond_to?(:payload) ? m.key : m[:key]
+              }
+            end
+          )
+        end
+
+        log_message
+      end
+
+    end
+  end
+end

data/lib/deimos/message.rb

@@ -17,17 +17,13 @@ module Deimos
     attr_accessor  :encoded_payload
     # @return [String]
     attr_accessor  :topic
-    # @return [String]
-    attr_accessor  :producer_name
 
     # @param payload [Hash]
-    # @param producer [Class]
     # @param topic [String]
     # @param key [String, Integer, Hash]
     # @param partition_key [Integer]
-    def initialize(payload, producer, topic: nil, key: nil, headers: nil, partition_key: nil)
+    def initialize(payload, topic: nil, key: nil, headers: nil, partition_key: nil)
       @payload = payload&.with_indifferent_access
-      @producer_name = producer&.name
       @topic = topic
       @key = key
       @headers = headers&.with_indifferent_access
@@ -64,11 +60,7 @@ module Deimos
         key: @encoded_key,
         headers: @headers,
         partition_key: @partition_key || @encoded_key,
-        payload: @encoded_payload,
-        metadata: {
-          decoded_payload: @payload,
-          producer_name: @producer_name
-        }
+        payload: @encoded_payload
       }.delete_if { |k, v| k == :headers && v.nil? }
     end
 
@@ -82,7 +74,6 @@ module Deimos
         payload: @payload,
         metadata: {
           decoded_payload: @payload,
-          producer_name: @producer_name
         }
       }.delete_if { |k, v| k == :headers && v.nil? }
     end

data/lib/deimos/metrics/datadog.rb

@@ -1,12 +1,15 @@
 # frozen_string_literal: true
 
 require 'deimos/metrics/provider'
+require 'karafka/instrumentation/vendors/datadog/metrics_listener'
+require 'waterdrop/instrumentation/vendors/datadog/metrics_listener'
 
 module Deimos
   module Metrics
     # A Metrics wrapper class for Datadog.
     class Datadog < Metrics::Provider
-      # @param config [Hash]
+
+      # @param config [Hash] a hash of both client and Karakfa MetricsListener configs.
       # @param logger [Logger]
       def initialize(config, logger)
         raise 'Metrics config must specify host_ip' if config[:host_ip].nil?
@@ -14,12 +17,48 @@ module Deimos
         raise 'Metrics config must specify namespace' if config[:namespace].nil?
 
         logger.info("DatadogMetricsProvider configured with: #{config}")
+
         @client = ::Datadog::Statsd.new(
           config[:host_ip],
           config[:host_port],
           tags: config[:tags],
           namespace: config[:namespace]
         )
+        setup_karafka(config)
+        setup_waterdrop(config)
+      end
+
+      def setup_karafka(config={})
+        karafka_listener = ::Karafka::Instrumentation::Vendors::Datadog::MetricsListener.new do |karafka_config|
+          karafka_config.client = @client
+          if config[:karafka_namespace]
+            karafka_config.namespace = config[:karafka_namespace]
+          end
+          if config[:karafka_distribution_mode]
+            karafka_config.distribution_mode = config[:karafka_distribution_mode]
+          end
+          if config[:rd_kafka_metrics]
+            karafka_config.rd_kafka_metrics = config[:rd_kafka_metrics]
+          end
+        end
+        Karafka.monitor.subscribe(karafka_listener)
+      end
+
+      def setup_waterdrop(config)
+        waterdrop_listener = ::WaterDrop::Instrumentation::Vendors::Datadog::MetricsListener.new do |waterdrop_config|
+          waterdrop_config.client = @client
+          if config[:karafka_namespace]
+            waterdrop_config.namespace = config[:karafka_namespace]
+          end
+          if config[:karafka_distribution_mode]
+            waterdrop_config.distribution_mode = config[:karafka_distribution_mode]
+          end
+          if config[:rd_kafka_metrics]
+            karafka_config.rd_kafka_metrics = [] # handled in Karafka
+          end
+        end
+        Karafka::Setup::Config.setup if Karafka.producer.nil?
+        Karafka.producer.monitor.subscribe(waterdrop_listener)
       end
 
       # :nodoc:

data/lib/deimos/metrics/provider.rb

@@ -9,7 +9,7 @@ module Deimos
       # @param options [Hash] Any additional options, e.g. :tags
       # @return [void]
       def increment(metric_name, options={})
-        raise NotImplementedError
+        raise MissingImplementationError
       end
 
       # Send an counter increment metric
@@ -18,7 +18,7 @@ module Deimos
       # @param options [Hash] Any additional options, e.g. :tags
       # @return [void]
       def gauge(metric_name, count, options={})
-        raise NotImplementedError
+        raise MissingImplementationError
       end
 
       # Send an counter increment metric
@@ -27,7 +27,7 @@ module Deimos
       # @param options [Hash] Any additional options, e.g. :tags
       # @return [void]
       def histogram(metric_name, count, options={})
-        raise NotImplementedError
+        raise MissingImplementationError
       end
 
       # Time a yielded block, and send a timer metric
@@ -35,7 +35,7 @@ module Deimos
       # @param options [Hash] Any additional options, e.g. :tags
       # @return [void]
       def time(metric_name, options={})
-        raise NotImplementedError
+        raise MissingImplementationError
       end
     end
   end

data/lib/deimos/producer.rb

@@ -2,12 +2,12 @@
 
 require 'deimos/message'
 require 'deimos/shared_config'
-require 'phobos/producer'
 require 'active_support/notifications'
 
 # :nodoc:
 module Deimos
   class << self
+
     # Run a block without allowing any messages to be produced to Kafka.
     # Optionally add a list of producer classes to limit the disabling to those
     # classes.
@@ -50,6 +50,8 @@ module Deimos
     # @param producer_class [Class]
     # @return [Boolean]
     def producers_disabled?(producer_class=nil)
+      return true if Deimos.config.producers.disabled
+
       Thread.current[:frk_disable_all_producers] ||
         Thread.current[:frk_disabled_producers]&.include?(producer_class)
     end
@@ -64,26 +66,6 @@ module Deimos
 
     class << self
 
-      # @return [Hash]
-      def config
-        @config ||= {
-          encode_key: true,
-          namespace: Deimos.config.producers.schema_namespace
-        }
-      end
-
-      # Set the topic.
-      # @param topic [String]
-      # @return [String] the current topic if no argument given.
-      def topic(topic=nil)
-        if topic
-          config[:topic] = topic
-          return
-        end
-        # accessor
-        "#{Deimos.config.producers.topic_prefix}#{config[:topic]}"
-      end
-
       # Override the default partition key (which is the payload key).
       # @param _payload [Hash] the payload being passed into the produce method.
       # Will include `payload_key` if it is part of the original payload.
@@ -98,7 +80,22 @@ module Deimos
       # @param headers [Hash] if specifying headers
       # @return [void]
       def publish(payload, topic: self.topic, headers: nil)
-        publish_list([payload], topic: topic, headers: headers)
+        produce([{payload: payload, topic: topic, headers: headers}])
+      end
+
+      # Produce a list of messages in WaterDrop message hash format.
+      # @param messages [Array<Hash>]
+      # @param backend [Class < Deimos::Backend]
+      def produce(messages, backend: determine_backend_class)
+        return if Deimos.producers_disabled?(self)
+
+        messages.each do |m|
+          m[:label] = m
+          m[:partition_key] ||= self.partition_key(m[:payload])
+        end
+        messages.in_groups_of(MAX_BATCH_SIZE, false) do |batch|
+          self.produce_batch(backend, batch)
+        end
       end
 
       # Publish a list of messages.
@@ -111,31 +108,31 @@ module Deimos
       # @param headers [Hash] if specifying headers
       # @return [void]
       def publish_list(payloads, sync: nil, force_send: false, topic: self.topic, headers: nil)
-        return if Deimos.config.kafka.seed_brokers.blank? ||
-                  Deimos.config.producers.disabled ||
-                  Deimos.producers_disabled?(self)
-
-        raise 'Topic not specified. Please specify the topic.' if topic.blank?
-
-        backend_class = determine_backend_class(sync, force_send)
-        Deimos.instrument(
-          'encode_messages',
-          producer: self,
-          topic: topic,
-          payloads: payloads
-        ) do
-          messages = Array(payloads).map { |p| Deimos::Message.new(p.to_h, self, headers: headers) }
-          messages.each { |m| _process_message(m, topic) }
-          messages.in_groups_of(MAX_BATCH_SIZE, false) do |batch|
-            self.produce_batch(backend_class, batch)
-          end
+        backend = determine_backend_class(sync, force_send)
+
+        messages = Array(payloads).map do |p|
+          {
+            payload: p&.to_h,
+            headers: headers,
+            topic: topic,
+            partition_key: self.partition_key(p)
+          }
         end
+        self.produce(messages, backend: backend)
+      end
+
+      def karafka_config
+        Deimos.karafka_configs.find { |topic| topic.producer_class == self }
+      end
+
+      def topic
+        karafka_config.name
       end
 
       # @param sync [Boolean]
       # @param force_send [Boolean]
       # @return [Class<Deimos::Backends::Base>]
-      def determine_backend_class(sync, force_send)
+      def determine_backend_class(sync=false, force_send=false)
         backend = if force_send
                     :kafka
                   else
@@ -151,86 +148,12 @@ module Deimos
 
       # Send a batch to the backend.
       # @param backend [Class<Deimos::Backends::Base>]
-      # @param batch [Array<Deimos::Message>]
+      # @param batch [Array<Hash>]
       # @return [void]
       def produce_batch(backend, batch)
         backend.publish(producer_class: self, messages: batch)
       end
 
-      # @return [Deimos::SchemaBackends::Base]
-      def encoder
-        @encoder ||= Deimos.schema_backend(schema: config[:schema],
-                                           namespace: config[:namespace])
-      end
-
-      # @return [Deimos::SchemaBackends::Base]
-      def key_encoder
-        @key_encoder ||= Deimos.schema_backend(schema: config[:key_schema],
-                                               namespace: config[:namespace])
-      end
-
-      # Override this in active record producers to add
-      # non-schema fields to check for updates
-      # @return [Array<String>] fields to check for updates
-      def watched_attributes
-        self.encoder.schema_fields.map(&:name)
-      end
-
-    private
-
-      # @param message [Message]
-      # @param topic [String]
-      def _process_message(message, topic)
-        # this violates the Law of Demeter but it has to happen in a very
-        # specific order and requires a bunch of methods on the producer
-        # to work correctly.
-        message.add_fields(encoder.schema_fields.map(&:name))
-        message.partition_key = self.partition_key(message.payload)
-        message.key = _retrieve_key(message.payload)
-        # need to do this before _coerce_fields because that might result
-        # in an empty payload which is an *error* whereas this is intended.
-        message.payload = nil if message.payload.blank?
-        message.coerce_fields(encoder)
-        message.encoded_key = _encode_key(message.key)
-        message.topic = topic
-        message.encoded_payload = if message.payload.nil?
-                                    nil
-                                  else
-                                    encoder.encode(message.payload,
-                                                   topic: "#{Deimos.config.producers.topic_prefix}#{config[:topic]}-value")
-                                  end
-      end
-
-      # @param key [Object]
-      # @return [String|Object]
-      def _encode_key(key)
-        if key.nil?
-          return nil if config[:no_keys] # no key is fine, otherwise it's a problem
-
-          raise 'No key given but a key is required! Use `key_config none: true` to avoid using keys.'
-        end
-        if config[:encode_key] && config[:key_field].nil? &&
-           config[:key_schema].nil?
-          raise 'No key config given - if you are not encoding keys, please use `key_config plain: true`'
-        end
-
-        if config[:key_field]
-          encoder.encode_key(config[:key_field], key, topic: "#{Deimos.config.producers.topic_prefix}#{config[:topic]}-key")
-        elsif config[:key_schema]
-          key_encoder.encode(key, topic: "#{Deimos.config.producers.topic_prefix}#{config[:topic]}-key")
-        else
-          key
-        end
-      end
-
-      # @param payload [Hash]
-      # @return [String]
-      def _retrieve_key(payload)
-        key = payload.delete(:payload_key)
-        return key if key
-
-        config[:key_field] ? payload[config[:key_field]] : nil
-      end
     end
   end
 end

data/lib/deimos/railtie.rb

@@ -2,6 +2,12 @@
 
 # Add rake task to Rails.
 class Deimos::Railtie < Rails::Railtie
+  config.before_initialize do
+    if ARGV[0] == "deimos:v2"
+      FigTree.keep_removed_configs = true
+    end
+  end
+
   rake_tasks do
     load 'tasks/deimos.rake'
   end

data/lib/deimos/schema_backends/avro_base.rb

@@ -20,15 +20,19 @@ module Deimos
 
       # @override
       def encode_key(key_id, key, topic: nil)
-        @key_schema ||= _generate_key_schema(key_id)
+        begin
+          @key_schema ||= @schema_store.find("#{@schema}_key")
+        rescue AvroTurf::SchemaNotFoundError
+          @key_schema = generate_key_schema(key_id)
+        end
         field_name = _field_name_from_schema(@key_schema)
-        payload = { field_name => key }
+        payload = key.is_a?(Hash) ? key : { field_name => key }
         encode(payload, schema: @key_schema['name'], topic: topic)
       end
 
       # @override
       def decode_key(payload, key_id)
-        @key_schema ||= _generate_key_schema(key_id)
+        @key_schema ||= generate_key_schema(key_id)
         field_name = _field_name_from_schema(@key_schema)
         decode(payload, schema: @key_schema['name'])[field_name]
       end
@@ -85,7 +89,7 @@ module Deimos
 
       # @override
       def self.mock_backend
-        :avro_validation
+        :avro_local
       end
 
       # @override
@@ -146,21 +150,8 @@ module Deimos
         end
       end
 
-    private
-
-      # @param schema [String]
-      # @return [Avro::Schema]
-      def avro_schema(schema=nil)
-        schema ||= @schema
-        @schema_store.find(schema, @namespace)
-      end
-
-      # Generate a key schema from the given value schema and key ID. This
-      # is used when encoding or decoding keys from an existing value schema.
-      # @param key_id [Symbol]
-      # @return [Hash]
-      def _generate_key_schema(key_id)
-        key_field = avro_schema.fields.find { |f| f.name == key_id.to_s }
+      def generate_key_schema(field_name)
+        key_field = avro_schema.fields.find { |f| f.name == field_name.to_s }
         name = _key_schema_name(@schema)
         key_schema = {
           'type' => 'record',
@@ -169,13 +160,22 @@ module Deimos
           'doc' => "Key for #{@namespace}.#{@schema} - autogenerated by Deimos",
           'fields' => [
             {
-              'name' => key_id,
+              'name' => field_name,
               'type' => key_field.type.type_sym.to_s
             }
           ]
         }
         @schema_store.add_schema(key_schema)
-        key_schema
+        @key_schema = key_schema
+      end
+
+    private
+
+      # @param schema [String]
+      # @return [Avro::Schema]
+      def avro_schema(schema=nil)
+        schema ||= @schema
+        @schema_store.find(schema, @namespace)
       end
 
       # @param value_schema [Hash]

data/lib/deimos/schema_backends/avro_schema_registry.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require_relative 'avro_base'
-require_relative 'avro_validation'
 require 'avro_turf/messaging'
 
 module Deimos
@@ -29,7 +28,7 @@ module Deimos
           user: Deimos.config.schema.user,
           password: Deimos.config.schema.password,
           namespace: @namespace,
-          logger: Deimos.config.logger
+          logger: Karafka.logger
         )
       end
     end

data/lib/deimos/schema_backends/avro_validation.rb

@@ -9,12 +9,12 @@ module Deimos
     class AvroValidation < AvroBase
       # @override
       def decode_payload(payload, schema: nil)
-        payload.with_indifferent_access
+        JSON.parse(payload)
       end
 
       # @override
       def encode_payload(payload, schema: nil, topic: nil)
-        payload.with_indifferent_access
+        payload.to_h.with_indifferent_access.to_json
       end
     end
   end