delivery_boy 1.3.1 → 2.0.0.alpha.2

data/lib/delivery_boy/datadog.rb

@@ -0,0 +1,192 @@
+# frozen_string_literal: true
+
+begin
+  require "datadog/statsd"
+rescue LoadError
+  warn "In order to report Kafka client metrics to Datadog you need to install the `dogstatsd-ruby` gem."
+  raise
+end
+
+require "active_support/subscriber"
+
+module DeliveryBoy
+  # Reports operational metrics to a Datadog agent using the Statsd protocol.
+  #
+  #     require "delivery_boy/datadog"
+  #
+  #     # Default is "ruby_kafka" (kept for backward compatibility).
+  #     DeliveryBoy::Datadog.namespace = "custom-namespace"
+  #
+  #     # Default is "127.0.0.1".
+  #     DeliveryBoy::Datadog.host = "statsd.something.com"
+  #
+  #     # Default is 8125.
+  #     DeliveryBoy::Datadog.port = 1234
+  #
+  module Datadog
+    STATSD_NAMESPACE = "ruby_kafka"
+
+    class << self
+      attr_reader :host, :port, :socket_path
+
+      def configure
+        yield self
+      end
+
+      def statsd
+        @statsd ||= if socket_path
+          ::Datadog::Statsd.new(socket_path: socket_path, namespace: namespace, tags: tags)
+        else
+          ::Datadog::Statsd.new(host, port, namespace: namespace, tags: tags)
+        end
+      end
+
+      def statsd=(statsd)
+        clear
+        @statsd = statsd
+      end
+
+      def host=(host)
+        @host = host
+        clear
+      end
+
+      def port=(port)
+        @port = port
+        clear
+      end
+
+      def socket_path=(socket_path)
+        @socket_path = socket_path
+        clear
+      end
+
+      def namespace
+        @namespace ||= STATSD_NAMESPACE
+      end
+
+      def namespace=(namespace)
+        @namespace = namespace
+        clear
+      end
+
+      def tags
+        @tags ||= []
+      end
+
+      def tags=(tags)
+        @tags = tags
+        clear
+      end
+
+      def close
+        @statsd&.close
+      end
+
+      private
+
+      def clear
+        close
+        @statsd = nil
+      end
+    end
+
+    class StatsdSubscriber < ActiveSupport::Subscriber
+      private
+
+      %w[increment histogram count timing gauge].each do |type|
+        define_method(type) do |*args, **kwargs|
+          emit(type, *args, **kwargs)
+        end
+      end
+
+      def emit(type, *args, tags: {})
+        tags = tags.map { |k, v| "#{k}:#{v}" }.to_a
+        DeliveryBoy::Datadog.statsd.send(type, *args, tags: tags)
+      end
+    end
+
+    class ProducerSubscriber < StatsdSubscriber
+      def produce_message(event)
+        client = event.payload.fetch(:client_id)
+        topic = event.payload.fetch(:topic)
+        message_size = event.payload.fetch(:message_size)
+        buffer_size = event.payload.fetch(:buffer_size)
+
+        tags = {client: client, topic: topic}
+
+        if event.payload.key?(:exception)
+          increment("producer.produce.errors", tags: tags)
+        else
+          increment("producer.produce.messages", tags: tags)
+          histogram("producer.produce.message_size", message_size, tags: tags)
+          count("producer.produce.message_size.sum", message_size, tags: tags)
+          histogram("producer.buffer.size", buffer_size, tags: tags)
+        end
+      end
+
+      def deliver_messages(event)
+        client = event.payload.fetch(:client_id)
+        message_count = event.payload.fetch(:delivered_message_count)
+
+        tags = {client: client}
+
+        increment("producer.deliver.errors", tags: tags) if event.payload.key?(:exception)
+        timing("producer.deliver.latency", event.duration, tags: tags)
+        count("producer.deliver.messages", message_count, tags: tags)
+      end
+
+      def deliver(event)
+        client = event.payload.fetch(:client_id)
+        topic = event.payload.fetch(:topic)
+        message_size = event.payload.fetch(:message_size)
+
+        tags = {client: client, topic: topic}
+
+        if event.payload.key?(:exception)
+          increment("producer.deliver.errors", tags: tags)
+        else
+          increment("producer.produce.messages", tags: tags)
+          histogram("producer.produce.message_size", message_size, tags: tags)
+          count("producer.produce.message_size.sum", message_size, tags: tags)
+          timing("producer.deliver.latency", event.duration, tags: tags)
+          count("producer.deliver.messages", 1, tags: tags)
+        end
+      end
+
+      def deliver_async(event)
+        client = event.payload.fetch(:client_id)
+        topic = event.payload.fetch(:topic)
+        message_size = event.payload.fetch(:message_size)
+        queue_size = event.payload.fetch(:queue_size, 0)
+
+        tags = {client: client, topic: topic}
+
+        if event.payload.key?(:exception)
+          increment("async_producer.produce.errors", tags: tags)
+        else
+          increment("producer.produce.messages", tags: tags)
+          histogram("producer.produce.message_size", message_size, tags: tags)
+          count("producer.produce.message_size.sum", message_size, tags: tags)
+          histogram("async_producer.queue.size", queue_size, tags: tags)
+        end
+      end
+
+      def ack_message(event)
+        tags = {
+          client: event.payload.fetch(:client_id),
+          topic: event.payload.fetch(:topic)
+        }
+
+        increment("producer.ack.messages", tags: tags)
+      end
+
+      def delivery_error(event)
+        tags = {client: event.payload.fetch(:client_id)}
+        increment("producer.ack.errors", tags: tags)
+      end
+
+      attach_to "delivery_boy"
+    end
+  end
+end

data/lib/delivery_boy/fake.rb

@@ -1,5 +1,4 @@
 module DeliveryBoy
-
   # A fake implementation that is useful for testing.
   class Fake
     FakeMessage = Struct.new(:value, :topic, :key, :headers, :offset, :partition, :partition_key, :create_time) do
@@ -9,8 +8,8 @@ module DeliveryBoy
     end
 
     def initialize
-      @messages = Hash.new {|h, k| h[k] = [] }
-      @buffer = Hash.new {|h, k| h[k] = [] }
+      @messages = Hash.new { |h, k| h[k] = [] }
+      @buffer = Hash.new { |h, k| h[k] = [] }
       @delivery_lock = Mutex.new
     end
 
@@ -25,7 +24,7 @@ module DeliveryBoy
       nil
     end
 
-    alias deliver_async! deliver
+    alias_method :deliver_async!, :deliver
 
     def produce(value, topic:, key: nil, headers: {}, partition: nil, partition_key: nil, create_time: Time.now)
       @delivery_lock.synchronize do

data/lib/delivery_boy/instance.rb

@@ -1,49 +1,106 @@
-module DeliveryBoy
+# frozen_string_literal: true
 
+module DeliveryBoy
   # This class implements the actual logic of DeliveryBoy. The DeliveryBoy module
   # has a module-level singleton instance.
   class Instance
-    def initialize(config, logger)
+    def initialize(config, logger, instrumenter: NullInstrumenter.new)
       @config = config
       @logger = logger
-      @async_producer = nil
+      @instrumenter = instrumenter
     end
 
     def deliver(value, topic:, **options)
-      sync_producer.produce(value, topic: topic, **options)
-      sync_producer.deliver_messages
-    rescue
-      # Make sure to clear any buffered messages if there's an error.
-      clear_buffer
+      options_clone = options.clone
+      if options[:create_time]
+        options_clone[:timestamp] = Time.at(options[:create_time])
+        options_clone.delete(:create_time)
+      end
+
+      message_size = value.to_s.bytesize
+
+      instrumentation_payload = {
+        client_id: config.client_id,
+        topic: topic,
+        message_size: message_size
+      }
 
-      raise
+      @instrumenter.instrument("deliver", instrumentation_payload) do
+        sync_producer
+          .produce(payload: value, topic: topic, **options_clone)
+          .wait
+      end
     end
 
     def deliver_async!(value, topic:, **options)
-      async_producer.produce(value, topic: topic, **options)
+      options_clone = options.clone
+      if options[:create_time]
+        options_clone[:timestamp] = Time.at(options[:create_time])
+        options_clone.delete(:create_time)
+      end
+
+      message_size = value.to_s.bytesize
+
+      instrumentation_payload = {
+        client_id: config.client_id,
+        topic: topic,
+        message_size: message_size,
+        queue_size: async_producer_queue_size
+      }
+
+      @instrumenter.instrument("deliver_async", instrumentation_payload) do
+        async_producer
+          .produce(payload: value, topic: topic, **options_clone)
+      end
     end
 
     def shutdown
-      sync_producer.shutdown if sync_producer?
-      async_producer.shutdown if async_producer?
-
-      Thread.current[:delivery_boy_sync_producer] = nil
+      sync_producer.close if sync_producer?
+      async_producer.close if async_producer?
     end
 
     def produce(value, topic:, **options)
-      sync_producer.produce(value, topic: topic, **options)
+      options_clone = options.clone
+      if options[:create_time]
+        options_clone[:timestamp] = Time.at(options[:create_time])
+        options_clone.delete(:create_time)
+      end
+
+      message_size = value.to_s.bytesize
+
+      instrumentation_payload = {
+        client_id: config.client_id,
+        topic: topic,
+        message_size: message_size,
+        buffer_size: handles.size
+      }
+
+      @instrumenter.instrument("produce_message", instrumentation_payload) do
+        handle = sync_producer.produce(payload: value, topic: topic, **options_clone)
+        handles.push(handle)
+      end
     end
 
     def deliver_messages
-      sync_producer.deliver_messages
+      message_count = handles.size
+
+      instrumentation_payload = {
+        client_id: config.client_id,
+        delivered_message_count: message_count
+      }
+
+      @instrumenter.instrument("deliver_messages", instrumentation_payload) do
+        sync_producer.flush
+        handles.clear
+      end
     end
 
     def clear_buffer
-      sync_producer.clear_buffer
+      handles.clear
     end
 
     def buffer_size
-      sync_producer.buffer_size
+      handles.size
     end
 
     private
@@ -53,7 +110,7 @@ module DeliveryBoy
     def sync_producer
       # We want synchronous producers to be per-thread in order to avoid problems with
       # concurrent deliveries.
-      Thread.current[:delivery_boy_sync_producer] ||= kafka.producer(**producer_options)
+      Thread.current[:delivery_boy_sync_producer] ||= kafka.producer
     end
 
     def sync_producer?
@@ -63,64 +120,240 @@ module DeliveryBoy
     def async_producer
       # The async producer doesn't have to be per-thread, since all deliveries are
       # performed by a single background thread.
-      @async_producer ||= kafka.async_producer(
-        max_queue_size: config.max_queue_size,
-        delivery_threshold: config.delivery_threshold,
-        delivery_interval: config.delivery_interval,
-        **producer_options,
-      )
+      @async_producer ||= begin
+        producer = Rdkafka::Config.new({
+          "bootstrap.servers": config.brokers.join(","),
+          "queue.buffering.backpressure.threshold": config.delivery_threshold,
+          "queue.buffering.max.ms": config.delivery_interval_ms
+        }.merge(producer_options)).producer
+
+        producer.delivery_callback = delivery_callback
+        producer
+      end
     end
 
     def async_producer?
       !@async_producer.nil?
     end
 
+    def async_producer_queue_size
+      return 0 unless async_producer?
+      # rdkafka doesn't expose queue size directly, return 0 as approximation
+      0
+    end
+
+    def delivery_callback
+      instrumenter = @instrumenter
+      client_id = config.client_id
+
+      proc do |delivery_report|
+        if delivery_report.error
+          instrumenter.instrument("delivery_error", {
+            client_id: client_id,
+            error: delivery_report.error
+          })
+        else
+          instrumenter.instrument("ack_message", {
+            client_id: client_id,
+            topic: delivery_report.topic_name,
+            partition: delivery_report.partition,
+            offset: delivery_report.offset
+          })
+        end
+      end
+    end
+
     def kafka
-      @kafka ||= Kafka.new(
-        seed_brokers: config.brokers,
-        client_id: config.client_id,
-        logger: logger,
-        connect_timeout: config.connect_timeout,
-        socket_timeout: config.socket_timeout,
-        ssl_ca_cert: config.ssl_ca_cert,
-        ssl_ca_cert_file_path: config.ssl_ca_cert_file_path,
-        ssl_client_cert: config.ssl_client_cert,
-        ssl_client_cert_key: config.ssl_client_cert_key,
-        ssl_client_cert_key_password: config.ssl_client_cert_key_password,
-        ssl_ca_certs_from_system: config.ssl_ca_certs_from_system,
-        ssl_verify_hostname: config.ssl_verify_hostname,
-        sasl_gssapi_principal: config.sasl_gssapi_principal,
-        sasl_gssapi_keytab: config.sasl_gssapi_keytab,
-        sasl_plain_authzid: config.sasl_plain_authzid,
-        sasl_plain_username: config.sasl_plain_username,
-        sasl_plain_password: config.sasl_plain_password,
-        sasl_scram_username: config.sasl_scram_username,
-        sasl_scram_password: config.sasl_scram_password,
-        sasl_scram_mechanism: config.sasl_scram_mechanism,
-        sasl_over_ssl: config.sasl_over_ssl,
-        sasl_oauth_token_provider: config.sasl_oauth_token_provider,
-        sasl_aws_msk_iam_access_key_id: config.sasl_aws_msk_iam_access_key_id,
-        sasl_aws_msk_iam_secret_key_id: config.sasl_aws_msk_iam_secret_key_id,
-        sasl_aws_msk_iam_session_token: config.sasl_aws_msk_iam_session_token,
-        sasl_aws_msk_iam_aws_region: config.sasl_aws_msk_iam_aws_region
-      )
-    end
-
-    # Options for both the sync and async producers.
-    def producer_options
+      @kafka ||= Rdkafka::Config.new({
+        "bootstrap.servers": config.brokers.join(",")
+      }.merge(producer_options))
+    end
+
+    def sasl_options
+      return {} unless config.sasl_mechanism && !config.sasl_mechanism.empty?
+
+      config.validate_aws_msk_iam! if config.sasl_enabled?
+
+      options = {}
+
+      mechanism = config.sasl_mechanism.upcase
+
+      case mechanism
+      when "GSSAPI"
+        options.merge!(gssapi_options)
+      when "PLAIN"
+        options.merge!(plain_options)
+      when "SCRAM-SHA-256", "SCRAM-SHA-512"
+        options["sasl.mechanism"] = mechanism
+        options.merge!(scram_options)
+      when "OAUTHBEARER"
+        options.merge!(oauthbearer_options)
+      else
+        logger.warn "Unknown SASL mechanism: #{config.sasl_mechanism}"
+      end
+
+      options.compact
+    end
+
+    def gssapi_options
       {
-        required_acks: config.required_acks,
-        ack_timeout: config.ack_timeout,
-        max_retries: config.max_retries,
-        retry_backoff: config.retry_backoff,
-        max_buffer_size: config.max_buffer_size,
-        max_buffer_bytesize: config.max_buffer_bytesize,
-        compression_codec: (config.compression_codec.to_sym if config.compression_codec),
-        compression_threshold: config.compression_threshold,
-        idempotent: config.idempotent,
-        transactional: config.transactional,
-        transactional_timeout: config.transactional_timeout,
+        "sasl.mechanism" => "GSSAPI",
+        "sasl.kerberos.principal" => config.sasl_gssapi_principal,
+        "sasl.kerberos.keytab" => config.sasl_gssapi_keytab
       }
     end
+
+    def plain_options
+      username = config.sasl_username || config.sasl_plain_username
+      password = config.sasl_password || config.sasl_plain_password
+
+      if username.nil? || username.to_s.empty? || password.nil? || password.to_s.empty?
+        raise ConfigError, "PLAIN authentication requires sasl_username and sasl_password to be set"
+      end
+
+      # Note: sasl_plain_authzid doesn't have a librdkafka equivalent
+      # Log warning if set, but don't fail
+      if config.sasl_plain_authzid && !config.sasl_plain_authzid.empty?
+        logger.warn "sasl_plain_authzid is not supported by librdkafka and will be ignored"
+      end
+
+      {
+        "sasl.mechanism" => "PLAIN",
+        "sasl.username" => username,
+        "sasl.password" => password
+      }
+    end
+
+    def scram_options
+      username = config.sasl_username || config.sasl_scram_username
+      password = config.sasl_password || config.sasl_scram_password
+
+      if username.nil? || username.to_s.empty? || password.nil? || password.to_s.empty?
+        raise ConfigError, "SCRAM authentication requires sasl_username and sasl_password to be set"
+      end
+
+      {
+        "sasl.username" => username,
+        "sasl.password" => password
+      }
+    end
+
+    def oauthbearer_options
+      # Check for legacy token provider (not supported)
+      if config.sasl_oauth_token_provider
+        raise ConfigError, <<~ERROR
+          sasl_oauth_token_provider is no longer supported with librdkafka.
+
+          Migration options:
+          1. Use OIDC configuration (recommended for OIDC providers like Auth0, Okta):
+             config.sasl_oauthbearer_method = "oidc"
+             config.sasl_oauthbearer_client_id = "your-client-id"
+             config.sasl_oauthbearer_client_secret = "your-client-secret"
+             config.sasl_oauthbearer_token_endpoint_url = "https://auth.example.com/oauth/token"
+
+          2. Use SCRAM-SHA-256/512 as an alternative authentication method.
+
+          See: https://github.com/zendesk/delivery_boy/blob/master/MIGRATION.md#oauthbearer
+        ERROR
+      end
+
+      if config.sasl_oauthbearer_method&.downcase == "oidc"
+        if config.sasl_oauthbearer_client_id.nil? || config.sasl_oauthbearer_client_id.empty?
+          raise ConfigError, "OAUTHBEARER OIDC requires sasl_oauthbearer_client_id to be set"
+        end
+        if config.sasl_oauthbearer_client_secret.nil? || config.sasl_oauthbearer_client_secret.empty?
+          raise ConfigError, "OAUTHBEARER OIDC requires sasl_oauthbearer_client_secret to be set"
+        end
+        if config.sasl_oauthbearer_token_endpoint_url.nil? || config.sasl_oauthbearer_token_endpoint_url.empty?
+          raise ConfigError, "OAUTHBEARER OIDC requires sasl_oauthbearer_token_endpoint_url to be set"
+        end
+      else
+        raise ConfigError, <<~ERROR
+          OAUTHBEARER requires OIDC configuration.
+
+          Set the following options:
+            config.sasl_oauthbearer_method = "oidc"
+            config.sasl_oauthbearer_client_id = "your-client-id"
+            config.sasl_oauthbearer_client_secret = "your-client-secret"
+            config.sasl_oauthbearer_token_endpoint_url = "https://auth.example.com/oauth/token"
+        ERROR
+      end
+
+      options = {
+        "sasl.mechanism" => "OAUTHBEARER",
+        "sasl.oauthbearer.method" => "oidc",
+        "sasl.oauthbearer.client.id" => config.sasl_oauthbearer_client_id,
+        "sasl.oauthbearer.client.secret" => config.sasl_oauthbearer_client_secret,
+        "sasl.oauthbearer.token.endpoint.url" => config.sasl_oauthbearer_token_endpoint_url
+      }
+
+      options["sasl.oauthbearer.scope"] = config.sasl_oauthbearer_scope if config.sasl_oauthbearer_scope
+      options["sasl.oauthbearer.extensions"] = config.sasl_oauthbearer_extensions if config.sasl_oauthbearer_extensions
+
+      options
+    end
+
+    def security_protocol
+      has_ssl = config.ssl_ca_cert || config.ssl_ca_cert_file_path
+      has_sasl = config.sasl_enabled? || config.sasl_gssapi_principal
+
+      if config.sasl_over_ssl == false && has_ssl
+        raise ConfigError, <<~ERROR
+          sasl_over_ssl=false with SSL certificates configured is not supported by librdkafka.
+
+          librdkafka's security.protocol cannot express "SSL for verification but SASL over plaintext".
+
+          Options:
+          1. Remove SSL certificate configuration to use SASL_PLAINTEXT
+          2. Remove sasl_over_ssl=false to use SASL_SSL (recommended)
+
+          Note: sasl_over_ssl is deprecated and will be removed in a future version.
+        ERROR
+      end
+
+      if has_sasl && has_ssl
+        "SASL_SSL"
+      elsif has_sasl
+        "SASL_PLAINTEXT"
+      elsif has_ssl
+        "SSL"
+      else
+        "PLAINTEXT"
+      end
+    end
+
+    def producer_options
+      if config.transactional? && config.transactional_id.nil?
+        raise "transactional_id must be set"
+      end
+
+      {
+        "client.id": config.client_id,
+        "socket.connection.setup.timeout.ms": config.connection_timeout_ms,
+        "socket.timeout.ms": config.socket_timeout_ms,
+        "request.required.acks": config.required_acks,
+        "request.timeout.ms": config.ack_timeout_ms,
+        "message.send.max.retries": config.max_retries,
+        "retry.backoff.ms": config.retry_backoff_ms,
+        "queue.buffering.max.messages": config.max_buffer_size,
+        "queue.buffering.max.kbytes": config.max_buffer_kbytesize,
+        "compression.codec": config.compression_codec, # values none, gzip, snappy, lz4, zstd
+        "enable.idempotence": config.idempotent,
+        "transactional.id": config.transactional_id,
+        "transaction.timeout.ms": config.transactional_timeout_ms,
+        "security.protocol": security_protocol,
+        "ssl.ca.pem": config.ssl_ca_cert,
+        "ssl.ca.location": config.ssl_ca_cert_file_path,
+        "ssl.certificate.pem": config.ssl_client_cert,
+        "ssl.key.pem": config.ssl_client_cert_key,
+        "ssl.key.password": config.ssl_client_cert_key_password
+        # ssl_ca_certs_from_system: config.ssl_ca_certs_from_system, # TODO: there is no corresponding librdkafka option. check what this does
+        # ssl_verify_hostname: config.ssl_verify_hostname, # check
+      }.merge(sasl_options)
+    end
+
+    def handles
+      Thread.current[:delivery_boy_handles] ||= []
+    end
   end
 end

data/lib/delivery_boy/instrumenter.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module DeliveryBoy
+  class Instrumenter
+    NAMESPACE = "delivery_boy"
+
+    def initialize(default_payload: {})
+      require "active_support/notifications"
+      @default_payload = default_payload
+    end
+
+    def instrument(event_name, payload = {}, &block)
+      ActiveSupport::Notifications.instrument(
+        "#{event_name}.#{NAMESPACE}",
+        @default_payload.merge(payload),
+        &block
+      )
+    end
+  end
+
+  class NullInstrumenter
+    def instrument(*, &block)
+      block&.call
+    end
+  end
+end

data/lib/delivery_boy/railtie.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module DeliveryBoy
   class Railtie < Rails::Railtie
     initializer "delivery_boy.load_config" do
@@ -12,12 +14,15 @@ module DeliveryBoy
       end
 
       if config.datadog_enabled
-        require "kafka/datadog"
+        require "delivery_boy/datadog"
+
+        DeliveryBoy::Datadog.host = config.datadog_host if config.datadog_host.present?
+        DeliveryBoy::Datadog.port = config.datadog_port if config.datadog_port.present?
+        DeliveryBoy::Datadog.namespace = config.datadog_namespace if config.datadog_namespace.present?
+        DeliveryBoy::Datadog.tags = config.datadog_tags if config.datadog_tags.present?
 
-        Kafka::Datadog.host = config.datadog_host if config.datadog_host.present?
-        Kafka::Datadog.port = config.datadog_port if config.datadog_port.present?
-        Kafka::Datadog.namespace = config.datadog_namespace if config.datadog_namespace.present?
-        Kafka::Datadog.tags = config.datadog_tags if config.datadog_tags.present?
+        # Enable instrumentation
+        DeliveryBoy.instrumenter = DeliveryBoy::Instrumenter.new(default_payload: {})
       end
     end
   end