delivery_boy 2.0.0.alpha.1 → 2.0.0.alpha.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d8bca84f797ae282f1568a53d1542f0bca409e56eaf27c079607e191ccb8841a
-  data.tar.gz: 012ee10f7057df815147ce29bb135d3f8397f1454a9fdff95afa72194c06e884
+  metadata.gz: a4dac7da9e416e7c55878674f91006d43b73811e27b1aa05897604d48c6e3162
+  data.tar.gz: b5536e1e66d7c8882fe6452b552f50f2fa4e37846870bea0c6b6f5ceef399028
 SHA512:
-  metadata.gz: 9381929d414fba52aabadc9d483c478bf1968b357db832825d57eedfd214f544cc95fdfee169ae54fb10f07f45ac485f22d3bc17198a614b649295b8341e9363
-  data.tar.gz: 598705aa9268eb0985262958ca4d2d1979d57a06bac366e99113ae302fb6935d5104053389f97236cdfb714430642a383284707b2a5ad9fbdc1615658671e371
+  metadata.gz: 8390928820e66c27e3ec762344df31d0e4101831e0f1cca21b2289939fbc23cad4f554b17314eb7f92d48e7f3bd1b6d90ebfceac3e3b41065dbabacf3f2aa0b6
+  data.tar.gz: 2bf809365bd9f693cf1dd03f61e6cb37c5c43682087ecc013cd0626d558a83bdc04b4dd4b065dc6a9e0170479dc192f2b26f56cc3501ce77aa820b351dddb817

data/.github/CODEOWNERS

@@ -0,0 +1,4 @@
+# CODEOWNERS file
+# This file defines who should review code changes in this repository.
+
+* @zendesk/core-gem-owners

data/CHANGELOG

@@ -1,6 +1,52 @@
 # Changelog
 
-## Unreleased
+## Unreleased (v2.0.0)
+
+### Migration to librdkafka
+
+* Migrated from ruby-kafka to rdkafka (librdkafka) for improved performance and stability
+
+### New Features
+
+* PLAIN and SCRAM-SHA-256/512 SASL authentication now fully functional with librdkafka
+* Added `sasl_username` and `sasl_password` as consolidated config options (work for both PLAIN and SCRAM)
+* Added automatic `security.protocol` detection (PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL)
+* Backward compatible SASL configuration - existing mechanism-specific options (sasl_plain_username, sasl_scram_username) continue to work
+* OAUTHBEARER authentication via OIDC configuration (librdkafka handles token refresh automatically)
+
+### Breaking Changes & Known Limitations
+
+* **AWS MSK IAM authentication is no longer supported** (librdkafka limitation)
+  - Alternative 1: Use AWS MSK SCRAM-SHA-512 authentication (recommended)
+  - Alternative 2: Use mTLS (mutual TLS) with client certificates
+* **OAUTHBEARER authentication changed from callback-based to OIDC configuration**
+  - `sasl_oauth_token_provider` is no longer supported
+  - Use new OIDC config options instead: `sasl_oauthbearer_method`, `sasl_oauthbearer_client_id`, `sasl_oauthbearer_client_secret`, `sasl_oauthbearer_token_endpoint_url`
+  - librdkafka handles token refresh automatically (no custom code needed)
+* `sasl_plain_authzid` is not supported by librdkafka and will be ignored with a warning
+* `compression_threshold` is no longer supported - librdkafka applies compression automatically to all batches when `compression_codec` is set
+
+### Config options no longer used
+
+The following config options are silently ignored (no error, but no effect):
+* `max_queue_size` - rdkafka manages its own internal queue
+* `ssl_ca_certs_from_system` - no librdkafka equivalent
+* `ssl_verify_hostname` - not mapped to librdkafka
+* `sasl_scram_mechanism` - use `sasl_mechanism` instead
+* `log_level` - rdkafka has its own logging configuration
+
+### Datadog Integration
+
+* Datadog metrics continue to work with the same metric names as before
+* Now uses `DeliveryBoy::Datadog` instead of `Kafka::Datadog`
+* Instrumentation is built on top of ActiveSupport::Notifications
+* Some metrics are no longer available due to rdkafka's different architecture:
+  - `producer.buffer.fill_ratio` / `producer.buffer.fill_percentage`
+  - `producer.deliver.attempts`
+  - `producer.ack.delay`
+  - `async_producer.queue.fill_ratio`
+
+### Other Changes
 
 * `compression_codec` in the `DeliveryBoy::Config` is now coerces its value into
   a symbol if the value is present.

data/Gemfile

@@ -10,3 +10,5 @@ gem "rake", "~> 13.0"
 gem "rspec", "~> 3.0"
 gem "standard", "~> 1.51.1" if RUBY_VERSION > "3.0"
 gem "testcontainers-kafka", github: "testcontainers/testcontainers-ruby"
+gem "activesupport"
+gem "dogstatsd-ruby"

data/README.md

@@ -172,7 +172,7 @@ The codec used to compress messages. Must be either `snappy` or `gzip`.
 
 ##### `compression_threshold`
 
-The minimum number of messages that must be buffered before compression is attempted. By default only one message is required. Only relevant if `compression_codec` is set.
+**Deprecated:** This setting has no effect with librdkafka. Compression is applied automatically to all batches when `compression_codec` is set.
 
 #### Network
 
@@ -226,88 +226,136 @@ The password required to read the ssl_client_cert_key. Must be used in combinati
 
 #### SASL Authentication and authorization
 
-See [ruby-kafka](https://github.com/zendesk/ruby-kafka#authentication-using-sasl) for more information.
+DeliveryBoy supports SASL authentication with multiple mechanisms.
 
-Use it through `GSSAPI`, `PLAIN` _or_ `OAUTHBEARER`.
+##### GSSAPI (Kerberos)
 
-##### `sasl_gssapi_principal`
+###### `sasl_gssapi_principal`
 
 The GSSAPI principal.
 
-##### `sasl_gssapi_keytab`
+###### `sasl_gssapi_keytab`
 
 Optional GSSAPI keytab.
 
-##### `sasl_plain_authzid`
+**Example:**
 
-The authorization identity to use.
+```ruby
+DeliveryBoy.configure do |config|
+  config.sasl_mechanism = "GSSAPI"
+  config.sasl_gssapi_principal = "kafka/hostname@REALM"
+  config.sasl_gssapi_keytab = "/path/to/keytab"
+end
+```
+
+##### PLAIN Authentication
+
+###### `sasl_plain_username`
+
+The username used to authenticate (legacy option).
 
-##### `sasl_plain_username`
+###### `sasl_plain_password`
 
-The username used to authenticate.
+The password used to authenticate (legacy option).
 
-##### `sasl_plain_password`
+###### `sasl_username`
 
-The password used to authenticate.
+The username used to authenticate (new consolidated option, works for both PLAIN and SCRAM).
 
-##### `sasl_oauth_token_provider`
+###### `sasl_password`
 
-A instance of a class which implements the `token` method.
-As described in [ruby-kafka](https://github.com/zendesk/ruby-kafka/tree/c3e90bc355fad1e27b9af1048966ff08d3d5735b#oauthbearer)
+The password used to authenticate (new consolidated option, works for both PLAIN and SCRAM).
+
+**Example:**
 
 ```ruby
-class TokenProvider
-  def token
-    "oauth-token"
-  end
+DeliveryBoy.configure do |config|
+  config.sasl_mechanism = "PLAIN"
+  # You can use either the new consolidated options:
+  config.sasl_username = "your-username"
+  config.sasl_password = "your-password"
+  # Or the legacy mechanism-specific options:
+  # config.sasl_plain_username = "your-username"
+  # config.sasl_plain_password = "your-password"
 end
+```
+
+##### SCRAM Authentication
+
+Supports SCRAM-SHA-256 and SCRAM-SHA-512 mechanisms.
+
+###### `sasl_scram_username`
 
+The username used to authenticate (legacy option).
+
+###### `sasl_scram_password`
+
+The password used to authenticate (legacy option).
+
+**Example:**
+
+```ruby
 DeliveryBoy.configure do |config|
-  config.sasl_oauth_token_provider = TokenProvider.new
-  config.ssl_ca_certs_from_system = true
+  config.sasl_mechanism = "SCRAM-SHA-256"  # or "SCRAM-SHA-512"
+  # You can use either the new consolidated options:
+  config.sasl_username = "your-username"
+  config.sasl_password = "your-password"
+  # Or the legacy mechanism-specific options:
+  # config.sasl_scram_username = "your-username"
+  # config.sasl_scram_password = "your-password"
 end
 ```
 
-#### AWS MSK IAM Authentication and Authorization
+##### OAUTHBEARER (OIDC)
+
+OAUTHBEARER authentication is supported via OIDC configuration. librdkafka handles token acquisition and refresh automatically.
+
+###### `sasl_oauthbearer_method`
 
-##### sasl_aws_msk_iam_access_key_id
+Set to `"oidc"` to enable OIDC-based OAUTHBEARER authentication.
 
-The AWS IAM access key. Required.
+###### `sasl_oauthbearer_client_id`
 
-##### sasl_aws_msk_iam_secret_key_id
+The OAuth client ID for your application.
 
-The AWS IAM secret access key. Required.
+###### `sasl_oauthbearer_client_secret`
 
-##### sasl_aws_msk_iam_aws_region
+The OAuth client secret for your application.
 
-The AWS region. Required.
+###### `sasl_oauthbearer_token_endpoint_url`
 
-##### sasl_aws_msk_iam_session_token
+The URL of the OAuth token endpoint (e.g., `https://auth.example.com/oauth/token`).
 
-The session token. This value can be optional.
+###### `sasl_oauthbearer_scope` (optional)
 
-###### Examples 
+OAuth scope to request.
 
-Using a role arn and web identity token to generate temporary credentials:
+###### `sasl_oauthbearer_extensions` (optional)
+
+Additional SASL extensions as comma-separated key=value pairs.
 
 ```ruby
-require "aws-sdk-core"
-require "delivery_boy"
-
-role = Aws::AssumeRoleWebIdentityCredentials.new(
-  role_arn: ENV["AWS_ROLE_ARN"],
-  web_identity_token_file: ENV["AWS_WEB_IDENTITY_TOKEN_FILE"]
-)
-
-DeliveryBoy.configure do |c|
-  c.sasl_aws_msk_iam_access_key_id = role.credentials.access_key_id
-  c.sasl_aws_msk_iam_secret_key_id = role.credentials.secret_access_key
-  c.sasl_aws_msk_iam_session_token = role.credentials.session_token
-  c.sasl_aws_msk_iam_aws_region    = ENV["AWS_REGION"]
-  c.ssl_ca_certs_from_system       = true
+DeliveryBoy.configure do |config|
+  config.sasl_mechanism = "OAUTHBEARER"
+  config.sasl_oauthbearer_method = "oidc"
+  config.sasl_oauthbearer_client_id = "your-client-id"
+  config.sasl_oauthbearer_client_secret = "your-client-secret"
+  config.sasl_oauthbearer_token_endpoint_url = "https://auth.example.com/oauth/token"
+  # Optional:
+  # config.sasl_oauthbearer_scope = "kafka"
 end
 ```
 
+**Note:** The legacy `sasl_oauth_token_provider` callback is no longer supported. Use OIDC configuration instead.
+
+#### AWS MSK IAM Authentication
+
+**Note:** AWS MSK IAM authentication is not supported by librdkafka.
+
+**Recommended alternatives:**
+1. **Use AWS MSK SCRAM authentication** - Create SCRAM credentials in AWS Secrets Manager and use SCRAM-SHA-512
+2. **Use mTLS** - Configure mutual TLS with client certificates
+
 ### Testing
 
 DeliveryBoy provides a test mode out of the box. When this mode is enabled, messages will be stored in memory rather than being sent to Kafka. If you use RSpec, enabling test mode is as easy as adding this to your spec helper:

data/delivery_boy.gemspec

@@ -19,7 +19,6 @@ Gem::Specification.new do |spec|
 
   spec.require_paths = ["lib"]
 
-  # spec.add_runtime_dependency "ruby-kafka", "~> 1.0"
   spec.add_runtime_dependency "king_konf", "~> 1.0"
   spec.add_runtime_dependency "rdkafka", "> 0.11"
 end

data/lib/delivery_boy/config.rb

@@ -16,10 +16,6 @@ module DeliveryBoy
       transactional_timeout * 1000
     end
 
-    def isolation_level
-      transactional ? "read_uncommitted" : "read_committed"
-    end
-
     def max_buffer_kbytesize
       max_buffer_bytesize / 1024
     end
@@ -28,7 +24,36 @@ module DeliveryBoy
       delivery_interval * 1000
     end
 
-    def sasl_username
+    def ack_timeout_ms
+      ack_timeout * 1000
+    end
+
+    def retry_backoff_ms
+      retry_backoff * 1000
+    end
+
+    def sasl_enabled?
+      return false unless sasl_mechanism && !sasl_mechanism.empty?
+      sasl_mechanism.upcase != "GSSAPI"
+    end
+
+    def validate_aws_msk_iam!
+      if sasl_aws_msk_iam_access_key_id || sasl_aws_msk_iam_secret_key_id || sasl_aws_msk_iam_aws_region
+        raise ConfigError, <<~ERROR
+          AWS MSK IAM authentication is not supported by librdkafka.
+
+          Alternatives:
+          1. Use AWS MSK SCRAM-SHA-512 authentication (recommended)
+             - Create SCRAM credentials in AWS Secrets Manager
+             - Set sasl_mechanism = "SCRAM-SHA-512"
+             - Set sasl_username and sasl_password
+
+          2. Use mTLS (mutual TLS) with client certificates
+             - Configure ssl_client_cert and ssl_client_cert_key
+
+          See migration guide: https://github.com/zendesk/delivery_boy/blob/master/MIGRATION.md#aws-msk-iam
+        ERROR
+      end
     end
 
     # Basic
@@ -36,10 +61,10 @@ module DeliveryBoy
     string :client_id, default: "delivery_boy"
     string :log_level, default: nil
 
-    # Buffering
+    # Buffering (defaults match librdkafka defaults to avoid queue overflow at high throughput)
     integer :max_buffer_bytesize, default: 10_000_000
-    integer :max_buffer_size, default: 1000
-    integer :max_queue_size, default: 1000
+    integer :max_buffer_size, default: 100_000
+    integer :max_queue_size, default: 100_000
 
     # Network timeouts
     integer :connect_timeout, default: 10
@@ -71,7 +96,7 @@ module DeliveryBoy
     boolean :ssl_verify_hostname, default: true
 
     # Supported: GSSAPI, PLAIN, SCRAM-SHA-256, SCRAM-SHA-512, OAUTHBEARER
-    string :sasl_mechanism, default: "GSSAPI"
+    string :sasl_mechanism, default: nil
     # SASL authentication
     string :sasl_gssapi_principal
     string :sasl_gssapi_keytab
@@ -83,9 +108,21 @@ module DeliveryBoy
     string :sasl_scram_mechanism
     boolean :sasl_over_ssl, default: true
 
-    # SASL OAUTHBEARER
+    # New consolidated SASL options (librdkafka-aligned)
+    string :sasl_username, default: nil
+    string :sasl_password, default: nil
+
+    # SASL OAUTHBEARER (legacy - callback-based, not supported with librdkafka)
     attr_accessor :sasl_oauth_token_provider
 
+    # SASL OAUTHBEARER OIDC (librdkafka native support)
+    string :sasl_oauthbearer_method, default: nil # "oidc" for OIDC-based auth
+    string :sasl_oauthbearer_client_id, default: nil
+    string :sasl_oauthbearer_client_secret, default: nil
+    string :sasl_oauthbearer_token_endpoint_url, default: nil
+    string :sasl_oauthbearer_scope, default: nil
+    string :sasl_oauthbearer_extensions, default: nil
+
     # AWS IAM authentication
     string :sasl_aws_msk_iam_access_key_id
     string :sasl_aws_msk_iam_secret_key_id

data/lib/delivery_boy/datadog.rb

@@ -0,0 +1,192 @@
+# frozen_string_literal: true
+
+begin
+  require "datadog/statsd"
+rescue LoadError
+  warn "In order to report Kafka client metrics to Datadog you need to install the `dogstatsd-ruby` gem."
+  raise
+end
+
+require "active_support/subscriber"
+
+module DeliveryBoy
+  # Reports operational metrics to a Datadog agent using the Statsd protocol.
+  #
+  #     require "delivery_boy/datadog"
+  #
+  #     # Default is "ruby_kafka" (kept for backward compatibility).
+  #     DeliveryBoy::Datadog.namespace = "custom-namespace"
+  #
+  #     # Default is "127.0.0.1".
+  #     DeliveryBoy::Datadog.host = "statsd.something.com"
+  #
+  #     # Default is 8125.
+  #     DeliveryBoy::Datadog.port = 1234
+  #
+  module Datadog
+    STATSD_NAMESPACE = "ruby_kafka"
+
+    class << self
+      attr_reader :host, :port, :socket_path
+
+      def configure
+        yield self
+      end
+
+      def statsd
+        @statsd ||= if socket_path
+          ::Datadog::Statsd.new(socket_path: socket_path, namespace: namespace, tags: tags)
+        else
+          ::Datadog::Statsd.new(host, port, namespace: namespace, tags: tags)
+        end
+      end
+
+      def statsd=(statsd)
+        clear
+        @statsd = statsd
+      end
+
+      def host=(host)
+        @host = host
+        clear
+      end
+
+      def port=(port)
+        @port = port
+        clear
+      end
+
+      def socket_path=(socket_path)
+        @socket_path = socket_path
+        clear
+      end
+
+      def namespace
+        @namespace ||= STATSD_NAMESPACE
+      end
+
+      def namespace=(namespace)
+        @namespace = namespace
+        clear
+      end
+
+      def tags
+        @tags ||= []
+      end
+
+      def tags=(tags)
+        @tags = tags
+        clear
+      end
+
+      def close
+        @statsd&.close
+      end
+
+      private
+
+      def clear
+        close
+        @statsd = nil
+      end
+    end
+
+    class StatsdSubscriber < ActiveSupport::Subscriber
+      private
+
+      %w[increment histogram count timing gauge].each do |type|
+        define_method(type) do |*args, **kwargs|
+          emit(type, *args, **kwargs)
+        end
+      end
+
+      def emit(type, *args, tags: {})
+        tags = tags.map { |k, v| "#{k}:#{v}" }.to_a
+        DeliveryBoy::Datadog.statsd.send(type, *args, tags: tags)
+      end
+    end
+
+    class ProducerSubscriber < StatsdSubscriber
+      def produce_message(event)
+        client = event.payload.fetch(:client_id)
+        topic = event.payload.fetch(:topic)
+        message_size = event.payload.fetch(:message_size)
+        buffer_size = event.payload.fetch(:buffer_size)
+
+        tags = {client: client, topic: topic}
+
+        if event.payload.key?(:exception)
+          increment("producer.produce.errors", tags: tags)
+        else
+          increment("producer.produce.messages", tags: tags)
+          histogram("producer.produce.message_size", message_size, tags: tags)
+          count("producer.produce.message_size.sum", message_size, tags: tags)
+          histogram("producer.buffer.size", buffer_size, tags: tags)
+        end
+      end
+
+      def deliver_messages(event)
+        client = event.payload.fetch(:client_id)
+        message_count = event.payload.fetch(:delivered_message_count)
+
+        tags = {client: client}
+
+        increment("producer.deliver.errors", tags: tags) if event.payload.key?(:exception)
+        timing("producer.deliver.latency", event.duration, tags: tags)
+        count("producer.deliver.messages", message_count, tags: tags)
+      end
+
+      def deliver(event)
+        client = event.payload.fetch(:client_id)
+        topic = event.payload.fetch(:topic)
+        message_size = event.payload.fetch(:message_size)
+
+        tags = {client: client, topic: topic}
+
+        if event.payload.key?(:exception)
+          increment("producer.deliver.errors", tags: tags)
+        else
+          increment("producer.produce.messages", tags: tags)
+          histogram("producer.produce.message_size", message_size, tags: tags)
+          count("producer.produce.message_size.sum", message_size, tags: tags)
+          timing("producer.deliver.latency", event.duration, tags: tags)
+          count("producer.deliver.messages", 1, tags: tags)
+        end
+      end
+
+      def deliver_async(event)
+        client = event.payload.fetch(:client_id)
+        topic = event.payload.fetch(:topic)
+        message_size = event.payload.fetch(:message_size)
+        queue_size = event.payload.fetch(:queue_size, 0)
+
+        tags = {client: client, topic: topic}
+
+        if event.payload.key?(:exception)
+          increment("async_producer.produce.errors", tags: tags)
+        else
+          increment("producer.produce.messages", tags: tags)
+          histogram("producer.produce.message_size", message_size, tags: tags)
+          count("producer.produce.message_size.sum", message_size, tags: tags)
+          histogram("async_producer.queue.size", queue_size, tags: tags)
+        end
+      end
+
+      def ack_message(event)
+        tags = {
+          client: event.payload.fetch(:client_id),
+          topic: event.payload.fetch(:topic)
+        }
+
+        increment("producer.ack.messages", tags: tags)
+      end
+
+      def delivery_error(event)
+        tags = {client: event.payload.fetch(:client_id)}
+        increment("producer.ack.errors", tags: tags)
+      end
+
+      attach_to "delivery_boy"
+    end
+  end
+end

data/lib/delivery_boy/instance.rb

@@ -1,11 +1,13 @@
+# frozen_string_literal: true
+
 module DeliveryBoy
   # This class implements the actual logic of DeliveryBoy. The DeliveryBoy module
   # has a module-level singleton instance.
   class Instance
-    def initialize(config, logger)
+    def initialize(config, logger, instrumenter: NullInstrumenter.new)
       @config = config
       @logger = logger
-      @handles = []
+      @instrumenter = instrumenter
     end
 
     def deliver(value, topic:, **options)
@@ -15,9 +17,19 @@ module DeliveryBoy
         options_clone.delete(:create_time)
       end
 
-      sync_producer
-        .produce(payload: value, topic: topic, **options_clone)
-        .wait
+      message_size = value.to_s.bytesize
+
+      instrumentation_payload = {
+        client_id: config.client_id,
+        topic: topic,
+        message_size: message_size
+      }
+
+      @instrumenter.instrument("deliver", instrumentation_payload) do
+        sync_producer
+          .produce(payload: value, topic: topic, **options_clone)
+          .wait
+      end
     end
 
     def deliver_async!(value, topic:, **options)
@@ -27,8 +39,19 @@ module DeliveryBoy
         options_clone.delete(:create_time)
       end
 
-      async_producer
-        .produce(payload: value, topic: topic, **options_clone)
+      message_size = value.to_s.bytesize
+
+      instrumentation_payload = {
+        client_id: config.client_id,
+        topic: topic,
+        message_size: message_size,
+        queue_size: async_producer_queue_size
+      }
+
+      @instrumenter.instrument("deliver_async", instrumentation_payload) do
+        async_producer
+          .produce(payload: value, topic: topic, **options_clone)
+      end
     end
 
     def shutdown
@@ -37,17 +60,43 @@ module DeliveryBoy
     end
 
     def produce(value, topic:, **options)
-      handle = sync_producer.produce(payload: value, topic: topic, **options)
-      handles.push(handle)
+      options_clone = options.clone
+      if options[:create_time]
+        options_clone[:timestamp] = Time.at(options[:create_time])
+        options_clone.delete(:create_time)
+      end
+
+      message_size = value.to_s.bytesize
+
+      instrumentation_payload = {
+        client_id: config.client_id,
+        topic: topic,
+        message_size: message_size,
+        buffer_size: handles.size
+      }
+
+      @instrumenter.instrument("produce_message", instrumentation_payload) do
+        handle = sync_producer.produce(payload: value, topic: topic, **options_clone)
+        handles.push(handle)
+      end
     end
 
     def deliver_messages
-      handles.each(&:wait)
-      handles.clear
+      message_count = handles.size
+
+      instrumentation_payload = {
+        client_id: config.client_id,
+        delivered_message_count: message_count
+      }
+
+      @instrumenter.instrument("deliver_messages", instrumentation_payload) do
+        sync_producer.flush
+        handles.clear
+      end
     end
 
     def clear_buffer
-      handles.clear_buffer
+      handles.clear
     end
 
     def buffer_size
@@ -71,70 +120,240 @@ module DeliveryBoy
     def async_producer
       # The async producer doesn't have to be per-thread, since all deliveries are
       # performed by a single background thread.
-      @async_producer ||= Rdkafka::Config.new({
-        "bootstrap.servers": config.brokers.join(","),
-        "queue.buffering.backpressure.threshold": config.delivery_threshold,
-        "queue.buffering.max.ms": config.delivery_interval_ms
-      }.merge(producer_options)).producer
+      @async_producer ||= begin
+        producer = Rdkafka::Config.new({
+          "bootstrap.servers": config.brokers.join(","),
+          "queue.buffering.backpressure.threshold": config.delivery_threshold,
+          "queue.buffering.max.ms": config.delivery_interval_ms
+        }.merge(producer_options)).producer
+
+        producer.delivery_callback = delivery_callback
+        producer
+      end
     end
 
     def async_producer?
       !@async_producer.nil?
     end
 
+    def async_producer_queue_size
+      return 0 unless async_producer?
+      # rdkafka doesn't expose queue size directly, return 0 as approximation
+      0
+    end
+
+    def delivery_callback
+      instrumenter = @instrumenter
+      client_id = config.client_id
+
+      proc do |delivery_report|
+        if delivery_report.error
+          instrumenter.instrument("delivery_error", {
+            client_id: client_id,
+            error: delivery_report.error
+          })
+        else
+          instrumenter.instrument("ack_message", {
+            client_id: client_id,
+            topic: delivery_report.topic_name,
+            partition: delivery_report.partition,
+            offset: delivery_report.offset
+          })
+        end
+      end
+    end
+
     def kafka
       @kafka ||= Rdkafka::Config.new({
         "bootstrap.servers": config.brokers.join(",")
       }.merge(producer_options))
     end
 
-    # Options for both the sync and async producers.
+    def sasl_options
+      return {} unless config.sasl_mechanism && !config.sasl_mechanism.empty?
+
+      config.validate_aws_msk_iam! if config.sasl_enabled?
+
+      options = {}
+
+      mechanism = config.sasl_mechanism.upcase
+
+      case mechanism
+      when "GSSAPI"
+        options.merge!(gssapi_options)
+      when "PLAIN"
+        options.merge!(plain_options)
+      when "SCRAM-SHA-256", "SCRAM-SHA-512"
+        options["sasl.mechanism"] = mechanism
+        options.merge!(scram_options)
+      when "OAUTHBEARER"
+        options.merge!(oauthbearer_options)
+      else
+        logger.warn "Unknown SASL mechanism: #{config.sasl_mechanism}"
+      end
+
+      options.compact
+    end
+
+    def gssapi_options
+      {
+        "sasl.mechanism" => "GSSAPI",
+        "sasl.kerberos.principal" => config.sasl_gssapi_principal,
+        "sasl.kerberos.keytab" => config.sasl_gssapi_keytab
+      }
+    end
+
+    def plain_options
+      username = config.sasl_username || config.sasl_plain_username
+      password = config.sasl_password || config.sasl_plain_password
+
+      if username.nil? || username.to_s.empty? || password.nil? || password.to_s.empty?
+        raise ConfigError, "PLAIN authentication requires sasl_username and sasl_password to be set"
+      end
+
+      # Note: sasl_plain_authzid doesn't have a librdkafka equivalent
+      # Log warning if set, but don't fail
+      if config.sasl_plain_authzid && !config.sasl_plain_authzid.empty?
+        logger.warn "sasl_plain_authzid is not supported by librdkafka and will be ignored"
+      end
+
+      {
+        "sasl.mechanism" => "PLAIN",
+        "sasl.username" => username,
+        "sasl.password" => password
+      }
+    end
+
+    def scram_options
+      username = config.sasl_username || config.sasl_scram_username
+      password = config.sasl_password || config.sasl_scram_password
+
+      if username.nil? || username.to_s.empty? || password.nil? || password.to_s.empty?
+        raise ConfigError, "SCRAM authentication requires sasl_username and sasl_password to be set"
+      end
+
+      {
+        "sasl.username" => username,
+        "sasl.password" => password
+      }
+    end
+
+    def oauthbearer_options
+      # Check for legacy token provider (not supported)
+      if config.sasl_oauth_token_provider
+        raise ConfigError, <<~ERROR
+          sasl_oauth_token_provider is no longer supported with librdkafka.
+
+          Migration options:
+          1. Use OIDC configuration (recommended for OIDC providers like Auth0, Okta):
+             config.sasl_oauthbearer_method = "oidc"
+             config.sasl_oauthbearer_client_id = "your-client-id"
+             config.sasl_oauthbearer_client_secret = "your-client-secret"
+             config.sasl_oauthbearer_token_endpoint_url = "https://auth.example.com/oauth/token"
+
+          2. Use SCRAM-SHA-256/512 as an alternative authentication method.
+
+          See: https://github.com/zendesk/delivery_boy/blob/master/MIGRATION.md#oauthbearer
+        ERROR
+      end
+
+      if config.sasl_oauthbearer_method&.downcase == "oidc"
+        if config.sasl_oauthbearer_client_id.nil? || config.sasl_oauthbearer_client_id.empty?
+          raise ConfigError, "OAUTHBEARER OIDC requires sasl_oauthbearer_client_id to be set"
+        end
+        if config.sasl_oauthbearer_client_secret.nil? || config.sasl_oauthbearer_client_secret.empty?
+          raise ConfigError, "OAUTHBEARER OIDC requires sasl_oauthbearer_client_secret to be set"
+        end
+        if config.sasl_oauthbearer_token_endpoint_url.nil? || config.sasl_oauthbearer_token_endpoint_url.empty?
+          raise ConfigError, "OAUTHBEARER OIDC requires sasl_oauthbearer_token_endpoint_url to be set"
+        end
+      else
+        raise ConfigError, <<~ERROR
+          OAUTHBEARER requires OIDC configuration.
+
+          Set the following options:
+            config.sasl_oauthbearer_method = "oidc"
+            config.sasl_oauthbearer_client_id = "your-client-id"
+            config.sasl_oauthbearer_client_secret = "your-client-secret"
+            config.sasl_oauthbearer_token_endpoint_url = "https://auth.example.com/oauth/token"
+        ERROR
+      end
+
+      options = {
+        "sasl.mechanism" => "OAUTHBEARER",
+        "sasl.oauthbearer.method" => "oidc",
+        "sasl.oauthbearer.client.id" => config.sasl_oauthbearer_client_id,
+        "sasl.oauthbearer.client.secret" => config.sasl_oauthbearer_client_secret,
+        "sasl.oauthbearer.token.endpoint.url" => config.sasl_oauthbearer_token_endpoint_url
+      }
+
+      options["sasl.oauthbearer.scope"] = config.sasl_oauthbearer_scope if config.sasl_oauthbearer_scope
+      options["sasl.oauthbearer.extensions"] = config.sasl_oauthbearer_extensions if config.sasl_oauthbearer_extensions
+
+      options
+    end
+
+    def security_protocol
+      has_ssl = config.ssl_ca_cert || config.ssl_ca_cert_file_path
+      has_sasl = config.sasl_enabled? || config.sasl_gssapi_principal
+
+      if config.sasl_over_ssl == false && has_ssl
+        raise ConfigError, <<~ERROR
+          sasl_over_ssl=false with SSL certificates configured is not supported by librdkafka.
+
+          librdkafka's security.protocol cannot express "SSL for verification but SASL over plaintext".
+
+          Options:
+          1. Remove SSL certificate configuration to use SASL_PLAINTEXT
+          2. Remove sasl_over_ssl=false to use SASL_SSL (recommended)
+
+          Note: sasl_over_ssl is deprecated and will be removed in a future version.
+        ERROR
+      end
+
+      if has_sasl && has_ssl
+        "SASL_SSL"
+      elsif has_sasl
+        "SASL_PLAINTEXT"
+      elsif has_ssl
+        "SSL"
+      else
+        "PLAINTEXT"
+      end
+    end
+
     def producer_options
       if config.transactional? && config.transactional_id.nil?
         raise "transactional_id must be set"
       end
 
       {
+        "client.id": config.client_id,
         "socket.connection.setup.timeout.ms": config.connection_timeout_ms,
         "socket.timeout.ms": config.socket_timeout_ms,
         "request.required.acks": config.required_acks,
-        "request.timeout.ms": config.ack_timeout,
+        "request.timeout.ms": config.ack_timeout_ms,
         "message.send.max.retries": config.max_retries,
-        "retry.backoff.ms": config.retry_backoff,
+        "retry.backoff.ms": config.retry_backoff_ms,
         "queue.buffering.max.messages": config.max_buffer_size,
-        "queue.buffering.max.kbytes": config.max_buffer_bytesize,
+        "queue.buffering.max.kbytes": config.max_buffer_kbytesize,
         "compression.codec": config.compression_codec, # values none, gzip, snappy, lz4, zstd
         "enable.idempotence": config.idempotent,
         "transactional.id": config.transactional_id,
         "transaction.timeout.ms": config.transactional_timeout_ms,
-
-        # SSL options
+        "security.protocol": security_protocol,
         "ssl.ca.pem": config.ssl_ca_cert,
         "ssl.ca.location": config.ssl_ca_cert_file_path,
         "ssl.certificate.pem": config.ssl_client_cert,
         "ssl.key.pem": config.ssl_client_cert_key,
-        "ssl.key.password": config.ssl_client_cert_key_password,
+        "ssl.key.password": config.ssl_client_cert_key_password
         # ssl_ca_certs_from_system: config.ssl_ca_certs_from_system, # TODO: there is no corresponding librdkafka option. check what this does
         # ssl_verify_hostname: config.ssl_verify_hostname, # check
-        "sasl.kerberos.principal": config.sasl_gssapi_principal,
-        "sasl.kerberos.keytab": config.sasl_gssapi_keytab
-        # sasl_plain_authzid: config.sasl_plain_authzid, # no corresponding librdkafka option, check
-        # 'sasl.username': config.sasl_plain_username,
-        # 'sasl.password': config.sasl_plain_password,
-        # 'sasl.username': config.sasl_scram_username,
-        # 'sasl.passord': config.sasl_scram_password,
-        # 'sasl.mechanism': config.sasl_scram_mechanism,
-        # sasl_over_ssl: config.sasl_over_ssl, # conditional value check again
-        # sasl_oauth_token_provider: config.sasl_oauth_token_provider, # cb code
-        # sasl_aws_msk_iam_access_key_id: config.sasl_aws_msk_iam_access_key_id, # not supported
-        # sasl_aws_msk_iam_secret_key_id: config.sasl_aws_msk_iam_secret_key_id, # not supported
-        # sasl_aws_msk_iam_session_token: config.sasl_aws_msk_iam_session_token, # not supported
-        # sasl_aws_msk_iam_aws_region: config.sasl_aws_msk_iam_aws_region # not supported
-      }
+      }.merge(sasl_options)
     end
 
-    private
-
-    attr_reader :handles
+    def handles
+      Thread.current[:delivery_boy_handles] ||= []
+    end
   end
 end

data/lib/delivery_boy/instrumenter.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module DeliveryBoy
+  class Instrumenter
+    NAMESPACE = "delivery_boy"
+
+    def initialize(default_payload: {})
+      require "active_support/notifications"
+      @default_payload = default_payload
+    end
+
+    def instrument(event_name, payload = {}, &block)
+      ActiveSupport::Notifications.instrument(
+        "#{event_name}.#{NAMESPACE}",
+        @default_payload.merge(payload),
+        &block
+      )
+    end
+  end
+
+  class NullInstrumenter
+    def instrument(*, &block)
+      block&.call
+    end
+  end
+end

data/lib/delivery_boy/railtie.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module DeliveryBoy
   class Railtie < Rails::Railtie
     initializer "delivery_boy.load_config" do
@@ -12,12 +14,15 @@ module DeliveryBoy
       end
 
       if config.datadog_enabled
-        require "kafka/datadog"
+        require "delivery_boy/datadog"
+
+        DeliveryBoy::Datadog.host = config.datadog_host if config.datadog_host.present?
+        DeliveryBoy::Datadog.port = config.datadog_port if config.datadog_port.present?
+        DeliveryBoy::Datadog.namespace = config.datadog_namespace if config.datadog_namespace.present?
+        DeliveryBoy::Datadog.tags = config.datadog_tags if config.datadog_tags.present?
 
-        Kafka::Datadog.host = config.datadog_host if config.datadog_host.present?
-        Kafka::Datadog.port = config.datadog_port if config.datadog_port.present?
-        Kafka::Datadog.namespace = config.datadog_namespace if config.datadog_namespace.present?
-        Kafka::Datadog.tags = config.datadog_tags if config.datadog_tags.present?
+        # Enable instrumentation
+        DeliveryBoy.instrumenter = DeliveryBoy::Instrumenter.new(default_payload: {})
       end
     end
   end

data/lib/delivery_boy/version.rb

@@ -1,3 +1,3 @@
 module DeliveryBoy
-  VERSION = "2.0.0.alpha.1"
+  VERSION = "2.0.0.alpha.2"
 end

data/lib/delivery_boy.rb

@@ -1,6 +1,8 @@
+# frozen_string_literal: true
+
 require "logger"
-# require "kafka"
 require "delivery_boy/version"
+require "delivery_boy/instrumenter"
 require "delivery_boy/instance"
 require "delivery_boy/fake"
 require "delivery_boy/config"
@@ -23,20 +25,19 @@ module DeliveryBoy
     # @param partition_key [String, nil] a key used to deterministically assign
     #   a partition to the message.
     # @return [nil]
-    # @raise [Kafka::BufferOverflow] if the producer's buffer is full.
-    # @raise [Kafka::DeliveryFailed] if delivery failed for some reason.
+    # @raise [Rdkafka::RdkafkaError] if delivery failed for some reason.
     def deliver(value, topic:, **options)
       instance.deliver(value, topic: topic, **options)
     end
 
-    # Like {.deliver_async!}, but handles +Kafka::BufferOverflow+ errors
+    # Like {.deliver_async!}, but handles +Rdkafka::RdkafkaError+ errors
     # by logging them and just going on with normal business.
     #
     # @return [nil]
     def deliver_async(value, topic:, **options)
       deliver_async!(value, topic: topic, **options)
-    rescue Kafka::BufferOverflow
-      logger.error "Message for `#{topic}` dropped due to buffer overflow"
+    rescue Rdkafka::RdkafkaError => e
+      logger.error "Message for `#{topic}` dropped due to error: #{e.message}"
     end
 
     # Like {.deliver}, but returns immediately.
@@ -48,14 +49,14 @@ module DeliveryBoy
       instance.deliver_async!(value, topic: topic, **options)
     end
 
-    # Like {.produce!}, but handles +Kafka::BufferOverflow+ errors
+    # Like {.produce!}, but handles +Rdkafka::RdkafkaError+ errors
     # by logging them and just going on with normal business.
     #
     # @return [nil]
     def produce(value, topic:, **options)
       produce!(value, topic: topic, **options)
-    rescue Kafka::BufferOverflow
-      logger.error "Message for `#{topic}` dropped due to buffer overflow"
+    rescue Rdkafka::RdkafkaError => e
+      logger.error "Message for `#{topic}` dropped due to error: #{e.message}"
     end
 
     # Appends the given message to the producer buffer but does not send it until {.deliver_messages} is called.
@@ -68,7 +69,7 @@ module DeliveryBoy
     # @param partition_key [String, nil] a key used to deterministically assign
     #   a partition to the message.
     # @return [nil]
-    # @raise [Kafka::BufferOverflow] if the producer's buffer is full.
+    # @raise [Rdkafka::RdkafkaError] if the producer's buffer is full.
     def produce!(value, topic:, **options)
       instance.produce(value, topic: topic, **options)
     end
@@ -76,7 +77,7 @@ module DeliveryBoy
     # Delivers the items currently in the producer buffer.
     #
     # @return [nil]
-    # @raise [Kafka::DeliveryFailed] if delivery failed for some reason.
+    # @raise [Rdkafka::RdkafkaError] if delivery failed for some reason.
     def deliver_messages
       instance.deliver_messages
     end
@@ -114,6 +115,15 @@ module DeliveryBoy
 
     attr_writer :logger
 
+    # The instrumenter used by DeliveryBoy for emitting metrics.
+    #
+    # @return [DeliveryBoy::Instrumenter, DeliveryBoy::NullInstrumenter]
+    def instrumenter
+      @instrumenter ||= NullInstrumenter.new
+    end
+
+    attr_writer :instrumenter
+
     # The configuration used by DeliveryBoy.
     #
     # @return [DeliveryBoy::Config]
@@ -150,7 +160,7 @@ module DeliveryBoy
     private
 
     def instance
-      @instance ||= Instance.new(config, logger)
+      @instance ||= Instance.new(config, logger, instrumenter: instrumenter)
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: delivery_boy
 version: !ruby/object:Gem::Version
-  version: 2.0.0.alpha.1
+  version: 2.0.0.alpha.2
 platform: ruby
 authors:
 - Daniel Schierbeck
@@ -44,8 +44,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/CODEOWNERS"
 - ".github/workflows/ci.yml"
-- ".github/workflows/codeql.yaml"
 - ".github/workflows/publish.yml"
 - ".github/workflows/stale.yml"
 - ".gitignore"
@@ -65,8 +65,10 @@ files:
 - lib/delivery_boy.rb
 - lib/delivery_boy/config.rb
 - lib/delivery_boy/config_error.rb
+- lib/delivery_boy/datadog.rb
 - lib/delivery_boy/fake.rb
 - lib/delivery_boy/instance.rb
+- lib/delivery_boy/instrumenter.rb
 - lib/delivery_boy/railtie.rb
 - lib/delivery_boy/rspec.rb
 - lib/delivery_boy/version.rb

data/.github/workflows/codeql.yaml

@@ -1,19 +0,0 @@
-name: "CodeQL public repository scanning"
-
-on:
-  push:
-  schedule:
-    - cron: "0 0 * * *"
-  pull_request_target:
-    types: [opened, synchronize, reopened]
-  workflow_dispatch:
-
-permissions:
-  contents: read
-  security-events: write
-  actions: read
-  packages: read
-
-jobs:
-  trigger-codeql:
-    uses: zendesk/prodsec-code-scanning/.github/workflows/codeql_advanced_shared.yml@production