delivery_boy 1.3.1 → 2.0.0.alpha.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7248183554e5a0d0bc54b355fe18595ffa6e3cbb6cc240c13025d3d15f9f1768
-  data.tar.gz: 2a58d4099652cb784853169a9fa1569c3d2c1902d1523a91d0fffad20467a235
+  metadata.gz: a4dac7da9e416e7c55878674f91006d43b73811e27b1aa05897604d48c6e3162
+  data.tar.gz: b5536e1e66d7c8882fe6452b552f50f2fa4e37846870bea0c6b6f5ceef399028
 SHA512:
-  metadata.gz: c4b00f742af460bcc6f76b0c04b54352fc370156a8a80c7a7f0a53915764f93195c804689a0782b267d42c534a2657e78c003cd0fb56cd016fd162bdf3c03622
-  data.tar.gz: 86844e67ddab037b4cb69cc953f19b1e9b3576db7dd8b0815b26e86f0874994ac830f8d1ee67b14dc21e7ad272e25e0d4bdb6ab5c74a50b40a35815c2f70f5da
+  metadata.gz: 8390928820e66c27e3ec762344df31d0e4101831e0f1cca21b2289939fbc23cad4f554b17314eb7f92d48e7f3bd1b6d90ebfceac3e3b41065dbabacf3f2aa0b6
+  data.tar.gz: 2bf809365bd9f693cf1dd03f61e6cb37c5c43682087ecc013cd0626d558a83bdc04b4dd4b065dc6a9e0170479dc192f2b26f56cc3501ce77aa820b351dddb817

data/.github/CODEOWNERS

@@ -0,0 +1,4 @@
+# CODEOWNERS file
+# This file defines who should review code changes in this repository.
+
+* @zendesk/core-gem-owners

data/.github/workflows/ci.yml

@@ -25,11 +25,6 @@ jobs:
         ruby-version: ['2.6', '2.7', '3.0', '3.1', '3.2', '3.3', '3.4']
 
     steps:
-    - name: Run Confluent Platform (Confluent Server)
-      uses: zendesk/cp-all-in-one-action@v0.2.1
-      with:
-        service: broker
-
     - uses: actions/checkout@v4
 
     - name: Set up Ruby
@@ -38,10 +33,17 @@ jobs:
         ruby-version: ${{ matrix.ruby-version }}
         bundler-cache: true # runs 'bundle install' and caches installed gems automatically
 
-    - name: Wait for broker to boot
-      run: 'while ! nc -z localhost 9092; do echo -n "."; sleep 0.1; done'
-
     - name: Run tests
       run: bundle exec rake
-      env:
-        DELIVERY_BOY_BROKERS: localhost:9092
+
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v5
+
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ruby
+        bundler-cache: true
+    - run: bundle exec rake standard

data/.rspec

@@ -1,2 +1,2 @@
---format documentation
+--order rand
 --color

data/.standard.yml

@@ -0,0 +1 @@
+ruby_version: 2.6

data/CHANGELOG

@@ -1,6 +1,55 @@
 # Changelog
 
-## Unreleased
+## Unreleased (v2.0.0)
+
+### Migration to librdkafka
+
+* Migrated from ruby-kafka to rdkafka (librdkafka) for improved performance and stability
+
+### New Features
+
+* PLAIN and SCRAM-SHA-256/512 SASL authentication now fully functional with librdkafka
+* Added `sasl_username` and `sasl_password` as consolidated config options (work for both PLAIN and SCRAM)
+* Added automatic `security.protocol` detection (PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL)
+* Backward compatible SASL configuration - existing mechanism-specific options (sasl_plain_username, sasl_scram_username) continue to work
+* OAUTHBEARER authentication via OIDC configuration (librdkafka handles token refresh automatically)
+
+### Breaking Changes & Known Limitations
+
+* **AWS MSK IAM authentication is no longer supported** (librdkafka limitation)
+  - Alternative 1: Use AWS MSK SCRAM-SHA-512 authentication (recommended)
+  - Alternative 2: Use mTLS (mutual TLS) with client certificates
+* **OAUTHBEARER authentication changed from callback-based to OIDC configuration**
+  - `sasl_oauth_token_provider` is no longer supported
+  - Use new OIDC config options instead: `sasl_oauthbearer_method`, `sasl_oauthbearer_client_id`, `sasl_oauthbearer_client_secret`, `sasl_oauthbearer_token_endpoint_url`
+  - librdkafka handles token refresh automatically (no custom code needed)
+* `sasl_plain_authzid` is not supported by librdkafka and will be ignored with a warning
+* `compression_threshold` is no longer supported - librdkafka applies compression automatically to all batches when `compression_codec` is set
+
+### Config options no longer used
+
+The following config options are silently ignored (no error, but no effect):
+* `max_queue_size` - rdkafka manages its own internal queue
+* `ssl_ca_certs_from_system` - no librdkafka equivalent
+* `ssl_verify_hostname` - not mapped to librdkafka
+* `sasl_scram_mechanism` - use `sasl_mechanism` instead
+* `log_level` - rdkafka has its own logging configuration
+
+### Datadog Integration
+
+* Datadog metrics continue to work with the same metric names as before
+* Now uses `DeliveryBoy::Datadog` instead of `Kafka::Datadog`
+* Instrumentation is built on top of ActiveSupport::Notifications
+* Some metrics are no longer available due to rdkafka's different architecture:
+  - `producer.buffer.fill_ratio` / `producer.buffer.fill_percentage`
+  - `producer.deliver.attempts`
+  - `producer.ack.delay`
+  - `async_producer.queue.fill_ratio`
+
+### Other Changes
+
+* `compression_codec` in the `DeliveryBoy::Config` is now coerces its value into
+  a symbol if the value is present.
 
 ## v1.3.1
 

data/Gemfile

@@ -4,5 +4,11 @@ gemspec
 
 gem "base64"
 gem "bigdecimal"
+gem "debug"
+gem "logger"
 gem "rake", "~> 13.0"
 gem "rspec", "~> 3.0"
+gem "standard", "~> 1.51.1" if RUBY_VERSION > "3.0"
+gem "testcontainers-kafka", github: "testcontainers/testcontainers-ruby"
+gem "activesupport"
+gem "dogstatsd-ruby"

data/README.md

@@ -172,7 +172,7 @@ The codec used to compress messages. Must be either `snappy` or `gzip`.
 
 ##### `compression_threshold`
 
-The minimum number of messages that must be buffered before compression is attempted. By default only one message is required. Only relevant if `compression_codec` is set.
+**Deprecated:** This setting has no effect with librdkafka. Compression is applied automatically to all batches when `compression_codec` is set.
 
 #### Network
 
@@ -226,88 +226,136 @@ The password required to read the ssl_client_cert_key. Must be used in combinati
 
 #### SASL Authentication and authorization
 
-See [ruby-kafka](https://github.com/zendesk/ruby-kafka#authentication-using-sasl) for more information.
+DeliveryBoy supports SASL authentication with multiple mechanisms.
 
-Use it through `GSSAPI`, `PLAIN` _or_ `OAUTHBEARER`.
+##### GSSAPI (Kerberos)
 
-##### `sasl_gssapi_principal`
+###### `sasl_gssapi_principal`
 
 The GSSAPI principal.
 
-##### `sasl_gssapi_keytab`
+###### `sasl_gssapi_keytab`
 
 Optional GSSAPI keytab.
 
-##### `sasl_plain_authzid`
+**Example:**
 
-The authorization identity to use.
+```ruby
+DeliveryBoy.configure do |config|
+  config.sasl_mechanism = "GSSAPI"
+  config.sasl_gssapi_principal = "kafka/hostname@REALM"
+  config.sasl_gssapi_keytab = "/path/to/keytab"
+end
+```
+
+##### PLAIN Authentication
+
+###### `sasl_plain_username`
+
+The username used to authenticate (legacy option).
 
-##### `sasl_plain_username`
+###### `sasl_plain_password`
 
-The username used to authenticate.
+The password used to authenticate (legacy option).
 
-##### `sasl_plain_password`
+###### `sasl_username`
 
-The password used to authenticate.
+The username used to authenticate (new consolidated option, works for both PLAIN and SCRAM).
 
-##### `sasl_oauth_token_provider`
+###### `sasl_password`
 
-A instance of a class which implements the `token` method.
-As described in [ruby-kafka](https://github.com/zendesk/ruby-kafka/tree/c3e90bc355fad1e27b9af1048966ff08d3d5735b#oauthbearer)
+The password used to authenticate (new consolidated option, works for both PLAIN and SCRAM).
+
+**Example:**
 
 ```ruby
-class TokenProvider
-  def token
-    "oauth-token"
-  end
+DeliveryBoy.configure do |config|
+  config.sasl_mechanism = "PLAIN"
+  # You can use either the new consolidated options:
+  config.sasl_username = "your-username"
+  config.sasl_password = "your-password"
+  # Or the legacy mechanism-specific options:
+  # config.sasl_plain_username = "your-username"
+  # config.sasl_plain_password = "your-password"
 end
+```
+
+##### SCRAM Authentication
+
+Supports SCRAM-SHA-256 and SCRAM-SHA-512 mechanisms.
+
+###### `sasl_scram_username`
 
+The username used to authenticate (legacy option).
+
+###### `sasl_scram_password`
+
+The password used to authenticate (legacy option).
+
+**Example:**
+
+```ruby
 DeliveryBoy.configure do |config|
-  config.sasl_oauth_token_provider = TokenProvider.new
-  config.ssl_ca_certs_from_system = true
+  config.sasl_mechanism = "SCRAM-SHA-256"  # or "SCRAM-SHA-512"
+  # You can use either the new consolidated options:
+  config.sasl_username = "your-username"
+  config.sasl_password = "your-password"
+  # Or the legacy mechanism-specific options:
+  # config.sasl_scram_username = "your-username"
+  # config.sasl_scram_password = "your-password"
 end
 ```
 
-#### AWS MSK IAM Authentication and Authorization
+##### OAUTHBEARER (OIDC)
+
+OAUTHBEARER authentication is supported via OIDC configuration. librdkafka handles token acquisition and refresh automatically.
+
+###### `sasl_oauthbearer_method`
 
-##### sasl_aws_msk_iam_access_key_id
+Set to `"oidc"` to enable OIDC-based OAUTHBEARER authentication.
 
-The AWS IAM access key. Required.
+###### `sasl_oauthbearer_client_id`
 
-##### sasl_aws_msk_iam_secret_key_id
+The OAuth client ID for your application.
 
-The AWS IAM secret access key. Required.
+###### `sasl_oauthbearer_client_secret`
 
-##### sasl_aws_msk_iam_aws_region
+The OAuth client secret for your application.
 
-The AWS region. Required.
+###### `sasl_oauthbearer_token_endpoint_url`
 
-##### sasl_aws_msk_iam_session_token
+The URL of the OAuth token endpoint (e.g., `https://auth.example.com/oauth/token`).
 
-The session token. This value can be optional.
+###### `sasl_oauthbearer_scope` (optional)
 
-###### Examples 
+OAuth scope to request.
 
-Using a role arn and web identity token to generate temporary credentials:
+###### `sasl_oauthbearer_extensions` (optional)
+
+Additional SASL extensions as comma-separated key=value pairs.
 
 ```ruby
-require "aws-sdk-core"
-require "delivery_boy"
-
-role = Aws::AssumeRoleWebIdentityCredentials.new(
-  role_arn: ENV["AWS_ROLE_ARN"],
-  web_identity_token_file: ENV["AWS_WEB_IDENTITY_TOKEN_FILE"]
-)
-
-DeliveryBoy.configure do |c|
-  c.sasl_aws_msk_iam_access_key_id = role.credentials.access_key_id
-  c.sasl_aws_msk_iam_secret_key_id = role.credentials.secret_access_key
-  c.sasl_aws_msk_iam_session_token = role.credentials.session_token
-  c.sasl_aws_msk_iam_aws_region    = ENV["AWS_REGION"]
-  c.ssl_ca_certs_from_system       = true
+DeliveryBoy.configure do |config|
+  config.sasl_mechanism = "OAUTHBEARER"
+  config.sasl_oauthbearer_method = "oidc"
+  config.sasl_oauthbearer_client_id = "your-client-id"
+  config.sasl_oauthbearer_client_secret = "your-client-secret"
+  config.sasl_oauthbearer_token_endpoint_url = "https://auth.example.com/oauth/token"
+  # Optional:
+  # config.sasl_oauthbearer_scope = "kafka"
 end
 ```
 
+**Note:** The legacy `sasl_oauth_token_provider` callback is no longer supported. Use OIDC configuration instead.
+
+#### AWS MSK IAM Authentication
+
+**Note:** AWS MSK IAM authentication is not supported by librdkafka.
+
+**Recommended alternatives:**
+1. **Use AWS MSK SCRAM authentication** - Create SCRAM credentials in AWS Secrets Manager and use SCRAM-SHA-512
+2. **Use mTLS** - Configure mutual TLS with client certificates
+
 ### Testing
 
 DeliveryBoy provides a test mode out of the box. When this mode is enabled, messages will be stored in memory rather than being sent to Kafka. If you use RSpec, enabling test mode is as easy as adding this to your spec helper:

data/Rakefile

@@ -1,7 +1,12 @@
 require "bundler/setup"
 require "bundler/gem_tasks"
+begin
+  require "standard/rake"
+rescue LoadError
+  :noop
+end
 require "rspec/core/rake_task"
 
 RSpec::Core::RakeTask.new(:spec)
 
-task :default => :spec
+task default: :spec

data/SECURITY.md

@@ -0,0 +1,5 @@
+# Security Policy
+
+## Reporting a Vulnerability
+
+Please report security vulnerabilities by e-mailing: security@zendesk.com

data/delivery_boy.gemspec

@@ -1,25 +1,24 @@
-# coding: utf-8
 lib = File.expand_path("../lib", __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require "delivery_boy/version"
 
 Gem::Specification.new do |spec|
-  spec.name          = "delivery_boy"
-  spec.version       = DeliveryBoy::VERSION
-  spec.authors       = ["Daniel Schierbeck"]
-  spec.email         = ["daniel.schierbeck@gmail.com"]
+  spec.name = "delivery_boy"
+  spec.version = DeliveryBoy::VERSION
+  spec.authors = ["Daniel Schierbeck"]
+  spec.email = ["daniel.schierbeck@gmail.com"]
 
-  spec.summary       = "A simple way to produce messages to Kafka from Ruby applications"
-  spec.description   = "A simple way to produce messages to Kafka from Ruby applications"
-  spec.homepage      = "https://github.com/zendesk/delivery_boy"
-  spec.license       = "Apache License Version 2.0"
+  spec.summary = "A simple way to produce messages to Kafka from Ruby applications"
+  spec.description = "A simple way to produce messages to Kafka from Ruby applications"
+  spec.homepage = "https://github.com/zendesk/delivery_boy"
+  spec.license = "Apache License Version 2.0"
 
-  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features)/})
   end
 
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "ruby-kafka", "~> 1.5"
-  spec.add_dependency "king_konf", "~> 1.0"
+  spec.add_runtime_dependency "king_konf", "~> 1.0"
+  spec.add_runtime_dependency "rdkafka", "> 0.11"
 end

data/lib/delivery_boy/config.rb

@@ -4,15 +4,67 @@ module DeliveryBoy
   class Config < KingKonf::Config
     env_prefix :delivery_boy
 
+    def connection_timeout_ms
+      connect_timeout * 1000
+    end
+
+    def socket_timeout_ms
+      socket_timeout * 1000
+    end
+
+    def transactional_timeout_ms
+      transactional_timeout * 1000
+    end
+
+    def max_buffer_kbytesize
+      max_buffer_bytesize / 1024
+    end
+
+    def delivery_interval_ms
+      delivery_interval * 1000
+    end
+
+    def ack_timeout_ms
+      ack_timeout * 1000
+    end
+
+    def retry_backoff_ms
+      retry_backoff * 1000
+    end
+
+    def sasl_enabled?
+      return false unless sasl_mechanism && !sasl_mechanism.empty?
+      sasl_mechanism.upcase != "GSSAPI"
+    end
+
+    def validate_aws_msk_iam!
+      if sasl_aws_msk_iam_access_key_id || sasl_aws_msk_iam_secret_key_id || sasl_aws_msk_iam_aws_region
+        raise ConfigError, <<~ERROR
+          AWS MSK IAM authentication is not supported by librdkafka.
+
+          Alternatives:
+          1. Use AWS MSK SCRAM-SHA-512 authentication (recommended)
+             - Create SCRAM credentials in AWS Secrets Manager
+             - Set sasl_mechanism = "SCRAM-SHA-512"
+             - Set sasl_username and sasl_password
+
+          2. Use mTLS (mutual TLS) with client certificates
+             - Configure ssl_client_cert and ssl_client_cert_key
+
+          See migration guide: https://github.com/zendesk/delivery_boy/blob/master/MIGRATION.md#aws-msk-iam
+        ERROR
+      end
+    end
+
     # Basic
     list :brokers, items: :string, sep: ",", default: ["localhost:9092"]
     string :client_id, default: "delivery_boy"
     string :log_level, default: nil
 
-    # Buffering
+    # Buffering (defaults match librdkafka defaults to avoid queue overflow at high throughput)
     integer :max_buffer_bytesize, default: 10_000_000
-    integer :max_buffer_size, default: 1000
-    integer :max_queue_size, default: 1000
+    integer :max_buffer_size, default: 100_000
+    integer :max_queue_size, default: 100_000
 
     # Network timeouts
     integer :connect_timeout, default: 10
@@ -27,11 +79,12 @@ module DeliveryBoy
     integer :retry_backoff, default: 1
     boolean :idempotent, default: false
     boolean :transactional, default: false
+    string :transactional_id, default: nil
     integer :transactional_timeout, default: 60
 
     # Compression
-    integer :compression_threshold, default: 1
-    string :compression_codec, default: nil
+    integer :compression_threshold, default: 1 # deprecated, not an option for RdKafka
+    string :compression_codec, default: "none"
 
     # SSL authentication
     string :ssl_ca_cert, default: nil
@@ -42,10 +95,12 @@ module DeliveryBoy
     boolean :ssl_ca_certs_from_system, default: false
     boolean :ssl_verify_hostname, default: true
 
+    # Supported: GSSAPI, PLAIN, SCRAM-SHA-256, SCRAM-SHA-512, OAUTHBEARER
+    string :sasl_mechanism, default: nil
     # SASL authentication
     string :sasl_gssapi_principal
     string :sasl_gssapi_keytab
-    string :sasl_plain_authzid, default: ''
+    string :sasl_plain_authzid, default: ""
     string :sasl_plain_username
     string :sasl_plain_password
     string :sasl_scram_username
@@ -53,9 +108,21 @@ module DeliveryBoy
     string :sasl_scram_mechanism
     boolean :sasl_over_ssl, default: true
 
-    # SASL OAUTHBEARER
+    # New consolidated SASL options (librdkafka-aligned)
+    string :sasl_username, default: nil
+    string :sasl_password, default: nil
+
+    # SASL OAUTHBEARER (legacy - callback-based, not supported with librdkafka)
     attr_accessor :sasl_oauth_token_provider
 
+    # SASL OAUTHBEARER OIDC (librdkafka native support)
+    string :sasl_oauthbearer_method, default: nil # "oidc" for OIDC-based auth
+    string :sasl_oauthbearer_client_id, default: nil
+    string :sasl_oauthbearer_client_secret, default: nil
+    string :sasl_oauthbearer_token_endpoint_url, default: nil
+    string :sasl_oauthbearer_scope, default: nil
+    string :sasl_oauthbearer_extensions, default: nil
+
     # AWS IAM authentication
     string :sasl_aws_msk_iam_access_key_id
     string :sasl_aws_msk_iam_secret_key_id

data/lib/delivery_boy/config_error.rb

@@ -1,4 +1,3 @@
 module DeliveryBoy
   ConfigError = Class.new(StandardError)
 end
-