delivery_boy 1.3.1 → 2.0.0.alpha.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7248183554e5a0d0bc54b355fe18595ffa6e3cbb6cc240c13025d3d15f9f1768
-  data.tar.gz: 2a58d4099652cb784853169a9fa1569c3d2c1902d1523a91d0fffad20467a235
+  metadata.gz: d8bca84f797ae282f1568a53d1542f0bca409e56eaf27c079607e191ccb8841a
+  data.tar.gz: 012ee10f7057df815147ce29bb135d3f8397f1454a9fdff95afa72194c06e884
 SHA512:
-  metadata.gz: c4b00f742af460bcc6f76b0c04b54352fc370156a8a80c7a7f0a53915764f93195c804689a0782b267d42c534a2657e78c003cd0fb56cd016fd162bdf3c03622
-  data.tar.gz: 86844e67ddab037b4cb69cc953f19b1e9b3576db7dd8b0815b26e86f0874994ac830f8d1ee67b14dc21e7ad272e25e0d4bdb6ab5c74a50b40a35815c2f70f5da
+  metadata.gz: 9381929d414fba52aabadc9d483c478bf1968b357db832825d57eedfd214f544cc95fdfee169ae54fb10f07f45ac485f22d3bc17198a614b649295b8341e9363
+  data.tar.gz: 598705aa9268eb0985262958ca4d2d1979d57a06bac366e99113ae302fb6935d5104053389f97236cdfb714430642a383284707b2a5ad9fbdc1615658671e371

data/.github/workflows/ci.yml

@@ -25,11 +25,6 @@ jobs:
         ruby-version: ['2.6', '2.7', '3.0', '3.1', '3.2', '3.3', '3.4']
 
     steps:
-    - name: Run Confluent Platform (Confluent Server)
-      uses: zendesk/cp-all-in-one-action@v0.2.1
-      with:
-        service: broker
-
     - uses: actions/checkout@v4
 
     - name: Set up Ruby
@@ -38,10 +33,17 @@ jobs:
         ruby-version: ${{ matrix.ruby-version }}
         bundler-cache: true # runs 'bundle install' and caches installed gems automatically
 
-    - name: Wait for broker to boot
-      run: 'while ! nc -z localhost 9092; do echo -n "."; sleep 0.1; done'
-
     - name: Run tests
       run: bundle exec rake
-      env:
-        DELIVERY_BOY_BROKERS: localhost:9092
+
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v5
+
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ruby
+        bundler-cache: true
+    - run: bundle exec rake standard

data/.github/workflows/codeql.yaml

@@ -0,0 +1,19 @@
+name: "CodeQL public repository scanning"
+
+on:
+  push:
+  schedule:
+    - cron: "0 0 * * *"
+  pull_request_target:
+    types: [opened, synchronize, reopened]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  security-events: write
+  actions: read
+  packages: read
+
+jobs:
+  trigger-codeql:
+    uses: zendesk/prodsec-code-scanning/.github/workflows/codeql_advanced_shared.yml@production

data/.rspec

@@ -1,2 +1,2 @@
---format documentation
+--order rand
 --color

data/.standard.yml

@@ -0,0 +1 @@
+ruby_version: 2.6

data/CHANGELOG

@@ -2,6 +2,9 @@
 
 ## Unreleased
 
+* `compression_codec` in the `DeliveryBoy::Config` is now coerces its value into
+  a symbol if the value is present.
+
 ## v1.3.1
 
 * Remove the `Kafka::Producer` instance from thread local variable when calling

data/Gemfile

@@ -4,5 +4,9 @@ gemspec
 
 gem "base64"
 gem "bigdecimal"
+gem "debug"
+gem "logger"
 gem "rake", "~> 13.0"
 gem "rspec", "~> 3.0"
+gem "standard", "~> 1.51.1" if RUBY_VERSION > "3.0"
+gem "testcontainers-kafka", github: "testcontainers/testcontainers-ruby"

data/Rakefile

@@ -1,7 +1,12 @@
 require "bundler/setup"
 require "bundler/gem_tasks"
+begin
+  require "standard/rake"
+rescue LoadError
+  :noop
+end
 require "rspec/core/rake_task"
 
 RSpec::Core::RakeTask.new(:spec)
 
-task :default => :spec
+task default: :spec

data/SECURITY.md

@@ -0,0 +1,5 @@
+# Security Policy
+
+## Reporting a Vulnerability
+
+Please report security vulnerabilities by e-mailing: security@zendesk.com

data/delivery_boy.gemspec

@@ -1,25 +1,25 @@
-# coding: utf-8
 lib = File.expand_path("../lib", __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require "delivery_boy/version"
 
 Gem::Specification.new do |spec|
-  spec.name          = "delivery_boy"
-  spec.version       = DeliveryBoy::VERSION
-  spec.authors       = ["Daniel Schierbeck"]
-  spec.email         = ["daniel.schierbeck@gmail.com"]
+  spec.name = "delivery_boy"
+  spec.version = DeliveryBoy::VERSION
+  spec.authors = ["Daniel Schierbeck"]
+  spec.email = ["daniel.schierbeck@gmail.com"]
 
-  spec.summary       = "A simple way to produce messages to Kafka from Ruby applications"
-  spec.description   = "A simple way to produce messages to Kafka from Ruby applications"
-  spec.homepage      = "https://github.com/zendesk/delivery_boy"
-  spec.license       = "Apache License Version 2.0"
+  spec.summary = "A simple way to produce messages to Kafka from Ruby applications"
+  spec.description = "A simple way to produce messages to Kafka from Ruby applications"
+  spec.homepage = "https://github.com/zendesk/delivery_boy"
+  spec.license = "Apache License Version 2.0"
 
-  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features)/})
   end
 
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "ruby-kafka", "~> 1.5"
-  spec.add_dependency "king_konf", "~> 1.0"
+  # spec.add_runtime_dependency "ruby-kafka", "~> 1.0"
+  spec.add_runtime_dependency "king_konf", "~> 1.0"
+  spec.add_runtime_dependency "rdkafka", "> 0.11"
 end

data/lib/delivery_boy/config.rb

@@ -4,6 +4,33 @@ module DeliveryBoy
   class Config < KingKonf::Config
     env_prefix :delivery_boy
 
+    def connection_timeout_ms
+      connect_timeout * 1000
+    end
+
+    def socket_timeout_ms
+      socket_timeout * 1000
+    end
+
+    def transactional_timeout_ms
+      transactional_timeout * 1000
+    end
+
+    def isolation_level
+      transactional ? "read_uncommitted" : "read_committed"
+    end
+
+    def max_buffer_kbytesize
+      max_buffer_bytesize / 1024
+    end
+
+    def delivery_interval_ms
+      delivery_interval * 1000
+    end
+
+    def sasl_username
+    end
+
     # Basic
     list :brokers, items: :string, sep: ",", default: ["localhost:9092"]
     string :client_id, default: "delivery_boy"
@@ -27,11 +54,12 @@ module DeliveryBoy
     integer :retry_backoff, default: 1
     boolean :idempotent, default: false
     boolean :transactional, default: false
+    string :transactional_id, default: nil
     integer :transactional_timeout, default: 60
 
     # Compression
-    integer :compression_threshold, default: 1
-    string :compression_codec, default: nil
+    integer :compression_threshold, default: 1 # deprecated, not an option for RdKafka
+    string :compression_codec, default: "none"
 
     # SSL authentication
     string :ssl_ca_cert, default: nil
@@ -42,10 +70,12 @@ module DeliveryBoy
     boolean :ssl_ca_certs_from_system, default: false
     boolean :ssl_verify_hostname, default: true
 
+    # Supported: GSSAPI, PLAIN, SCRAM-SHA-256, SCRAM-SHA-512, OAUTHBEARER
+    string :sasl_mechanism, default: "GSSAPI"
     # SASL authentication
     string :sasl_gssapi_principal
     string :sasl_gssapi_keytab
-    string :sasl_plain_authzid, default: ''
+    string :sasl_plain_authzid, default: ""
     string :sasl_plain_username
     string :sasl_plain_password
     string :sasl_scram_username

data/lib/delivery_boy/config_error.rb

@@ -1,4 +1,3 @@
 module DeliveryBoy
   ConfigError = Class.new(StandardError)
 end
-

data/lib/delivery_boy/fake.rb

@@ -1,5 +1,4 @@
 module DeliveryBoy
-
   # A fake implementation that is useful for testing.
   class Fake
     FakeMessage = Struct.new(:value, :topic, :key, :headers, :offset, :partition, :partition_key, :create_time) do
@@ -9,8 +8,8 @@ module DeliveryBoy
     end
 
     def initialize
-      @messages = Hash.new {|h, k| h[k] = [] }
-      @buffer = Hash.new {|h, k| h[k] = [] }
+      @messages = Hash.new { |h, k| h[k] = [] }
+      @buffer = Hash.new { |h, k| h[k] = [] }
       @delivery_lock = Mutex.new
     end
 
@@ -25,7 +24,7 @@ module DeliveryBoy
       nil
     end
 
-    alias deliver_async! deliver
+    alias_method :deliver_async!, :deliver
 
     def produce(value, topic:, key: nil, headers: {}, partition: nil, partition_key: nil, create_time: Time.now)
       @delivery_lock.synchronize do

data/lib/delivery_boy/instance.rb

@@ -1,49 +1,57 @@
 module DeliveryBoy
-
   # This class implements the actual logic of DeliveryBoy. The DeliveryBoy module
   # has a module-level singleton instance.
   class Instance
     def initialize(config, logger)
       @config = config
       @logger = logger
-      @async_producer = nil
+      @handles = []
     end
 
     def deliver(value, topic:, **options)
-      sync_producer.produce(value, topic: topic, **options)
-      sync_producer.deliver_messages
-    rescue
-      # Make sure to clear any buffered messages if there's an error.
-      clear_buffer
-
-      raise
+      options_clone = options.clone
+      if options[:create_time]
+        options_clone[:timestamp] = Time.at(options[:create_time])
+        options_clone.delete(:create_time)
+      end
+
+      sync_producer
+        .produce(payload: value, topic: topic, **options_clone)
+        .wait
     end
 
     def deliver_async!(value, topic:, **options)
-      async_producer.produce(value, topic: topic, **options)
+      options_clone = options.clone
+      if options[:create_time]
+        options_clone[:timestamp] = Time.at(options[:create_time])
+        options_clone.delete(:create_time)
+      end
+
+      async_producer
+        .produce(payload: value, topic: topic, **options_clone)
     end
 
     def shutdown
-      sync_producer.shutdown if sync_producer?
-      async_producer.shutdown if async_producer?
-
-      Thread.current[:delivery_boy_sync_producer] = nil
+      sync_producer.close if sync_producer?
+      async_producer.close if async_producer?
     end
 
     def produce(value, topic:, **options)
-      sync_producer.produce(value, topic: topic, **options)
+      handle = sync_producer.produce(payload: value, topic: topic, **options)
+      handles.push(handle)
     end
 
     def deliver_messages
-      sync_producer.deliver_messages
+      handles.each(&:wait)
+      handles.clear
     end
 
     def clear_buffer
-      sync_producer.clear_buffer
+      handles.clear_buffer
     end
 
     def buffer_size
-      sync_producer.buffer_size
+      handles.size
     end
 
     private
@@ -53,7 +61,7 @@ module DeliveryBoy
     def sync_producer
       # We want synchronous producers to be per-thread in order to avoid problems with
       # concurrent deliveries.
-      Thread.current[:delivery_boy_sync_producer] ||= kafka.producer(**producer_options)
+      Thread.current[:delivery_boy_sync_producer] ||= kafka.producer
     end
 
     def sync_producer?
@@ -63,12 +71,11 @@ module DeliveryBoy
     def async_producer
       # The async producer doesn't have to be per-thread, since all deliveries are
       # performed by a single background thread.
-      @async_producer ||= kafka.async_producer(
-        max_queue_size: config.max_queue_size,
-        delivery_threshold: config.delivery_threshold,
-        delivery_interval: config.delivery_interval,
-        **producer_options,
-      )
+      @async_producer ||= Rdkafka::Config.new({
+        "bootstrap.servers": config.brokers.join(","),
+        "queue.buffering.backpressure.threshold": config.delivery_threshold,
+        "queue.buffering.max.ms": config.delivery_interval_ms
+      }.merge(producer_options)).producer
     end
 
     def async_producer?
@@ -76,51 +83,58 @@ module DeliveryBoy
     end
 
     def kafka
-      @kafka ||= Kafka.new(
-        seed_brokers: config.brokers,
-        client_id: config.client_id,
-        logger: logger,
-        connect_timeout: config.connect_timeout,
-        socket_timeout: config.socket_timeout,
-        ssl_ca_cert: config.ssl_ca_cert,
-        ssl_ca_cert_file_path: config.ssl_ca_cert_file_path,
-        ssl_client_cert: config.ssl_client_cert,
-        ssl_client_cert_key: config.ssl_client_cert_key,
-        ssl_client_cert_key_password: config.ssl_client_cert_key_password,
-        ssl_ca_certs_from_system: config.ssl_ca_certs_from_system,
-        ssl_verify_hostname: config.ssl_verify_hostname,
-        sasl_gssapi_principal: config.sasl_gssapi_principal,
-        sasl_gssapi_keytab: config.sasl_gssapi_keytab,
-        sasl_plain_authzid: config.sasl_plain_authzid,
-        sasl_plain_username: config.sasl_plain_username,
-        sasl_plain_password: config.sasl_plain_password,
-        sasl_scram_username: config.sasl_scram_username,
-        sasl_scram_password: config.sasl_scram_password,
-        sasl_scram_mechanism: config.sasl_scram_mechanism,
-        sasl_over_ssl: config.sasl_over_ssl,
-        sasl_oauth_token_provider: config.sasl_oauth_token_provider,
-        sasl_aws_msk_iam_access_key_id: config.sasl_aws_msk_iam_access_key_id,
-        sasl_aws_msk_iam_secret_key_id: config.sasl_aws_msk_iam_secret_key_id,
-        sasl_aws_msk_iam_session_token: config.sasl_aws_msk_iam_session_token,
-        sasl_aws_msk_iam_aws_region: config.sasl_aws_msk_iam_aws_region
-      )
+      @kafka ||= Rdkafka::Config.new({
+        "bootstrap.servers": config.brokers.join(",")
+      }.merge(producer_options))
     end
 
     # Options for both the sync and async producers.
     def producer_options
+      if config.transactional? && config.transactional_id.nil?
+        raise "transactional_id must be set"
+      end
+
       {
-        required_acks: config.required_acks,
-        ack_timeout: config.ack_timeout,
-        max_retries: config.max_retries,
-        retry_backoff: config.retry_backoff,
-        max_buffer_size: config.max_buffer_size,
-        max_buffer_bytesize: config.max_buffer_bytesize,
-        compression_codec: (config.compression_codec.to_sym if config.compression_codec),
-        compression_threshold: config.compression_threshold,
-        idempotent: config.idempotent,
-        transactional: config.transactional,
-        transactional_timeout: config.transactional_timeout,
+        "socket.connection.setup.timeout.ms": config.connection_timeout_ms,
+        "socket.timeout.ms": config.socket_timeout_ms,
+        "request.required.acks": config.required_acks,
+        "request.timeout.ms": config.ack_timeout,
+        "message.send.max.retries": config.max_retries,
+        "retry.backoff.ms": config.retry_backoff,
+        "queue.buffering.max.messages": config.max_buffer_size,
+        "queue.buffering.max.kbytes": config.max_buffer_bytesize,
+        "compression.codec": config.compression_codec, # values none, gzip, snappy, lz4, zstd
+        "enable.idempotence": config.idempotent,
+        "transactional.id": config.transactional_id,
+        "transaction.timeout.ms": config.transactional_timeout_ms,
+
+        # SSL options
+        "ssl.ca.pem": config.ssl_ca_cert,
+        "ssl.ca.location": config.ssl_ca_cert_file_path,
+        "ssl.certificate.pem": config.ssl_client_cert,
+        "ssl.key.pem": config.ssl_client_cert_key,
+        "ssl.key.password": config.ssl_client_cert_key_password,
+        # ssl_ca_certs_from_system: config.ssl_ca_certs_from_system, # TODO: there is no corresponding librdkafka option. check what this does
+        # ssl_verify_hostname: config.ssl_verify_hostname, # check
+        "sasl.kerberos.principal": config.sasl_gssapi_principal,
+        "sasl.kerberos.keytab": config.sasl_gssapi_keytab
+        # sasl_plain_authzid: config.sasl_plain_authzid, # no corresponding librdkafka option, check
+        # 'sasl.username': config.sasl_plain_username,
+        # 'sasl.password': config.sasl_plain_password,
+        # 'sasl.username': config.sasl_scram_username,
+        # 'sasl.passord': config.sasl_scram_password,
+        # 'sasl.mechanism': config.sasl_scram_mechanism,
+        # sasl_over_ssl: config.sasl_over_ssl, # conditional value check again
+        # sasl_oauth_token_provider: config.sasl_oauth_token_provider, # cb code
+        # sasl_aws_msk_iam_access_key_id: config.sasl_aws_msk_iam_access_key_id, # not supported
+        # sasl_aws_msk_iam_secret_key_id: config.sasl_aws_msk_iam_secret_key_id, # not supported
+        # sasl_aws_msk_iam_session_token: config.sasl_aws_msk_iam_session_token, # not supported
+        # sasl_aws_msk_iam_aws_region: config.sasl_aws_msk_iam_aws_region # not supported
       }
     end
+
+    private
+
+    attr_reader :handles
   end
 end

data/lib/delivery_boy/version.rb

@@ -1,3 +1,3 @@
 module DeliveryBoy
-  VERSION = "1.3.1"
+  VERSION = "2.0.0.alpha.1"
 end

data/lib/delivery_boy.rb

@@ -1,15 +1,15 @@
 require "logger"
-require "kafka"
+# require "kafka"
 require "delivery_boy/version"
 require "delivery_boy/instance"
 require "delivery_boy/fake"
 require "delivery_boy/config"
 require "delivery_boy/config_error"
 require "delivery_boy/railtie" if defined?(Rails::Railtie)
+require "rdkafka"
 
 module DeliveryBoy
   class << self
-
     # Write a message to a specified Kafka topic synchronously.
     #
     # Keep in mind that the client will block until the message has been

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: delivery_boy
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 2.0.0.alpha.1
 platform: ruby
 authors:
 - Daniel Schierbeck
@@ -10,33 +10,33 @@ cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: ruby-kafka
+  name: king_konf
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.5'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.5'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: king_konf
+  name: rdkafka
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0.11'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0.11'
 description: A simple way to produce messages to Kafka from Ruby applications
 email:
 - daniel.schierbeck@gmail.com
@@ -45,15 +45,18 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".github/workflows/ci.yml"
+- ".github/workflows/codeql.yaml"
 - ".github/workflows/publish.yml"
 - ".github/workflows/stale.yml"
 - ".gitignore"
 - ".rspec"
+- ".standard.yml"
 - CHANGELOG
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
+- SECURITY.md
 - bin/console
 - bin/setup
 - delivery_boy.gemspec
@@ -87,7 +90,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.7
+rubygems_version: 3.6.9
 specification_version: 4
 summary: A simple way to produce messages to Kafka from Ruby applications
 test_files: []