karafka-rdkafka 0.15.0.alpha1 → 0.15.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2cd7ecb658a7aedb5953f4d39c8941a282d6dc7001c7600129d86ebb05a550ce
-  data.tar.gz: 1dc2ebcb1deebe94197f216f0da3d78e2ed5d55de9b1be8b0654a1042e2d4289
+  metadata.gz: bc8e5edfc4eccc512562aa6937a6b3c7d6ad417b83fa136b9090af3f7bfe7bc3
+  data.tar.gz: f1b4645880ffe22d1512acee977d045d3390d3dc1f8573ca41a8db22d6c1e3d1
 SHA512:
-  metadata.gz: a300e2dcdf5aac16b59b0289a71fec440747e9238c5a801572c7d1ce6a433843ab47e76915a239f860a24cd832323dab952e09b62c7184462c806183958b2a85
-  data.tar.gz: c1a3b1d23522f2941b9d39d3b5955f40d3205a3820188c7cba02f7825b17bf534963fec68956271908394e443f7b8a84edcdf972021a74f3ed4bcf93d76b1b6d
+  metadata.gz: aeba97f575ceea5a67dcf6fe3e31fc9ec98ba6049ece2b23daa7af8d959e8074be5f20eca71134e32ec86076a75bb44dfc3ee9b0aa4eb85f27af6933b922138a
+  data.tar.gz: 2089b7f52c3a8a7fa58e682cc73f6c5a8d363affd419a9f4d7c96228fbdc3b96129a4e2b3739fba4ef82cf44c6fe2d00551aedfa33fa3b5d5d2a7ad80f91a18f

data/.gitignore

@@ -10,3 +10,5 @@ ext/librdkafka.*
 doc
 coverage
 vendor
+.idea/
+out/

data/CHANGELOG.md

@@ -1,8 +1,10 @@
 # Rdkafka Changelog
 
 ## 0.15.0 (Unreleased)
-- [Feature] Support incremental config describe + alter API.
+- **[Feature]** Oauthbearer token refresh callback (bruce-szalwinski-he)
+- **[Feature]** Support incremental config describe + alter API (mensfeld)
 - [Enhancement] Replace time poll based wait engine with an event based to improve response times on blocking operations and wait (nijikon + mensfeld)
+- [Enhancement] Allow for usage of the second regex engine of librdkafka by setting `RDKAFKA_DISABLE_REGEX_EXT` during build (mensfeld)
 - [Change] The `wait_timeout` argument in `AbstractHandle.wait` method is deprecated and will be removed in future versions without replacement. We don't rely on it's value anymore (nijikon)
 
 ## 0.14.10 (2024-02-08)

data/ext/Rakefile

@@ -27,6 +27,14 @@ task :default => :clean do
       :sha256 => Rdkafka::LIBRDKAFKA_SOURCE_SHA256
     }
     recipe.configure_options = ["--host=#{recipe.host}"]
+
+    # Disable using libc regex engine in favor of the embedded one
+    # The default regex engine of librdkafka does not always work exactly as most of the users
+    # would expect, hence this flag allows for changing it to the other one
+    if ENV.key?('RDKAFKA_DISABLE_REGEX_EXT')
+      recipe.configure_options << '--disable-regex-ext'
+    end
+
     recipe.cook
     # Move dynamic library we're interested in
     if recipe.host.include?('darwin')

data/lib/rdkafka/admin.rb

@@ -2,6 +2,8 @@
 
 module Rdkafka
   class Admin
+    include Helpers::OAuth
+
     # @private
     def initialize(native_kafka)
       @native_kafka = native_kafka

data/lib/rdkafka/bindings.rb

@@ -26,6 +26,7 @@ module Rdkafka
 
     RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS = -175
     RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS = -174
+    RD_KAFKA_RESP_ERR__STATE = -172
     RD_KAFKA_RESP_ERR__NOENT = -156
     RD_KAFKA_RESP_ERR_NO_ERROR = 0
 
@@ -167,7 +168,10 @@ module Rdkafka
     callback :error_cb, [:pointer, :int, :string, :pointer], :void
     attach_function :rd_kafka_conf_set_error_cb, [:pointer, :error_cb], :void
     attach_function :rd_kafka_rebalance_protocol, [:pointer], :string
-
+    callback :oauthbearer_token_refresh_cb, [:pointer, :string, :pointer], :void
+    attach_function :rd_kafka_conf_set_oauthbearer_token_refresh_cb, [:pointer, :oauthbearer_token_refresh_cb], :void
+    attach_function :rd_kafka_oauthbearer_set_token, [:pointer, :string, :int64, :pointer, :pointer, :int, :pointer, :int], :int
+    attach_function :rd_kafka_oauthbearer_set_token_failure, [:pointer, :string], :int
     # Log queue
     attach_function :rd_kafka_set_log_queue, [:pointer, :pointer], :void
     attach_function :rd_kafka_queue_get_main, [:pointer], :pointer
@@ -217,6 +221,32 @@ module Rdkafka
       end
     end
 
+    # The OAuth callback is currently global and contextless.
+    # This means that the callback will be called for all instances, and the callback must be able to determine to which instance it is associated.
+    # The instance name will be provided in the callback, allowing the callback to reference the correct instance.
+    #
+    # An example of how to use the instance name in the callback is given below.
+    # The `refresh_token` is configured as the `oauthbearer_token_refresh_callback`.
+    # `instances` is a map of client names to client instances, maintained by the user.
+    #
+    # ```
+    #   def refresh_token(config, client_name)
+    #     client = instances[client_name]
+    #     client.oauthbearer_set_token(
+    #       token: 'new-token-value',
+    #       lifetime_ms: token-lifetime-ms,
+    #       principal_name: 'principal-name'
+    #     )
+    #   end
+    # ```
+    OAuthbearerTokenRefreshCallback = FFI::Function.new(
+      :void, [:pointer, :string, :pointer]
+    ) do |client_ptr, config, _opaque|
+      if Rdkafka::Config.oauthbearer_token_refresh_callback
+        Rdkafka::Config.oauthbearer_token_refresh_callback.call(config, Rdkafka::Bindings.rd_kafka_name(client_ptr))
+      end
+    end
+
     # Handle
 
     enum :kafka_type, [

data/lib/rdkafka/config.rb

@@ -15,12 +15,13 @@ module Rdkafka
     @@opaques = ObjectSpace::WeakMap.new
     # @private
     @@log_queue = Queue.new
-    # @private
     # We memoize thread on the first log flush
     # This allows us also to restart logger thread on forks
     @@log_thread = nil
     # @private
     @@log_mutex = Mutex.new
+    # @private
+    @@oauthbearer_token_refresh_callback = nil
 
     # Returns the current logger, by default this is a logger to stdout.
     #
@@ -104,6 +105,24 @@ module Rdkafka
       @@error_callback
     end
 
+    # Sets the SASL/OAUTHBEARER token refresh callback.
+    # This callback will be triggered when it is time to refresh the client's OAUTHBEARER token
+    #
+    # @param callback [Proc, #call] The callback
+    #
+    # @return [nil]
+    def self.oauthbearer_token_refresh_callback=(callback)
+      raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call) || callback == nil
+      @@oauthbearer_token_refresh_callback = callback
+    end
+
+    # Returns the current oauthbearer_token_refresh_callback callback, by default this is nil.
+    #
+    # @return [Proc, nil]
+    def self.oauthbearer_token_refresh_callback
+      @@oauthbearer_token_refresh_callback
+    end
+
     # @private
     def self.opaques
       @@opaques
@@ -300,6 +319,9 @@ module Rdkafka
 
         # Set error callback
         Rdkafka::Bindings.rd_kafka_conf_set_error_cb(config, Rdkafka::Bindings::ErrorCallback)
+
+        # Set oauth callback
+        Rdkafka::Bindings.rd_kafka_conf_set_oauthbearer_token_refresh_cb(config, Rdkafka::Bindings::OAuthbearerTokenRefreshCallback)
       end
     end
 

data/lib/rdkafka/consumer.rb

@@ -13,6 +13,7 @@ module Rdkafka
   class Consumer
     include Enumerable
     include Helpers::Time
+    include Helpers::OAuth
 
     # @private
     def initialize(native_kafka)

data/lib/rdkafka/helpers/oauth.rb

@@ -0,0 +1,58 @@
+module Rdkafka
+  module Helpers
+
+    module OAuth
+
+      # Set the OAuthBearer token
+      #
+      # @param token [String] the mandatory token value to set, often (but not necessarily) a JWS compact serialization as per https://tools.ietf.org/html/rfc7515#section-3.1.
+      # @param lifetime_ms [Integer] when the token expires, in terms of the number of milliseconds since the epoch. See https://currentmillis.com/.
+      # @param principal_name [String] the mandatory Kafka principal name associated with the token.
+      # @param extensions [Hash] optional SASL extensions key-value pairs to be communicated to the broker as additional key-value pairs during the initial client response as per https://tools.ietf.org/html/rfc7628#section-3.1.
+      # @return [Integer] 0 on success
+      def oauthbearer_set_token(token:, lifetime_ms:, principal_name:, extensions: nil)
+        error_buffer = FFI::MemoryPointer.from_string(" " * 256)
+
+        response = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_oauthbearer_set_token(
+            inner, token, lifetime_ms, principal_name,
+            flatten_extensions(extensions), extension_size(extensions), error_buffer, 256
+          )
+        end
+
+        return response if response.zero?
+
+        oauthbearer_set_token_failure("Failed to set token: #{error_buffer.read_string}")
+
+        response
+      end
+
+      # Marks failed oauth token acquire in librdkafka
+      #
+      # @param reason [String] human readable error reason for failing to acquire token
+      def oauthbearer_set_token_failure(reason)
+        @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_oauthbearer_set_token_failure(
+            inner,
+            reason
+          )
+        end
+      end
+
+      private
+
+      # Flatten the extensions hash into a string according to the spec, https://datatracker.ietf.org/doc/html/rfc7628#section-3.1
+      def flatten_extensions(extensions)
+        return nil unless extensions
+        "\x01#{extensions.map { |e| e.join("=") }.join("\x01")}"
+      end
+
+      # extension_size is the number of keys + values which should be a non-negative even number
+      # https://github.com/confluentinc/librdkafka/blob/master/src/rdkafka_sasl_oauthbearer.c#L327-L347
+      def extension_size(extensions)
+        return 0 unless extensions
+        extensions.size * 2
+      end
+    end
+  end
+end

data/lib/rdkafka/producer.rb

@@ -4,6 +4,7 @@ module Rdkafka
   # A producer for Kafka messages. To create a producer set up a {Config} and call {Config#producer producer} on that.
   class Producer
     include Helpers::Time
+    include Helpers::OAuth
 
     # Cache partitions count for 30 seconds
     PARTITIONS_COUNT_TTL = 30

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.15.0.alpha1"
+  VERSION = "0.15.0.beta1"
   LIBRDKAFKA_VERSION = "2.3.0"
   LIBRDKAFKA_SOURCE_SHA256 = "2d49c35c77eeb3d42fa61c43757fcbb6a206daa560247154e60642bcdcc14d12"
 end

data/lib/rdkafka.rb

@@ -7,6 +7,7 @@ require "json"
 
 require "rdkafka/version"
 require "rdkafka/helpers/time"
+require "rdkafka/helpers/oauth"
 require "rdkafka/abstract_handle"
 require "rdkafka/admin"
 require "rdkafka/admin/create_topic_handle"

data/spec/rdkafka/admin_spec.rb

@@ -676,4 +676,41 @@ describe Rdkafka::Admin do
       end
     end
   end
+
+  describe '#oauthbearer_set_token' do
+    context 'when sasl not configured' do
+      it 'should return RD_KAFKA_RESP_ERR__STATE' do
+        response = admin.oauthbearer_set_token(
+          token: "foo",
+          lifetime_ms: Time.now.to_i*1000 + 900 * 1000,
+          principal_name: "kafka-cluster"
+        )
+        expect(response).to eq(Rdkafka::Bindings::RD_KAFKA_RESP_ERR__STATE)
+      end
+    end
+
+    context 'when sasl configured' do
+      before do
+        config_sasl = rdkafka_config(
+          "security.protocol": "sasl_ssl",
+          "sasl.mechanisms": 'OAUTHBEARER'
+        )
+        $admin_sasl = config_sasl.admin
+      end
+
+      after do
+        $admin_sasl.close
+      end
+
+      it 'should succeed' do
+
+        response = $admin_sasl.oauthbearer_set_token(
+          token: "foo",
+          lifetime_ms: Time.now.to_i*1000 + 900 * 1000,
+          principal_name: "kafka-cluster"
+        )
+        expect(response).to eq(0)
+      end
+    end
+  end
 end

data/spec/rdkafka/bindings_spec.rb

@@ -132,4 +132,86 @@ describe Rdkafka::Bindings do
       end
     end
   end
+
+  describe "oauthbearer set token" do
+
+    context "without args" do
+      it "should raise argument error" do
+        expect {
+          Rdkafka::Bindings.rd_kafka_oauthbearer_set_token
+        }.to raise_error(ArgumentError)
+      end
+    end
+
+    context "with args" do
+      before do
+        DEFAULT_TOKEN_EXPIRY_SECONDS = 900
+        $token_value = "token"
+        $md_lifetime_ms = Time.now.to_i*1000 + DEFAULT_TOKEN_EXPIRY_SECONDS * 1000
+        $md_principal_name = "kafka-cluster"
+        $extensions = nil
+        $extension_size = 0
+        $error_buffer = FFI::MemoryPointer.from_string(" " * 256)
+      end
+
+      it "should set token or capture failure" do
+        RdKafkaTestConsumer.with do |consumer_ptr|
+          response = Rdkafka::Bindings.rd_kafka_oauthbearer_set_token(consumer_ptr, $token_value, $md_lifetime_ms, $md_principal_name, $extensions, $extension_size, $error_buffer, 256)
+          expect(response).to eq(Rdkafka::Bindings::RD_KAFKA_RESP_ERR__STATE)
+          expect($error_buffer.read_string).to eq("SASL/OAUTHBEARER is not the configured authentication mechanism")
+        end
+      end
+    end
+  end
+
+  describe "oauthbearer set token failure" do
+
+    context "without args" do
+
+      it "should fail" do
+        expect {
+          Rdkafka::Bindings.rd_kafka_oauthbearer_set_token_failure
+        }.to raise_error(ArgumentError)
+      end
+    end
+
+    context "with args" do
+      it "should succeed" do
+        expect {
+          errstr = "error"
+          RdKafkaTestConsumer.with do |consumer_ptr|
+            Rdkafka::Bindings.rd_kafka_oauthbearer_set_token_failure(consumer_ptr, errstr)
+          end
+        }.to_not raise_error
+      end
+    end
+  end
+
+  describe "oauthbearer callback" do
+
+    context "without an oauthbearer callback" do
+      it "should do nothing" do
+        expect {
+          Rdkafka::Bindings::OAuthbearerTokenRefreshCallback.call(nil, "", nil)
+        }.not_to raise_error
+      end
+    end
+
+    context "with an oauthbearer callback" do
+      before do
+        Rdkafka::Config.oauthbearer_token_refresh_callback = lambda do |config, client_name|
+          $received_config = config
+          $received_client_name = client_name
+        end
+      end
+
+      it "should call the oauth bearer callback and receive config and client name" do
+        RdKafkaTestConsumer.with do |consumer_ptr|
+          Rdkafka::Bindings::OAuthbearerTokenRefreshCallback.call(consumer_ptr, "{}", nil)
+            expect($received_config).to eq("{}")
+            expect($received_client_name).to match(/consumer/)
+        end
+      end
+    end
+  end
 end

data/spec/rdkafka/config_spec.rb

@@ -115,6 +115,39 @@ describe Rdkafka::Config do
     end
   end
 
+  context "oauthbearer calllback" do
+    context "with a proc/lambda" do
+      it "should set the callback" do
+        expect {
+          Rdkafka::Config.oauthbearer_token_refresh_callback = lambda do |config, client_name|
+            puts config
+            puts client_name
+          end
+        }.not_to raise_error
+        expect(Rdkafka::Config.oauthbearer_token_refresh_callback).to respond_to :call
+      end
+    end
+
+    context "with a callable object" do
+      it "should set the callback" do
+        callback = Class.new do
+          def call(config, client_name); end
+        end
+
+        expect {
+          Rdkafka::Config.oauthbearer_token_refresh_callback = callback.new
+        }.not_to raise_error
+        expect(Rdkafka::Config.oauthbearer_token_refresh_callback).to respond_to :call
+      end
+    end
+
+    it "should not accept a callback that's not callable" do
+      expect {
+        Rdkafka::Config.oauthbearer_token_refresh_callback = 'not a callback'
+      }.to raise_error(TypeError)
+    end
+  end
+
   context "configuration" do
     it "should store configuration" do
       config = Rdkafka::Config.new

data/spec/rdkafka/consumer_spec.rb

@@ -1329,4 +1329,40 @@ describe Rdkafka::Consumer do
       ])
     end
   end
+
+  describe '#oauthbearer_set_token' do
+    context 'when sasl not configured' do
+      it 'should return RD_KAFKA_RESP_ERR__STATE' do
+        response = consumer.oauthbearer_set_token(
+          token: "foo",
+          lifetime_ms: Time.now.to_i*1000 + 900 * 1000,
+          principal_name: "kafka-cluster"
+        )
+        expect(response).to eq(Rdkafka::Bindings::RD_KAFKA_RESP_ERR__STATE)
+      end
+    end
+
+    context 'when sasl configured' do
+      before do
+        $consumer_sasl = rdkafka_producer_config(
+          "security.protocol": "sasl_ssl",
+          "sasl.mechanisms": 'OAUTHBEARER'
+        ).consumer
+      end
+
+      after do
+        $consumer_sasl.close
+      end
+
+      it 'should succeed' do
+
+        response = $consumer_sasl.oauthbearer_set_token(
+          token: "foo",
+          lifetime_ms: Time.now.to_i*1000 + 900 * 1000,
+          principal_name: "kafka-cluster"
+        )
+        expect(response).to eq(0)
+      end
+    end
+  end
 end

data/spec/rdkafka/producer_spec.rb

@@ -917,4 +917,34 @@ describe Rdkafka::Producer do
       end
     end
   end
+
+  describe '#oauthbearer_set_token' do
+    context 'when sasl not configured' do
+      it 'should return RD_KAFKA_RESP_ERR__STATE' do
+        response = producer.oauthbearer_set_token(
+              token: "foo",
+              lifetime_ms: Time.now.to_i*1000 + 900 * 1000,
+              principal_name: "kafka-cluster"
+            )
+        expect(response).to eq(Rdkafka::Bindings::RD_KAFKA_RESP_ERR__STATE)
+      end
+    end
+
+    context 'when sasl configured' do
+      it 'should succeed' do
+        producer_sasl = rdkafka_producer_config(
+          {
+            "security.protocol": "sasl_ssl",
+            "sasl.mechanisms": 'OAUTHBEARER'
+          }
+        ).producer
+        response = producer_sasl.oauthbearer_set_token(
+          token: "foo",
+          lifetime_ms: Time.now.to_i*1000 + 900 * 1000,
+          principal_name: "kafka-cluster"
+        )
+        expect(response).to eq(0)
+      end
+    end
+  end
 end

data/spec/spec_helper.rb

@@ -155,3 +155,18 @@ RSpec.configure do |config|
     end
   end
 end
+
+class RdKafkaTestConsumer
+  def self.with
+    consumer = Rdkafka::Bindings.rd_kafka_new(
+      :rd_kafka_consumer,
+      nil,
+      nil,
+      0
+    )
+    yield consumer
+  ensure
+    Rdkafka::Bindings.rd_kafka_consumer_close(consumer)
+    Rdkafka::Bindings.rd_kafka_destroy(consumer)
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.15.0.alpha1
+  version: 0.15.0.beta1
 platform: ruby
 authors:
 - Thijs Cadier
@@ -36,7 +36,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-03-17 00:00:00.000000000 Z
+date: 2024-03-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -223,6 +223,7 @@ files:
 - lib/rdkafka/consumer/partition.rb
 - lib/rdkafka/consumer/topic_partition_list.rb
 - lib/rdkafka/error.rb
+- lib/rdkafka/helpers/oauth.rb
 - lib/rdkafka/helpers/time.rb
 - lib/rdkafka/metadata.rb
 - lib/rdkafka/native_kafka.rb