karafka-rdkafka 0.15.2 → 0.16.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a9cfc42920e216878543e0ceb81d3f13930f43ecd14c954855d7f8d57824e4db
-  data.tar.gz: 912bcbe8ace5d02f8add80de5fb026cd0bc6f37845339b373094d943610a3191
+  metadata.gz: f42274ae25769bb5ac70ea53d4137270dac420dc9a8074175e5090304c6690df
+  data.tar.gz: dcc6c9d4cc815b5a742a95640b7131e5820042621006dd4bb776da578b0d9b22
 SHA512:
-  metadata.gz: 89ac5f595837ff33ce10eb4e3c50fbf21216069bb65498341fb2d1faf2646a7996be9eaf9a2d60c42d3e376190fbb06fb131134a1802258f154d6877cc854037
-  data.tar.gz: e2152f5eddf2bd88b7ee2dc144dd91b22a5fc94e11615cac3d8ab869609a90576627363d8dc1ce58ebc52b55276bc1043b375d5faa4ef4c977b5ecb71b30f987
+  metadata.gz: 72ba8300c81ef41c947788a4329c531f64bca9a255bf147da2477c23a6481d485e6f1a6eefdd3c348c966c4980f6c56bc9b25c2dddf7bed3783816548ce761b3
+  data.tar.gz: 3a7282405f3b58ac8bd3ddbd197256dbfa15e64384cef03370ba6cb3fe87967a2978ea2e9b6532d532f0a38e55ca823d94da14d46b0603238e1a37027411949f

data/.github/workflows/ci.yml

@@ -22,11 +22,11 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
+          - '3.4.0-preview1'
           - '3.3'
           - '3.2'
           - '3.1'
           - '3.0'
-          - '2.7'
         include:
           - ruby: '3.3'
             coverage: 'true'

data/.ruby-version

@@ -1 +1 @@
-3.3.1
+3.3.3

data/CHANGELOG.md

@@ -1,7 +1,11 @@
 # Rdkafka Changelog
 
-## 0.15.2 (2024-07-10)
-- [Fix] Switch to local release of librdkafka to mitigate its unavailability.
+## 0.16.0 (Unreleased)
+- **[Breaking]** Messages without headers returned by `#poll` contain frozen empty hash.
+- **[Breaking]** `HashWithSymbolKeysTreatedLikeStrings` has been removed so headers are regular hashes with string keys.
+- [Enhancement] Bump librdkafka to 2.4.0
+- [Enhancement] Save two objects on message produced and lower CPU usage on message produced with small improvements.
+- [Fix] Remove support for Ruby 2.7. Supporting it was a bug since rest of the karafka ecosystem no longer supports it.
 
 ## 0.15.1 (2024-05-09)
 - **[Feature]** Provide ability to use topic config on a producer for custom behaviors per dispatch.

data/README.md

@@ -163,6 +163,7 @@ bundle exec rake produce_messages
 
 | rdkafka-ruby | librdkafka |
 |-|-|
+| 0.16.0 (2024-06-13) | 2.3.0 (2023-10-25) |
 | 0.15.0 (2023-12-03) | 2.3.0 (2023-10-25) |
 | 0.14.0 (2023-11-21) | 2.2.0 (2023-07-12) |
 | 0.13.0 (2023-07-24) | 2.0.2 (2023-01-20) |

data/ext/Rakefile

@@ -22,10 +22,8 @@ task :default => :clean do
       ENV["LDFLAGS"] = "-L#{homebrew_prefix}/lib" unless ENV["LDFLAGS"]
     end
 
-    releases = File.expand_path(File.join(File.dirname(__FILE__), '../dist'))
-
     recipe.files << {
-      :url => "file://#{releases}/librdkafka_#{Rdkafka::LIBRDKAFKA_VERSION}.tar.gz",
+      :url => "https://codeload.github.com/edenhill/librdkafka/tar.gz/v#{Rdkafka::LIBRDKAFKA_VERSION}",
       :sha256 => Rdkafka::LIBRDKAFKA_SOURCE_SHA256
     }
     recipe.configure_options = ["--host=#{recipe.host}"]

data/karafka-rdkafka.gemspec

@@ -15,7 +15,7 @@ Gem::Specification.new do |gem|
   gem.name = 'karafka-rdkafka'
   gem.require_paths = ['lib']
   gem.version = Rdkafka::VERSION
-  gem.required_ruby_version = '>= 2.7'
+  gem.required_ruby_version = '>= 3.0'
   gem.extensions = %w(ext/Rakefile)
   gem.cert_chain = %w[certs/cert_chain.pem]
 

data/lib/rdkafka/consumer/headers.rb

@@ -4,17 +4,7 @@ module Rdkafka
   class Consumer
     # Interface to return headers for a consumer message
     module Headers
-      class HashWithSymbolKeysTreatedLikeStrings < Hash
-        def [](key)
-          if key.is_a?(Symbol)
-            Kernel.warn("rdkafka deprecation warning: header access with Symbol key #{key.inspect} treated as a String. " \
-                        "Please change your code to use String keys to avoid this warning. Symbol keys will break in version 1.")
-            super(key.to_s)
-          else
-            super
-          end
-        end
-      end
+      EMPTY_HEADERS = {}.freeze
 
       # Reads a librdkafka native message's headers and returns them as a Ruby Hash
       #
@@ -28,7 +18,7 @@ module Rdkafka
         err = Rdkafka::Bindings.rd_kafka_message_headers(native_message, headers_ptrptr)
 
         if err == Rdkafka::Bindings::RD_KAFKA_RESP_ERR__NOENT
-          return {}
+          return EMPTY_HEADERS
         elsif err != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
           raise Rdkafka::RdkafkaError.new(err, "Error reading message headers")
         end
@@ -39,7 +29,7 @@ module Rdkafka
         value_ptrptr = FFI::MemoryPointer.new(:pointer)
         size_ptr = Rdkafka::Bindings::SizePtr.new
 
-        headers = HashWithSymbolKeysTreatedLikeStrings.new
+        headers = {}
 
         idx = 0
         loop do

data/lib/rdkafka/producer/delivery_handle.rb

@@ -14,6 +14,10 @@ module Rdkafka
       # @return [Object, nil] label set during message production or nil by default
       attr_accessor :label
 
+      # @return [String] topic where we are trying to send the message
+      # We use this instead of reading from `topic_name` pointer to save on memory allocations
+      attr_accessor :topic
+
       # @return [String] the name of the operation (e.g. "delivery")
       def operation_name
         "delivery"
@@ -25,7 +29,7 @@ module Rdkafka
           DeliveryReport.new(
             self[:partition],
             self[:offset],
-            self[:topic_name].read_string,
+            topic,
             nil,
             label
           )
@@ -33,9 +37,7 @@ module Rdkafka
           DeliveryReport.new(
             self[:partition],
             self[:offset],
-            # For part of errors, we will not get a topic name reference and in cases like this
-            # we should not return it
-            self[:topic_name].null? ? nil : self[:topic_name].read_string,
+            topic,
             Rdkafka::RdkafkaError.build(self[:response]),
             label
           )

data/lib/rdkafka/producer.rb

@@ -381,6 +381,7 @@ module Rdkafka
 
       delivery_handle = DeliveryHandle.new
       delivery_handle.label = label
+      delivery_handle.topic = topic
       delivery_handle[:pending] = true
       delivery_handle[:response] = -1
       delivery_handle[:partition] = -1
@@ -404,7 +405,7 @@ module Rdkafka
           args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_HEADER
           args << :string << key
           args << :pointer << value
-          args << :size_t << value.bytes.size
+          args << :size_t << value.bytesize
         end
       end
 

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.15.2"
-  LIBRDKAFKA_VERSION = "2.3.0"
-  LIBRDKAFKA_SOURCE_SHA256 = "2d49c35c77eeb3d42fa61c43757fcbb6a206daa560247154e60642bcdcc14d12"
+  VERSION = "0.16.0.rc1"
+  LIBRDKAFKA_VERSION = "2.4.0"
+  LIBRDKAFKA_SOURCE_SHA256 = "d645e47d961db47f1ead29652606a502bdd2a880c85c1e060e94eea040f1a19a"
 end

data/spec/rdkafka/admin_spec.rb

@@ -34,7 +34,7 @@ describe Rdkafka::Admin do
   describe '#describe_errors' do
     let(:errors) { admin.class.describe_errors }
 
-    it { expect(errors.size).to eq(162) }
+    it { expect(errors.size).to eq(168) }
     it { expect(errors[-184]).to eq(code: -184, description: 'Local: Queue full', name: '_QUEUE_FULL') }
     it { expect(errors[21]).to eq(code: 21, description: 'Broker: Invalid required acks value', name: 'INVALID_REQUIRED_ACKS') }
   end

data/spec/rdkafka/consumer/headers_spec.rb

@@ -50,11 +50,8 @@ describe Rdkafka::Consumer::Headers do
       expect(subject['version']).to eq("2.1.3")
     end
 
-    it 'allows Symbol key, but warns' do
-      expect(Kernel).to \
-        receive(:warn).with("rdkafka deprecation warning: header access with Symbol key :version treated as a String. " \
-                            "Please change your code to use String keys to avoid this warning. Symbol keys will break in version 1.")
-      expect(subject[:version]).to eq("2.1.3")
+    it 'does not support symbols mappings' do
+      expect(subject.key?(:version)).to eq(false)
     end
   end
 end

data/spec/rdkafka/producer/delivery_handle_spec.rb

@@ -9,7 +9,7 @@ describe Rdkafka::Producer::DeliveryHandle do
       handle[:response] = response
       handle[:partition] = 2
       handle[:offset] = 100
-      handle[:topic_name] = FFI::MemoryPointer.from_string("produce_test_topic")
+      handle.topic = "produce_test_topic"
     end
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.15.2
+  version: 0.16.0.rc1
 platform: ruby
 authors:
 - Thijs Cadier
@@ -36,7 +36,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-07-10 00:00:00.000000000 Z
+date: 2024-06-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -186,7 +186,6 @@ files:
 - README.md
 - Rakefile
 - certs/cert_chain.pem
-- dist/librdkafka_2.3.0.tar.gz
 - docker-compose.yml
 - ext/README.md
 - ext/Rakefile
@@ -279,14 +278,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.7'
+      version: '3.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.9
+rubygems_version: 3.5.11
 signing_key: 
 specification_version: 4
 summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.