rdkafka 0.24.2-aarch64-linux-gnu → 0.25.1-aarch64-linux-gnu

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +18 -0
  3. data/Gemfile +8 -0
  4. data/Gemfile.lint +14 -0
  5. data/Gemfile.lint.lock +123 -0
  6. data/README.md +2 -1
  7. data/Rakefile +21 -21
  8. data/docker-compose-ssl.yml +1 -1
  9. data/docker-compose.yml +1 -1
  10. data/ext/librdkafka.so +0 -0
  11. data/lib/rdkafka/abstract_handle.rb +23 -5
  12. data/lib/rdkafka/admin/acl_binding_result.rb +5 -5
  13. data/lib/rdkafka/admin/config_resource_binding_result.rb +1 -0
  14. data/lib/rdkafka/admin/create_acl_handle.rb +7 -4
  15. data/lib/rdkafka/admin/create_acl_report.rb +3 -2
  16. data/lib/rdkafka/admin/create_partitions_handle.rb +8 -5
  17. data/lib/rdkafka/admin/create_partitions_report.rb +1 -0
  18. data/lib/rdkafka/admin/create_topic_handle.rb +8 -5
  19. data/lib/rdkafka/admin/create_topic_report.rb +3 -0
  20. data/lib/rdkafka/admin/delete_acl_handle.rb +9 -6
  21. data/lib/rdkafka/admin/delete_acl_report.rb +5 -3
  22. data/lib/rdkafka/admin/delete_groups_handle.rb +10 -5
  23. data/lib/rdkafka/admin/delete_groups_report.rb +3 -0
  24. data/lib/rdkafka/admin/delete_topic_handle.rb +8 -5
  25. data/lib/rdkafka/admin/delete_topic_report.rb +3 -0
  26. data/lib/rdkafka/admin/describe_acl_handle.rb +9 -6
  27. data/lib/rdkafka/admin/describe_acl_report.rb +5 -3
  28. data/lib/rdkafka/admin/describe_configs_handle.rb +7 -4
  29. data/lib/rdkafka/admin/describe_configs_report.rb +7 -1
  30. data/lib/rdkafka/admin/incremental_alter_configs_handle.rb +7 -4
  31. data/lib/rdkafka/admin/incremental_alter_configs_report.rb +7 -1
  32. data/lib/rdkafka/admin.rb +194 -132
  33. data/lib/rdkafka/bindings.rb +155 -107
  34. data/lib/rdkafka/callbacks.rb +81 -21
  35. data/lib/rdkafka/config.rb +36 -24
  36. data/lib/rdkafka/consumer/headers.rb +3 -2
  37. data/lib/rdkafka/consumer/message.rb +12 -11
  38. data/lib/rdkafka/consumer/partition.rb +8 -4
  39. data/lib/rdkafka/consumer/topic_partition_list.rb +18 -18
  40. data/lib/rdkafka/consumer.rb +247 -42
  41. data/lib/rdkafka/defaults.rb +106 -0
  42. data/lib/rdkafka/error.rb +28 -13
  43. data/lib/rdkafka/helpers/oauth.rb +11 -6
  44. data/lib/rdkafka/helpers/time.rb +5 -0
  45. data/lib/rdkafka/metadata.rb +45 -21
  46. data/lib/rdkafka/native_kafka.rb +89 -4
  47. data/lib/rdkafka/producer/delivery_handle.rb +5 -5
  48. data/lib/rdkafka/producer/delivery_report.rb +8 -4
  49. data/lib/rdkafka/producer/partitions_count_cache.rb +29 -19
  50. data/lib/rdkafka/producer.rb +165 -79
  51. data/lib/rdkafka/version.rb +6 -3
  52. data/lib/rdkafka.rb +1 -0
  53. data/package-lock.json +331 -0
  54. data/package.json +9 -0
  55. data/rdkafka.gemspec +39 -47
  56. data/renovate.json +22 -8
  57. metadata +7 -86
@@ -0,0 +1,106 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ # Default timeout and timing values used throughout rdkafka-ruby.
5
+ #
6
+ # All timeout values can be overridden per-call via method parameters.
7
+ # These constants provide a central place to understand and reference
8
+ # the default values used across the library.
9
+ #
10
+ # @note These are rdkafka-ruby defaults, not librdkafka configuration options.
11
+ # For librdkafka options, see:
12
+ # https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md
13
+ #
14
+ # @example Overriding a timeout per-call
15
+ # consumer.committed(timeout_ms: 5_000) # Use 5 seconds instead of default 2 seconds
16
+ #
17
+ # @example Checking the default value
18
+ # Rdkafka::Defaults::CONSUMER_COMMITTED_TIMEOUT_MS # => 2000
19
+ module Defaults
20
+ # Consumer timeouts (in milliseconds)
21
+
22
+ # Default timeout for fetching committed offsets
23
+ # @see Consumer#committed
24
+ CONSUMER_COMMITTED_TIMEOUT_MS = 2_000
25
+
26
+ # Default timeout for querying watermark offsets
27
+ # @see Consumer#query_watermark_offsets
28
+ CONSUMER_QUERY_WATERMARK_TIMEOUT_MS = 1_000
29
+
30
+ # Default timeout for lag calculation watermark queries
31
+ # @see Consumer#lag
32
+ CONSUMER_LAG_TIMEOUT_MS = 1_000
33
+
34
+ # Default timeout for offsets_for_times operation
35
+ # @see Consumer#offsets_for_times
36
+ CONSUMER_OFFSETS_FOR_TIMES_TIMEOUT_MS = 1_000
37
+
38
+ # Default poll timeout for Consumer#each iterator
39
+ # @see Consumer#each
40
+ CONSUMER_POLL_TIMEOUT_MS = 250
41
+
42
+ # Seek operation timeout (0 = non-blocking)
43
+ # @see Consumer#seek_by
44
+ CONSUMER_SEEK_TIMEOUT_MS = 0
45
+
46
+ # Events poll timeout (0 = non-blocking/async)
47
+ # @see Consumer#events_poll
48
+ CONSUMER_EVENTS_POLL_TIMEOUT_MS = 0
49
+
50
+ # Producer timeouts (in milliseconds)
51
+
52
+ # Default timeout for producer flush operation
53
+ # @see Producer#flush
54
+ PRODUCER_FLUSH_TIMEOUT_MS = 5_000
55
+
56
+ # Default flush timeout during purge operation
57
+ # @see Producer#purge
58
+ PRODUCER_PURGE_FLUSH_TIMEOUT_MS = 100
59
+
60
+ # Metadata timeouts (in milliseconds)
61
+
62
+ # Default timeout for metadata requests
63
+ # @see Admin#metadata
64
+ # @see Metadata#initialize
65
+ METADATA_TIMEOUT_MS = 2_000
66
+
67
+ # Handle wait timeouts (in milliseconds)
68
+
69
+ # Default maximum wait timeout for async handles (delivery, admin operations)
70
+ # @see AbstractHandle#wait
71
+ HANDLE_WAIT_TIMEOUT_MS = 60_000
72
+
73
+ # Native Kafka polling (in milliseconds)
74
+
75
+ # Default poll timeout for producer/admin native polling thread
76
+ # @see Config#producer
77
+ # @see Config#admin
78
+ NATIVE_KAFKA_POLL_TIMEOUT_MS = 100
79
+
80
+ # Internal timing (in milliseconds)
81
+
82
+ # Sleep interval during purge wait loop
83
+ # @see Producer#purge
84
+ PRODUCER_PURGE_SLEEP_INTERVAL_MS = 1
85
+
86
+ # Sleep interval while waiting for operations to complete in NativeKafka#synchronize
87
+ # @see NativeKafka#synchronize
88
+ NATIVE_KAFKA_SYNCHRONIZE_SLEEP_INTERVAL_MS = 10
89
+
90
+ # Base backoff factor for metadata retry in milliseconds (multiplied by 2^attempt)
91
+ # @see Metadata#initialize
92
+ METADATA_RETRY_BACKOFF_BASE_MS = 100
93
+
94
+ # Cache settings (in milliseconds)
95
+
96
+ # Default time-to-live for cached partition counts
97
+ # @see Producer::PartitionsCountCache
98
+ PARTITIONS_COUNT_CACHE_TTL_MS = 30_000
99
+
100
+ # Configuration values (not time-based)
101
+
102
+ # Maximum number of metadata fetch retry attempts
103
+ # @see Metadata#initialize
104
+ METADATA_MAX_RETRIES = 10
105
+ end
106
+ end
data/lib/rdkafka/error.rb CHANGED
@@ -19,7 +19,10 @@ module Rdkafka
19
19
  attr_reader :broker_message
20
20
 
21
21
  # @private
22
- def initialize(response, message_prefix=nil, broker_message: nil)
22
+ # @param response [Integer] the raw error response code from librdkafka
23
+ # @param message_prefix [String, nil] optional prefix for error messages
24
+ # @param broker_message [String, nil] optional error message from the broker
25
+ def initialize(response, message_prefix = nil, broker_message: nil)
23
26
  raise TypeError.new("Response has to be an integer") unless response.is_a? Integer
24
27
  @rdkafka_response = response
25
28
  @message_prefix = message_prefix
@@ -31,7 +34,7 @@ module Rdkafka
31
34
  def code
32
35
  code = Rdkafka::Bindings.rd_kafka_err2name(@rdkafka_response).downcase
33
36
  if code[0] == "_"
34
- code[1..-1].to_sym
37
+ code[1..].to_sym
35
38
  else
36
39
  code.to_sym
37
40
  end
@@ -41,10 +44,10 @@ module Rdkafka
41
44
  # @return [String]
42
45
  def to_s
43
46
  message_prefix_part = if message_prefix
44
- "#{message_prefix} - "
45
- else
46
- ''
47
- end
47
+ "#{message_prefix} - "
48
+ else
49
+ ""
50
+ end
48
51
  "#{message_prefix_part}#{Rdkafka::Bindings.rd_kafka_err2str(@rdkafka_response)} (#{code})"
49
52
  end
50
53
 
@@ -55,8 +58,10 @@ module Rdkafka
55
58
  end
56
59
 
57
60
  # Error comparison
58
- def ==(another_error)
59
- another_error.is_a?(self.class) && (self.to_s == another_error.to_s)
61
+ # @param other [Object] object to compare with
62
+ # @return [Boolean]
63
+ def ==(other)
64
+ other.is_a?(self.class) && (to_s == other.to_s)
60
65
  end
61
66
  end
62
67
 
@@ -66,7 +71,10 @@ module Rdkafka
66
71
  attr_reader :topic_partition_list
67
72
 
68
73
  # @private
69
- def initialize(response, topic_partition_list, message_prefix=nil)
74
+ # @param response [Integer] the raw error response code from librdkafka
75
+ # @param topic_partition_list [TopicPartitionList] the topic partition list with error info
76
+ # @param message_prefix [String, nil] optional prefix for error messages
77
+ def initialize(response, topic_partition_list, message_prefix = nil)
70
78
  super(response, message_prefix)
71
79
  @topic_partition_list = topic_partition_list
72
80
  end
@@ -74,28 +82,35 @@ module Rdkafka
74
82
 
75
83
  # Error class for public consumer method calls on a closed consumer.
76
84
  class ClosedConsumerError < BaseError
85
+ # @param method [Symbol] the method that was called
77
86
  def initialize(method)
78
- super("Illegal call to #{method.to_s} on a closed consumer")
87
+ super("Illegal call to #{method} on a closed consumer")
79
88
  end
80
89
  end
81
90
 
82
91
  # Error class for public producer method calls on a closed producer.
83
92
  class ClosedProducerError < BaseError
93
+ # @param method [Symbol] the method that was called
84
94
  def initialize(method)
85
- super("Illegal call to #{method.to_s} on a closed producer")
95
+ super("Illegal call to #{method} on a closed producer")
86
96
  end
87
97
  end
88
98
 
89
- # Error class for public consumer method calls on a closed admin.
99
+ # Error class for public admin method calls on a closed admin.
90
100
  class ClosedAdminError < BaseError
101
+ # @param method [Symbol] the method that was called
91
102
  def initialize(method)
92
- super("Illegal call to #{method.to_s} on a closed admin")
103
+ super("Illegal call to #{method} on a closed admin")
93
104
  end
94
105
  end
95
106
 
107
+ # Error class for calls on a closed inner librdkafka instance.
96
108
  class ClosedInnerError < BaseError
97
109
  def initialize
98
110
  super("Illegal call to a closed inner librdkafka instance")
99
111
  end
100
112
  end
113
+
114
+ # Error class for librdkafka library loading failures (e.g., glibc compatibility issues).
115
+ class LibraryLoadError < BaseError; end
101
116
  end
@@ -1,8 +1,7 @@
1
1
  module Rdkafka
2
2
  module Helpers
3
-
3
+ # OAuth helper methods for setting and refreshing SASL/OAUTHBEARER tokens
4
4
  module OAuth
5
-
6
5
  # Set the OAuthBearer token
7
6
  #
8
7
  # @param token [String] the mandatory token value to set, often (but not necessarily) a JWS compact serialization as per https://tools.ietf.org/html/rfc7515#section-3.1.
@@ -47,8 +46,11 @@ module Rdkafka
47
46
 
48
47
  private
49
48
 
50
- # Convert extensions hash to FFI::MemoryPointer (const char **).
51
- # Note: the returned pointers must be freed manually (autorelease = false).
49
+ # Convert extensions hash to FFI::MemoryPointer (`const char **`).
50
+ #
51
+ # @param extensions [Hash, nil] extension key-value pairs
52
+ # @return [Array<FFI::MemoryPointer, Array<FFI::MemoryPointer>>] array pointer and string pointers
53
+ # @note The returned pointers must be freed manually (autorelease = false).
52
54
  def map_extensions(extensions)
53
55
  return [nil, nil] if extensions.nil? || extensions.empty?
54
56
 
@@ -74,8 +76,11 @@ module Rdkafka
74
76
  [array_ptr, str_ptrs]
75
77
  end
76
78
 
77
- # extension_size is the number of keys + values which should be a non-negative even number
78
- # https://github.com/confluentinc/librdkafka/blob/master/src/rdkafka_sasl_oauthbearer.c#L327-L347
79
+ # Returns the extension size (number of keys + values).
80
+ #
81
+ # @param extensions [Hash, nil] extension key-value pairs
82
+ # @return [Integer] non-negative even number representing keys + values count
83
+ # @see https://github.com/confluentinc/librdkafka/blob/master/src/rdkafka_sasl_oauthbearer.c#L327-L347
79
84
  def extension_size(extensions)
80
85
  return 0 unless extensions
81
86
  extensions.size * 2
@@ -9,6 +9,11 @@ module Rdkafka
9
9
  def monotonic_now
10
10
  ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
11
11
  end
12
+
13
+ # @return [Integer] current monotonic time in milliseconds
14
+ def monotonic_now_ms
15
+ ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :millisecond)
16
+ end
12
17
  end
13
18
  end
14
19
  end
@@ -1,8 +1,12 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Rdkafka
4
+ # Provides cluster metadata information
4
5
  class Metadata
5
- attr_reader :brokers, :topics
6
+ # @return [Array<Hash>] list of broker metadata
7
+ attr_reader :brokers
8
+ # @return [Array<Hash>] list of topic metadata
9
+ attr_reader :topics
6
10
 
7
11
  # Errors upon which we retry the metadata fetch
8
12
  RETRIED_ERRORS = %i[
@@ -12,7 +16,13 @@ module Rdkafka
12
16
 
13
17
  private_constant :RETRIED_ERRORS
14
18
 
15
- def initialize(native_client, topic_name = nil, timeout_ms = 2_000)
19
+ # Fetches metadata from the Kafka cluster
20
+ #
21
+ # @param native_client [FFI::Pointer] pointer to the native Kafka client
22
+ # @param topic_name [String, nil] specific topic to fetch metadata for, or nil for all topics
23
+ # @param timeout_ms [Integer] timeout in milliseconds
24
+ # @raise [RdkafkaError] when metadata fetch fails
25
+ def initialize(native_client, topic_name = nil, timeout_ms = Defaults::METADATA_TIMEOUT_MS)
16
26
  attempt ||= 0
17
27
  attempt += 1
18
28
 
@@ -35,12 +45,12 @@ module Rdkafka
35
45
  metadata_from_native(ptr.read_pointer)
36
46
  rescue ::Rdkafka::RdkafkaError => e
37
47
  raise unless RETRIED_ERRORS.include?(e.code)
38
- raise if attempt > 10
48
+ raise if attempt > Defaults::METADATA_MAX_RETRIES
39
49
 
40
50
  backoff_factor = 2**attempt
41
- timeout = backoff_factor * 0.1
51
+ timeout_ms = backoff_factor * Defaults::METADATA_RETRY_BACKOFF_BASE_MS
42
52
 
43
- sleep(timeout)
53
+ sleep(timeout_ms / 1000.0)
44
54
 
45
55
  retry
46
56
  ensure
@@ -50,6 +60,8 @@ module Rdkafka
50
60
 
51
61
  private
52
62
 
63
+ # Extracts metadata from native pointer
64
+ # @param ptr [FFI::Pointer] pointer to native metadata
53
65
  def metadata_from_native(ptr)
54
66
  metadata = Metadata.new(ptr)
55
67
  @brokers = Array.new(metadata[:brokers_count]) do |i|
@@ -69,7 +81,11 @@ module Rdkafka
69
81
  end
70
82
  end
71
83
 
84
+ # Base class for metadata FFI structs with hash conversion
85
+ # @private
72
86
  class CustomFFIStruct < FFI::Struct
87
+ # Converts struct to a hash
88
+ # @return [Hash]
73
89
  def to_h
74
90
  members.each_with_object({}) do |mem, hsh|
75
91
  val = self.[](mem)
@@ -80,36 +96,44 @@ module Rdkafka
80
96
  end
81
97
  end
82
98
 
99
+ # @private
100
+ # FFI struct for rd_kafka_metadata_t
83
101
  class Metadata < CustomFFIStruct
84
102
  layout :brokers_count, :int,
85
- :brokers_metadata, :pointer,
86
- :topics_count, :int,
87
- :topics_metadata, :pointer,
88
- :broker_id, :int32,
89
- :broker_name, :string
103
+ :brokers_metadata, :pointer,
104
+ :topics_count, :int,
105
+ :topics_metadata, :pointer,
106
+ :broker_id, :int32,
107
+ :broker_name, :string
90
108
  end
91
109
 
110
+ # @private
111
+ # FFI struct for rd_kafka_metadata_broker_t
92
112
  class BrokerMetadata < CustomFFIStruct
93
113
  layout :broker_id, :int32,
94
- :broker_name, :string,
95
- :broker_port, :int
114
+ :broker_name, :string,
115
+ :broker_port, :int
96
116
  end
97
117
 
118
+ # @private
119
+ # FFI struct for rd_kafka_metadata_topic_t
98
120
  class TopicMetadata < CustomFFIStruct
99
121
  layout :topic_name, :string,
100
- :partition_count, :int,
101
- :partitions_metadata, :pointer,
102
- :rd_kafka_resp_err, :int
122
+ :partition_count, :int,
123
+ :partitions_metadata, :pointer,
124
+ :rd_kafka_resp_err, :int
103
125
  end
104
126
 
127
+ # @private
128
+ # FFI struct for rd_kafka_metadata_partition_t
105
129
  class PartitionMetadata < CustomFFIStruct
106
130
  layout :partition_id, :int32,
107
- :rd_kafka_resp_err, :int,
108
- :leader, :int32,
109
- :replica_count, :int,
110
- :replicas, :pointer,
111
- :in_sync_replica_brokers, :int,
112
- :isrs, :pointer
131
+ :rd_kafka_resp_err, :int,
132
+ :leader, :int32,
133
+ :replica_count, :int,
134
+ :replicas, :pointer,
135
+ :in_sync_replica_brokers, :int,
136
+ :isrs, :pointer
113
137
  end
114
138
  end
115
139
  end
@@ -4,7 +4,13 @@ module Rdkafka
4
4
  # @private
5
5
  # A wrapper around a native kafka that polls and cleanly exits
6
6
  class NativeKafka
7
- def initialize(inner, run_polling_thread:, opaque:, auto_start: true, timeout_ms: 100)
7
+ # Creates a new NativeKafka wrapper
8
+ # @param inner [FFI::Pointer] pointer to the native Kafka handle
9
+ # @param run_polling_thread [Boolean] whether to run a background polling thread
10
+ # @param opaque [Rdkafka::Opaque] opaque object for callback context
11
+ # @param auto_start [Boolean] whether to start the polling thread automatically
12
+ # @param timeout_ms [Integer] poll timeout in milliseconds
13
+ def initialize(inner, run_polling_thread:, opaque:, auto_start: true, timeout_ms: Defaults::NATIVE_KAFKA_POLL_TIMEOUT_MS)
8
14
  @inner = inner
9
15
  @opaque = opaque
10
16
  # Lock around external access
@@ -37,6 +43,8 @@ module Rdkafka
37
43
  @closing = false
38
44
  end
39
45
 
46
+ # Starts the polling thread if configured
47
+ # @return [nil]
40
48
  def start
41
49
  synchronize do
42
50
  return if @started
@@ -62,13 +70,17 @@ module Rdkafka
62
70
  end
63
71
  end
64
72
 
65
- @polling_thread.name = "rdkafka.native_kafka##{Rdkafka::Bindings.rd_kafka_name(@inner).gsub('rdkafka', '')}"
73
+ @polling_thread.name = "rdkafka.native_kafka##{Rdkafka::Bindings.rd_kafka_name(@inner).gsub("rdkafka", "")}"
66
74
  @polling_thread.abort_on_exception = true
67
75
  @polling_thread[:closing] = false
68
76
  end
69
77
  end
70
78
  end
71
79
 
80
+ # Executes a block with the inner native Kafka handle
81
+ # @yield [FFI::Pointer] the inner native Kafka handle
82
+ # @return [Object] the result of the block
83
+ # @raise [ClosedInnerError] when the inner handle is nil
72
84
  def with_inner
73
85
  if @access_mutex.owned?
74
86
  @operations_in_progress += 1
@@ -81,27 +93,100 @@ module Rdkafka
81
93
  @decrement_mutex.synchronize { @operations_in_progress -= 1 }
82
94
  end
83
95
 
96
+ # Executes a block while holding exclusive access to the native Kafka handle
97
+ # @param block [Proc] block to execute with the native handle
98
+ # @yield [FFI::Pointer] the inner native Kafka handle
99
+ # @return [Object] the result of the block
84
100
  def synchronize(&block)
85
101
  @access_mutex.synchronize do
86
102
  # Wait for any commands using the inner to finish
87
103
  # This can take a while on blocking operations like polling but is essential not to proceed
88
104
  # with certain types of operations like resources destruction as it can cause the process
89
105
  # to hang or crash
90
- sleep(0.01) until @operations_in_progress.zero?
106
+ sleep(Defaults::NATIVE_KAFKA_SYNCHRONIZE_SLEEP_INTERVAL_MS / 1000.0) until @operations_in_progress.zero?
91
107
 
92
108
  with_inner(&block)
93
109
  end
94
110
  end
95
111
 
112
+ # Returns a finalizer proc for closing this native Kafka handle
113
+ # @return [Proc] finalizer proc
96
114
  def finalizer
97
115
  ->(_) { close }
98
116
  end
99
117
 
118
+ # Returns whether this native Kafka handle is closed or closing
119
+ # @return [Boolean] true if closed or closing
100
120
  def closed?
101
121
  @closing || @inner.nil?
102
122
  end
103
123
 
104
- def close(object_id=nil)
124
+ # Enable IO event notifications on the main queue
125
+ # Librdkafka will write to your FD when the queue transitions from empty to non-empty
126
+ #
127
+ # @note This method is incompatible with background polling threads.
128
+ # If background polling is enabled, use manual polling instead (e.g., consumer.poll)
129
+ #
130
+ # @param fd [Integer] your file descriptor (from IO.pipe or eventfd)
131
+ # @param payload [String] data to write to fd when queue has data (default: "\x01")
132
+ # @return [nil]
133
+ # @raise [ClosedInnerError] when the handle is closed
134
+ # @raise [RuntimeError] when background polling thread is active
135
+ #
136
+ # @example
137
+ # # Create your own signaling FD
138
+ # signal_r, signal_w = IO.pipe
139
+ # native_kafka.enable_main_queue_io_events(signal_w.fileno)
140
+ #
141
+ # # Monitor it with select
142
+ # readable, = IO.select([signal_r], nil, nil, timeout)
143
+ # if readable
144
+ # consumer.poll(0) # Get messages
145
+ # end
146
+ def enable_main_queue_io_events(fd, payload = "\x01")
147
+ if @run_polling_thread
148
+ raise "Cannot enable IO events while background polling thread is active. " \
149
+ "Either disable background polling by setting run_polling_thread: false, " \
150
+ "or use manual polling with consumer.poll() instead of the FD API."
151
+ end
152
+
153
+ with_inner do |inner|
154
+ queue_ptr = Bindings.rd_kafka_queue_get_main(inner)
155
+ Bindings.rd_kafka_queue_io_event_enable(queue_ptr, fd, payload, payload.bytesize)
156
+ Bindings.rd_kafka_queue_destroy(queue_ptr)
157
+ end
158
+ end
159
+
160
+ # Enable IO event notifications on the background queue
161
+ # Librdkafka will write to your FD when the background queue transitions from empty to non-empty
162
+ #
163
+ # @note This method is incompatible with background polling threads.
164
+ # If background polling is enabled, use manual polling instead (e.g., consumer.poll)
165
+ #
166
+ # @param fd [Integer] your file descriptor (from IO.pipe or eventfd)
167
+ # @param payload [String] data to write to fd when queue has data (default: "\x01")
168
+ # @return [nil]
169
+ # @raise [ClosedInnerError] when the handle is closed
170
+ # @raise [RuntimeError] when background polling thread is active
171
+ def enable_background_queue_io_events(fd, payload = "\x01")
172
+ if @run_polling_thread
173
+ raise "Cannot enable IO events while background polling thread is active. " \
174
+ "Either disable background polling by setting run_polling_thread: false, " \
175
+ "or use manual polling with consumer.poll() instead of the FD API."
176
+ end
177
+
178
+ with_inner do |inner|
179
+ queue_ptr = Bindings.rd_kafka_queue_get_background(inner)
180
+ Bindings.rd_kafka_queue_io_event_enable(queue_ptr, fd, payload, payload.bytesize)
181
+ Bindings.rd_kafka_queue_destroy(queue_ptr)
182
+ end
183
+ end
184
+
185
+ # Closes the native Kafka handle and cleans up resources
186
+ # @param object_id [Integer, nil] optional object ID (unused, for finalizer compatibility)
187
+ # @yield optional block to execute before destroying the handle
188
+ # @return [nil]
189
+ def close(object_id = nil)
105
190
  return if closed?
106
191
 
107
192
  synchronize do
@@ -6,10 +6,10 @@ module Rdkafka
6
6
  # producing a message.
7
7
  class DeliveryHandle < Rdkafka::AbstractHandle
8
8
  layout :pending, :bool,
9
- :response, :int,
10
- :partition, :int,
11
- :offset, :int64,
12
- :topic_name, :pointer
9
+ :response, :int,
10
+ :partition, :int,
11
+ :offset, :int64,
12
+ :topic_name, :pointer
13
13
 
14
14
  # @return [Object, nil] label set during message production or nil by default
15
15
  attr_accessor :label
@@ -31,7 +31,7 @@ module Rdkafka
31
31
  # For part of errors, we will not get a topic name reference and in cases like this
32
32
  # we should not return it
33
33
  topic,
34
- self[:response] != 0 ? RdkafkaError.new(self[:response]) : nil,
34
+ (self[:response] != 0) ? RdkafkaError.new(self[:response]) : nil,
35
35
  label
36
36
  )
37
37
  end
@@ -30,10 +30,14 @@ module Rdkafka
30
30
  # is present in both places
31
31
  #
32
32
  # We do not remove the original `#topic_name` because of backwards compatibility
33
- alias topic topic_name
34
-
35
- private
36
-
33
+ alias_method :topic, :topic_name
34
+
35
+ # @private
36
+ # @param partition [Integer] partition number
37
+ # @param offset [Integer] message offset
38
+ # @param topic_name [String, nil] topic name
39
+ # @param error [Integer, nil] error code if any
40
+ # @param label [Object, nil] user-defined label
37
41
  def initialize(partition, offset, topic_name = nil, error = nil, label = nil)
38
42
  @partition = partition
39
43
  @offset = offset
@@ -17,9 +17,9 @@ module Rdkafka
17
17
  #
18
18
  # 2. Edge case handling
19
19
  # If a user configures `statistics.interval.ms` much higher than the default cache TTL
20
- # (30 seconds), the cache will still function correctly. When statistics updates don't
21
- # occur frequently enough, the cache entries will expire naturally, triggering a
22
- # blocking refresh when needed.
20
+ # ({Defaults::PARTITIONS_COUNT_CACHE_TTL_MS}ms), the cache will still function correctly.
21
+ # When statistics updates don't occur frequently enough, the cache entries will expire
22
+ # naturally, triggering a blocking refresh when needed.
23
23
  #
24
24
  # 3. User configuration awareness
25
25
  # The cache respects user-defined settings. If `topic.metadata.refresh.interval.ms` is
@@ -46,22 +46,32 @@ module Rdkafka
46
46
  class PartitionsCountCache
47
47
  include Helpers::Time
48
48
 
49
- # Default time-to-live for cached partition counts in seconds
50
- #
51
- # @note This default was chosen to balance freshness of metadata with performance
52
- # optimization. Most Kafka cluster topology changes are planned operations, making 30
53
- # seconds a reasonable compromise.
54
- DEFAULT_TTL = 30
55
-
56
49
  # Creates a new partition count cache
57
50
  #
58
- # @param ttl [Integer] Time-to-live in seconds for cached values
59
- def initialize(ttl = DEFAULT_TTL)
51
+ # @param ttl [Integer, nil] DEPRECATED: Use ttl_ms instead.
52
+ # Time-to-live in seconds for cached values. Will be removed in v1.0.0.
53
+ # @param ttl_ms [Integer, nil] Time-to-live in milliseconds for cached values.
54
+ # Defaults to {Defaults::PARTITIONS_COUNT_CACHE_TTL_MS}.
55
+ def initialize(ttl = :not_provided, ttl_ms: :not_provided)
60
56
  @counts = {}
61
57
  @mutex_hash = {}
62
58
  # Used only for @mutex_hash access to ensure thread-safety when creating new mutexes
63
59
  @mutex_for_hash = Mutex.new
64
- @ttl = ttl
60
+
61
+ # Determine which TTL value to use
62
+ if ttl != :not_provided && ttl_ms != :not_provided
63
+ warn "DEPRECATION WARNING: Both ttl and ttl_ms were provided to PartitionsCountCache. " \
64
+ "Using ttl_ms. The ttl parameter is deprecated and will be removed in v1.0.0."
65
+ @ttl_ms = ttl_ms
66
+ elsif ttl != :not_provided
67
+ warn "DEPRECATION WARNING: ttl (seconds) parameter for PartitionsCountCache is deprecated. " \
68
+ "Use ttl_ms (milliseconds) instead. This parameter will be removed in v1.0.0."
69
+ @ttl_ms = (ttl * 1000).to_i
70
+ elsif ttl_ms == :not_provided
71
+ @ttl_ms = Defaults::PARTITIONS_COUNT_CACHE_TTL_MS
72
+ else
73
+ @ttl_ms = ttl_ms
74
+ end
65
75
  end
66
76
 
67
77
  # Reads partition count for a topic with automatic refresh when expired
@@ -137,17 +147,17 @@ module Rdkafka
137
147
 
138
148
  if current_info.nil?
139
149
  # Create new entry
140
- @counts[topic] = [monotonic_now, new_count]
150
+ @counts[topic] = [monotonic_now_ms, new_count]
141
151
  else
142
152
  current_count = current_info[1]
143
153
 
144
154
  if new_count > current_count
145
155
  # Update to higher count value
146
- current_info[0] = monotonic_now
156
+ current_info[0] = monotonic_now_ms
147
157
  current_info[1] = new_count
148
158
  else
149
159
  # Same or lower count, update timestamp only
150
- current_info[0] = monotonic_now
160
+ current_info[0] = monotonic_now_ms
151
161
  end
152
162
  end
153
163
  end
@@ -201,15 +211,15 @@ module Rdkafka
201
211
  return unless current_info
202
212
 
203
213
  # Update the timestamp in-place
204
- current_info[0] = monotonic_now
214
+ current_info[0] = monotonic_now_ms
205
215
  end
206
216
 
207
217
  # Check if a timestamp has expired based on the TTL
208
218
  #
209
- # @param timestamp [Float] Monotonic timestamp to check
219
+ # @param timestamp [Integer] Monotonic timestamp in milliseconds to check
210
220
  # @return [Boolean] true if expired, false otherwise
211
221
  def expired?(timestamp)
212
- monotonic_now - timestamp > @ttl
222
+ monotonic_now_ms - timestamp > @ttl_ms
213
223
  end
214
224
  end
215
225
  end