rdkafka 0.13.0 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +2 -0
  3. data/.github/FUNDING.yml +1 -0
  4. data/.github/workflows/ci.yml +58 -0
  5. data/.gitignore +4 -0
  6. data/.rspec +1 -0
  7. data/.ruby-gemset +1 -0
  8. data/.ruby-version +1 -0
  9. data/CHANGELOG.md +141 -111
  10. data/{LICENSE → MIT-LICENSE} +2 -1
  11. data/README.md +48 -39
  12. data/certs/cert_chain.pem +26 -0
  13. data/docker-compose.yml +18 -15
  14. data/ext/README.md +1 -1
  15. data/ext/Rakefile +1 -1
  16. data/lib/rdkafka/abstract_handle.rb +40 -26
  17. data/lib/rdkafka/admin/acl_binding_result.rb +37 -0
  18. data/lib/rdkafka/admin/create_acl_handle.rb +28 -0
  19. data/lib/rdkafka/admin/create_acl_report.rb +24 -0
  20. data/lib/rdkafka/admin/create_partitions_handle.rb +27 -0
  21. data/lib/rdkafka/admin/create_partitions_report.rb +6 -0
  22. data/lib/rdkafka/admin/delete_acl_handle.rb +30 -0
  23. data/lib/rdkafka/admin/delete_acl_report.rb +23 -0
  24. data/lib/rdkafka/admin/delete_groups_handle.rb +28 -0
  25. data/lib/rdkafka/admin/delete_groups_report.rb +24 -0
  26. data/lib/rdkafka/admin/describe_acl_handle.rb +30 -0
  27. data/lib/rdkafka/admin/describe_acl_report.rb +23 -0
  28. data/lib/rdkafka/admin.rb +449 -7
  29. data/lib/rdkafka/bindings.rb +127 -5
  30. data/lib/rdkafka/callbacks.rb +187 -0
  31. data/lib/rdkafka/config.rb +53 -19
  32. data/lib/rdkafka/consumer/headers.rb +2 -4
  33. data/lib/rdkafka/consumer/topic_partition_list.rb +11 -8
  34. data/lib/rdkafka/consumer.rb +134 -59
  35. data/lib/rdkafka/helpers/time.rb +14 -0
  36. data/lib/rdkafka/metadata.rb +22 -1
  37. data/lib/rdkafka/native_kafka.rb +6 -1
  38. data/lib/rdkafka/producer.rb +87 -9
  39. data/lib/rdkafka/version.rb +3 -3
  40. data/lib/rdkafka.rb +21 -1
  41. data/rdkafka.gemspec +17 -3
  42. data/renovate.json +6 -0
  43. data/spec/rdkafka/abstract_handle_spec.rb +0 -2
  44. data/spec/rdkafka/admin/create_acl_handle_spec.rb +56 -0
  45. data/spec/rdkafka/admin/create_acl_report_spec.rb +18 -0
  46. data/spec/rdkafka/admin/create_topic_handle_spec.rb +0 -2
  47. data/spec/rdkafka/admin/create_topic_report_spec.rb +0 -2
  48. data/spec/rdkafka/admin/delete_acl_handle_spec.rb +85 -0
  49. data/spec/rdkafka/admin/delete_acl_report_spec.rb +71 -0
  50. data/spec/rdkafka/admin/delete_topic_handle_spec.rb +0 -2
  51. data/spec/rdkafka/admin/delete_topic_report_spec.rb +0 -2
  52. data/spec/rdkafka/admin/describe_acl_handle_spec.rb +85 -0
  53. data/spec/rdkafka/admin/describe_acl_report_spec.rb +72 -0
  54. data/spec/rdkafka/admin_spec.rb +205 -2
  55. data/spec/rdkafka/bindings_spec.rb +0 -1
  56. data/spec/rdkafka/callbacks_spec.rb +0 -2
  57. data/spec/rdkafka/config_spec.rb +8 -2
  58. data/spec/rdkafka/consumer/headers_spec.rb +0 -2
  59. data/spec/rdkafka/consumer/message_spec.rb +0 -2
  60. data/spec/rdkafka/consumer/partition_spec.rb +0 -2
  61. data/spec/rdkafka/consumer/topic_partition_list_spec.rb +19 -2
  62. data/spec/rdkafka/consumer_spec.rb +212 -39
  63. data/spec/rdkafka/error_spec.rb +0 -2
  64. data/spec/rdkafka/metadata_spec.rb +2 -3
  65. data/spec/rdkafka/native_kafka_spec.rb +2 -3
  66. data/spec/rdkafka/producer/delivery_handle_spec.rb +0 -2
  67. data/spec/rdkafka/producer/delivery_report_spec.rb +0 -2
  68. data/spec/rdkafka/producer_spec.rb +157 -1
  69. data/spec/spec_helper.rb +3 -1
  70. data.tar.gz.sig +3 -0
  71. metadata +76 -13
  72. metadata.gz.sig +3 -0
  73. data/.semaphore/semaphore.yml +0 -27
@@ -1,28 +1,37 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "ffi"
4
-
5
3
  module Rdkafka
4
+ # This class serves as an abstract base class to represent handles within the Rdkafka module.
5
+ # As a subclass of `FFI::Struct`, this class provides a blueprint for other specific handle
6
+ # classes to inherit from, ensuring they adhere to a particular structure and behavior.
7
+ #
8
+ # Subclasses must define their own layout, and the layout must start with:
9
+ #
10
+ # layout :pending, :bool,
11
+ # :response, :int
6
12
  class AbstractHandle < FFI::Struct
7
- # Subclasses must define their own layout, and the layout must start with:
8
- #
9
- # layout :pending, :bool,
10
- # :response, :int
13
+ include Helpers::Time
11
14
 
15
+ # Registry for registering all the handles.
12
16
  REGISTRY = {}
13
17
 
14
- CURRENT_TIME = -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }.freeze
15
-
16
- private_constant :CURRENT_TIME
18
+ class << self
19
+ # Adds handle to the register
20
+ #
21
+ # @param handle [AbstractHandle] any handle we want to register
22
+ def register(handle)
23
+ address = handle.to_ptr.address
24
+ REGISTRY[address] = handle
25
+ end
17
26
 
18
- def self.register(handle)
19
- address = handle.to_ptr.address
20
- REGISTRY[address] = handle
27
+ # Removes handle from the register based on the handle address
28
+ #
29
+ # @param address [Integer] address of the registered handle we want to remove
30
+ def remove(address)
31
+ REGISTRY.delete(address)
32
+ end
21
33
  end
22
34
 
23
- def self.remove(address)
24
- REGISTRY.delete(address)
25
- end
26
35
 
27
36
  # Whether the handle is still pending.
28
37
  #
@@ -32,29 +41,34 @@ module Rdkafka
32
41
  end
33
42
 
34
43
  # Wait for the operation to complete or raise an error if this takes longer than the timeout.
35
- # If there is a timeout this does not mean the operation failed, rdkafka might still be working on the operation.
36
- # In this case it is possible to call wait again.
44
+ # If there is a timeout this does not mean the operation failed, rdkafka might still be working
45
+ # on the operation. In this case it is possible to call wait again.
46
+ #
47
+ # @param max_wait_timeout [Numeric, nil] Amount of time to wait before timing out.
48
+ # If this is nil it does not time out.
49
+ # @param wait_timeout [Numeric] Amount of time we should wait before we recheck if the
50
+ # operation has completed
51
+ # @param raise_response_error [Boolean] should we raise error when waiting finishes
37
52
  #
38
- # @param max_wait_timeout [Numeric, nil] Amount of time to wait before timing out. If this is nil it does not time out.
39
- # @param wait_timeout [Numeric] Amount of time we should wait before we recheck if the operation has completed
53
+ # @return [Object] Operation-specific result
40
54
  #
41
55
  # @raise [RdkafkaError] When the operation failed
42
56
  # @raise [WaitTimeoutError] When the timeout has been reached and the handle is still pending
43
- #
44
- # @return [Object] Operation-specific result
45
- def wait(max_wait_timeout: 60, wait_timeout: 0.1)
57
+ def wait(max_wait_timeout: 60, wait_timeout: 0.1, raise_response_error: true)
46
58
  timeout = if max_wait_timeout
47
- CURRENT_TIME.call + max_wait_timeout
59
+ monotonic_now + max_wait_timeout
48
60
  else
49
61
  nil
50
62
  end
51
63
  loop do
52
64
  if pending?
53
- if timeout && timeout <= CURRENT_TIME.call
54
- raise WaitTimeoutError.new("Waiting for #{operation_name} timed out after #{max_wait_timeout} seconds")
65
+ if timeout && timeout <= monotonic_now
66
+ raise WaitTimeoutError.new(
67
+ "Waiting for #{operation_name} timed out after #{max_wait_timeout} seconds"
68
+ )
55
69
  end
56
70
  sleep wait_timeout
57
- elsif self[:response] != 0
71
+ elsif self[:response] != 0 && raise_response_error
58
72
  raise_error
59
73
  else
60
74
  return create_result
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ class Admin
5
+
6
+ # Extracts attributes of rd_kafka_AclBinding_t
7
+ #
8
+ class AclBindingResult
9
+ attr_reader :result_error, :error_string, :matching_acl_resource_type, :matching_acl_resource_name, :matching_acl_pattern_type, :matching_acl_principal, :matching_acl_host, :matching_acl_operation, :matching_acl_permission_type
10
+
11
+ def initialize(matching_acl)
12
+ rd_kafka_error_pointer = Rdkafka::Bindings.rd_kafka_AclBinding_error(matching_acl)
13
+ @result_error = Rdkafka::Bindings.rd_kafka_error_code(rd_kafka_error_pointer)
14
+ error_string = Rdkafka::Bindings.rd_kafka_error_string(rd_kafka_error_pointer)
15
+ if error_string != FFI::Pointer::NULL
16
+ @error_string = error_string.read_string
17
+ end
18
+ @matching_acl_resource_type = Rdkafka::Bindings.rd_kafka_AclBinding_restype(matching_acl)
19
+ matching_acl_resource_name = Rdkafka::Bindings.rd_kafka_AclBinding_name(matching_acl)
20
+ if matching_acl_resource_name != FFI::Pointer::NULL
21
+ @matching_acl_resource_name = matching_acl_resource_name.read_string
22
+ end
23
+ @matching_acl_pattern_type = Rdkafka::Bindings.rd_kafka_AclBinding_resource_pattern_type(matching_acl)
24
+ matching_acl_principal = Rdkafka::Bindings.rd_kafka_AclBinding_principal(matching_acl)
25
+ if matching_acl_principal != FFI::Pointer::NULL
26
+ @matching_acl_principal = matching_acl_principal.read_string
27
+ end
28
+ matching_acl_host = Rdkafka::Bindings.rd_kafka_AclBinding_host(matching_acl)
29
+ if matching_acl_host != FFI::Pointer::NULL
30
+ @matching_acl_host = matching_acl_host.read_string
31
+ end
32
+ @matching_acl_operation = Rdkafka::Bindings.rd_kafka_AclBinding_operation(matching_acl)
33
+ @matching_acl_permission_type = Rdkafka::Bindings.rd_kafka_AclBinding_permission_type(matching_acl)
34
+ end
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ class Admin
5
+ class CreateAclHandle < AbstractHandle
6
+ layout :pending, :bool,
7
+ :response, :int,
8
+ :response_string, :pointer
9
+
10
+ # @return [String] the name of the operation
11
+ def operation_name
12
+ "create acl"
13
+ end
14
+
15
+ # @return [CreateAclReport] instance with rdkafka_response value as 0 and rdkafka_response_string value as empty string if the acl creation was successful
16
+ def create_result
17
+ CreateAclReport.new(rdkafka_response: self[:response], rdkafka_response_string: self[:response_string])
18
+ end
19
+
20
+ def raise_error
21
+ raise RdkafkaError.new(
22
+ self[:response],
23
+ broker_message: self[:response_string].read_string
24
+ )
25
+ end
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ class Admin
5
+ class CreateAclReport
6
+
7
+ # Upon successful creation of Acl RD_KAFKA_RESP_ERR_NO_ERROR - 0 is returned as rdkafka_response
8
+ # @return [Integer]
9
+ attr_reader :rdkafka_response
10
+
11
+
12
+ # Upon successful creation of Acl empty string will be returned as rdkafka_response_string
13
+ # @return [String]
14
+ attr_reader :rdkafka_response_string
15
+
16
+ def initialize(rdkafka_response:, rdkafka_response_string:)
17
+ @rdkafka_response = rdkafka_response
18
+ if rdkafka_response_string != FFI::Pointer::NULL
19
+ @rdkafka_response_string = rdkafka_response_string.read_string
20
+ end
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,27 @@
1
+ module Rdkafka
2
+ class Admin
3
+ class CreatePartitionsHandle < AbstractHandle
4
+ layout :pending, :bool,
5
+ :response, :int,
6
+ :error_string, :pointer,
7
+ :result_name, :pointer
8
+
9
+ # @return [String] the name of the operation
10
+ def operation_name
11
+ "create partitions"
12
+ end
13
+
14
+ # @return [Boolean] whether the create topic was successful
15
+ def create_result
16
+ CreatePartitionsReport.new(self[:error_string], self[:result_name])
17
+ end
18
+
19
+ def raise_error
20
+ raise RdkafkaError.new(
21
+ self[:response],
22
+ broker_message: CreateTopicReport.new(self[:error_string], self[:result_name]).error_string
23
+ )
24
+ end
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,6 @@
1
+ module Rdkafka
2
+ class Admin
3
+ class CreatePartitionsReport < CreateTopicReport
4
+ end
5
+ end
6
+ end
@@ -0,0 +1,30 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ class Admin
5
+ class DeleteAclHandle < AbstractHandle
6
+ layout :pending, :bool,
7
+ :response, :int,
8
+ :response_string, :pointer,
9
+ :matching_acls, :pointer,
10
+ :matching_acls_count, :int
11
+
12
+ # @return [String] the name of the operation
13
+ def operation_name
14
+ "delete acl"
15
+ end
16
+
17
+ # @return [DeleteAclReport] instance with an array of matching_acls
18
+ def create_result
19
+ DeleteAclReport.new(matching_acls: self[:matching_acls], matching_acls_count: self[:matching_acls_count])
20
+ end
21
+
22
+ def raise_error
23
+ raise RdkafkaError.new(
24
+ self[:response],
25
+ broker_message: self[:response_string].read_string
26
+ )
27
+ end
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ class Admin
5
+ class DeleteAclReport
6
+
7
+ # deleted acls
8
+ # @return [Rdkafka::Bindings::AclBindingResult]
9
+ attr_reader :deleted_acls
10
+
11
+ def initialize(matching_acls:, matching_acls_count:)
12
+ @deleted_acls=[]
13
+ if matching_acls != FFI::Pointer::NULL
14
+ acl_binding_result_pointers = matching_acls.read_array_of_pointer(matching_acls_count)
15
+ (1..matching_acls_count).map do |matching_acl_index|
16
+ acl_binding_result = AclBindingResult.new(acl_binding_result_pointers[matching_acl_index - 1])
17
+ @deleted_acls << acl_binding_result
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ class Admin
5
+ class DeleteGroupsHandle < AbstractHandle
6
+ layout :pending, :bool, # TODO: ???
7
+ :response, :int,
8
+ :error_string, :pointer,
9
+ :result_name, :pointer
10
+
11
+ # @return [String] the name of the operation
12
+ def operation_name
13
+ "delete groups"
14
+ end
15
+
16
+ def create_result
17
+ DeleteGroupsReport.new(self[:error_string], self[:result_name])
18
+ end
19
+
20
+ def raise_error
21
+ raise RdkafkaError.new(
22
+ self[:response],
23
+ broker_message: create_result.error_string
24
+ )
25
+ end
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ class Admin
5
+ class DeleteGroupsReport
6
+ # Any error message generated from the DeleteTopic
7
+ # @return [String]
8
+ attr_reader :error_string
9
+
10
+ # The name of the topic deleted
11
+ # @return [String]
12
+ attr_reader :result_name
13
+
14
+ def initialize(error_string, result_name)
15
+ if error_string != FFI::Pointer::NULL
16
+ @error_string = error_string.read_string
17
+ end
18
+ if result_name != FFI::Pointer::NULL
19
+ @result_name = @result_name = result_name.read_string
20
+ end
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,30 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ class Admin
5
+ class DescribeAclHandle < AbstractHandle
6
+ layout :pending, :bool,
7
+ :response, :int,
8
+ :response_string, :pointer,
9
+ :acls, :pointer,
10
+ :acls_count, :int
11
+
12
+ # @return [String] the name of the operation.
13
+ def operation_name
14
+ "describe acl"
15
+ end
16
+
17
+ # @return [DescribeAclReport] instance with an array of acls that matches the request filters.
18
+ def create_result
19
+ DescribeAclReport.new(acls: self[:acls], acls_count: self[:acls_count])
20
+ end
21
+
22
+ def raise_error
23
+ raise RdkafkaError.new(
24
+ self[:response],
25
+ broker_message: self[:response_string].read_string
26
+ )
27
+ end
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ class Admin
5
+ class DescribeAclReport
6
+
7
+ # acls that exists in the cluster for the resource_type, resource_name and pattern_type filters provided in the request.
8
+ # @return [Rdkafka::Bindings::AclBindingResult] array of matching acls.
9
+ attr_reader :acls
10
+
11
+ def initialize(acls:, acls_count:)
12
+ @acls=[]
13
+ if acls != FFI::Pointer::NULL
14
+ acl_binding_result_pointers = acls.read_array_of_pointer(acls_count)
15
+ (1..acls_count).map do |acl_index|
16
+ acl_binding_result = AclBindingResult.new(acl_binding_result_pointers[acl_index - 1])
17
+ @acls << acl_binding_result
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end