rdkafka 0.13.0 → 0.15.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (73) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +2 -0
  3. data/.github/FUNDING.yml +1 -0
  4. data/.github/workflows/ci.yml +58 -0
  5. data/.gitignore +4 -0
  6. data/.rspec +1 -0
  7. data/.ruby-gemset +1 -0
  8. data/.ruby-version +1 -0
  9. data/CHANGELOG.md +141 -111
  10. data/{LICENSE → MIT-LICENSE} +2 -1
  11. data/README.md +48 -39
  12. data/certs/cert_chain.pem +26 -0
  13. data/docker-compose.yml +18 -15
  14. data/ext/README.md +1 -1
  15. data/ext/Rakefile +1 -1
  16. data/lib/rdkafka/abstract_handle.rb +40 -26
  17. data/lib/rdkafka/admin/acl_binding_result.rb +37 -0
  18. data/lib/rdkafka/admin/create_acl_handle.rb +28 -0
  19. data/lib/rdkafka/admin/create_acl_report.rb +24 -0
  20. data/lib/rdkafka/admin/create_partitions_handle.rb +27 -0
  21. data/lib/rdkafka/admin/create_partitions_report.rb +6 -0
  22. data/lib/rdkafka/admin/delete_acl_handle.rb +30 -0
  23. data/lib/rdkafka/admin/delete_acl_report.rb +23 -0
  24. data/lib/rdkafka/admin/delete_groups_handle.rb +28 -0
  25. data/lib/rdkafka/admin/delete_groups_report.rb +24 -0
  26. data/lib/rdkafka/admin/describe_acl_handle.rb +30 -0
  27. data/lib/rdkafka/admin/describe_acl_report.rb +23 -0
  28. data/lib/rdkafka/admin.rb +449 -7
  29. data/lib/rdkafka/bindings.rb +127 -5
  30. data/lib/rdkafka/callbacks.rb +187 -0
  31. data/lib/rdkafka/config.rb +53 -19
  32. data/lib/rdkafka/consumer/headers.rb +2 -4
  33. data/lib/rdkafka/consumer/topic_partition_list.rb +11 -8
  34. data/lib/rdkafka/consumer.rb +134 -59
  35. data/lib/rdkafka/helpers/time.rb +14 -0
  36. data/lib/rdkafka/metadata.rb +22 -1
  37. data/lib/rdkafka/native_kafka.rb +6 -1
  38. data/lib/rdkafka/producer.rb +87 -9
  39. data/lib/rdkafka/version.rb +3 -3
  40. data/lib/rdkafka.rb +21 -1
  41. data/rdkafka.gemspec +17 -3
  42. data/renovate.json +6 -0
  43. data/spec/rdkafka/abstract_handle_spec.rb +0 -2
  44. data/spec/rdkafka/admin/create_acl_handle_spec.rb +56 -0
  45. data/spec/rdkafka/admin/create_acl_report_spec.rb +18 -0
  46. data/spec/rdkafka/admin/create_topic_handle_spec.rb +0 -2
  47. data/spec/rdkafka/admin/create_topic_report_spec.rb +0 -2
  48. data/spec/rdkafka/admin/delete_acl_handle_spec.rb +85 -0
  49. data/spec/rdkafka/admin/delete_acl_report_spec.rb +71 -0
  50. data/spec/rdkafka/admin/delete_topic_handle_spec.rb +0 -2
  51. data/spec/rdkafka/admin/delete_topic_report_spec.rb +0 -2
  52. data/spec/rdkafka/admin/describe_acl_handle_spec.rb +85 -0
  53. data/spec/rdkafka/admin/describe_acl_report_spec.rb +72 -0
  54. data/spec/rdkafka/admin_spec.rb +205 -2
  55. data/spec/rdkafka/bindings_spec.rb +0 -1
  56. data/spec/rdkafka/callbacks_spec.rb +0 -2
  57. data/spec/rdkafka/config_spec.rb +8 -2
  58. data/spec/rdkafka/consumer/headers_spec.rb +0 -2
  59. data/spec/rdkafka/consumer/message_spec.rb +0 -2
  60. data/spec/rdkafka/consumer/partition_spec.rb +0 -2
  61. data/spec/rdkafka/consumer/topic_partition_list_spec.rb +19 -2
  62. data/spec/rdkafka/consumer_spec.rb +212 -39
  63. data/spec/rdkafka/error_spec.rb +0 -2
  64. data/spec/rdkafka/metadata_spec.rb +2 -3
  65. data/spec/rdkafka/native_kafka_spec.rb +2 -3
  66. data/spec/rdkafka/producer/delivery_handle_spec.rb +0 -2
  67. data/spec/rdkafka/producer/delivery_report_spec.rb +0 -2
  68. data/spec/rdkafka/producer_spec.rb +157 -1
  69. data/spec/spec_helper.rb +3 -1
  70. data.tar.gz.sig +3 -0
  71. metadata +76 -13
  72. metadata.gz.sig +3 -0
  73. data/.semaphore/semaphore.yml +0 -27
@@ -1,28 +1,37 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "ffi"
4
-
5
3
  module Rdkafka
4
+ # This class serves as an abstract base class to represent handles within the Rdkafka module.
5
+ # As a subclass of `FFI::Struct`, this class provides a blueprint for other specific handle
6
+ # classes to inherit from, ensuring they adhere to a particular structure and behavior.
7
+ #
8
+ # Subclasses must define their own layout, and the layout must start with:
9
+ #
10
+ # layout :pending, :bool,
11
+ # :response, :int
6
12
  class AbstractHandle < FFI::Struct
7
- # Subclasses must define their own layout, and the layout must start with:
8
- #
9
- # layout :pending, :bool,
10
- # :response, :int
13
+ include Helpers::Time
11
14
 
15
+ # Registry for registering all the handles.
12
16
  REGISTRY = {}
13
17
 
14
- CURRENT_TIME = -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }.freeze
15
-
16
- private_constant :CURRENT_TIME
18
+ class << self
19
+ # Adds handle to the register
20
+ #
21
+ # @param handle [AbstractHandle] any handle we want to register
22
+ def register(handle)
23
+ address = handle.to_ptr.address
24
+ REGISTRY[address] = handle
25
+ end
17
26
 
18
- def self.register(handle)
19
- address = handle.to_ptr.address
20
- REGISTRY[address] = handle
27
+ # Removes handle from the register based on the handle address
28
+ #
29
+ # @param address [Integer] address of the registered handle we want to remove
30
+ def remove(address)
31
+ REGISTRY.delete(address)
32
+ end
21
33
  end
22
34
 
23
- def self.remove(address)
24
- REGISTRY.delete(address)
25
- end
26
35
 
27
36
  # Whether the handle is still pending.
28
37
  #
@@ -32,29 +41,34 @@ module Rdkafka
32
41
  end
33
42
 
34
43
  # Wait for the operation to complete or raise an error if this takes longer than the timeout.
35
- # If there is a timeout this does not mean the operation failed, rdkafka might still be working on the operation.
36
- # In this case it is possible to call wait again.
44
+ # If there is a timeout this does not mean the operation failed, rdkafka might still be working
45
+ # on the operation. In this case it is possible to call wait again.
46
+ #
47
+ # @param max_wait_timeout [Numeric, nil] Amount of time to wait before timing out.
48
+ # If this is nil it does not time out.
49
+ # @param wait_timeout [Numeric] Amount of time we should wait before we recheck if the
50
+ # operation has completed
51
+ # @param raise_response_error [Boolean] should we raise error when waiting finishes
37
52
  #
38
- # @param max_wait_timeout [Numeric, nil] Amount of time to wait before timing out. If this is nil it does not time out.
39
- # @param wait_timeout [Numeric] Amount of time we should wait before we recheck if the operation has completed
53
+ # @return [Object] Operation-specific result
40
54
  #
41
55
  # @raise [RdkafkaError] When the operation failed
42
56
  # @raise [WaitTimeoutError] When the timeout has been reached and the handle is still pending
43
- #
44
- # @return [Object] Operation-specific result
45
- def wait(max_wait_timeout: 60, wait_timeout: 0.1)
57
+ def wait(max_wait_timeout: 60, wait_timeout: 0.1, raise_response_error: true)
46
58
  timeout = if max_wait_timeout
47
- CURRENT_TIME.call + max_wait_timeout
59
+ monotonic_now + max_wait_timeout
48
60
  else
49
61
  nil
50
62
  end
51
63
  loop do
52
64
  if pending?
53
- if timeout && timeout <= CURRENT_TIME.call
54
- raise WaitTimeoutError.new("Waiting for #{operation_name} timed out after #{max_wait_timeout} seconds")
65
+ if timeout && timeout <= monotonic_now
66
+ raise WaitTimeoutError.new(
67
+ "Waiting for #{operation_name} timed out after #{max_wait_timeout} seconds"
68
+ )
55
69
  end
56
70
  sleep wait_timeout
57
- elsif self[:response] != 0
71
+ elsif self[:response] != 0 && raise_response_error
58
72
  raise_error
59
73
  else
60
74
  return create_result
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ class Admin
5
+
6
+ # Extracts attributes of rd_kafka_AclBinding_t
7
+ #
8
+ class AclBindingResult
9
+ attr_reader :result_error, :error_string, :matching_acl_resource_type, :matching_acl_resource_name, :matching_acl_pattern_type, :matching_acl_principal, :matching_acl_host, :matching_acl_operation, :matching_acl_permission_type
10
+
11
+ def initialize(matching_acl)
12
+ rd_kafka_error_pointer = Rdkafka::Bindings.rd_kafka_AclBinding_error(matching_acl)
13
+ @result_error = Rdkafka::Bindings.rd_kafka_error_code(rd_kafka_error_pointer)
14
+ error_string = Rdkafka::Bindings.rd_kafka_error_string(rd_kafka_error_pointer)
15
+ if error_string != FFI::Pointer::NULL
16
+ @error_string = error_string.read_string
17
+ end
18
+ @matching_acl_resource_type = Rdkafka::Bindings.rd_kafka_AclBinding_restype(matching_acl)
19
+ matching_acl_resource_name = Rdkafka::Bindings.rd_kafka_AclBinding_name(matching_acl)
20
+ if matching_acl_resource_name != FFI::Pointer::NULL
21
+ @matching_acl_resource_name = matching_acl_resource_name.read_string
22
+ end
23
+ @matching_acl_pattern_type = Rdkafka::Bindings.rd_kafka_AclBinding_resource_pattern_type(matching_acl)
24
+ matching_acl_principal = Rdkafka::Bindings.rd_kafka_AclBinding_principal(matching_acl)
25
+ if matching_acl_principal != FFI::Pointer::NULL
26
+ @matching_acl_principal = matching_acl_principal.read_string
27
+ end
28
+ matching_acl_host = Rdkafka::Bindings.rd_kafka_AclBinding_host(matching_acl)
29
+ if matching_acl_host != FFI::Pointer::NULL
30
+ @matching_acl_host = matching_acl_host.read_string
31
+ end
32
+ @matching_acl_operation = Rdkafka::Bindings.rd_kafka_AclBinding_operation(matching_acl)
33
+ @matching_acl_permission_type = Rdkafka::Bindings.rd_kafka_AclBinding_permission_type(matching_acl)
34
+ end
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ class Admin
5
+ class CreateAclHandle < AbstractHandle
6
+ layout :pending, :bool,
7
+ :response, :int,
8
+ :response_string, :pointer
9
+
10
+ # @return [String] the name of the operation
11
+ def operation_name
12
+ "create acl"
13
+ end
14
+
15
+ # @return [CreateAclReport] instance with rdkafka_response value as 0 and rdkafka_response_string value as empty string if the acl creation was successful
16
+ def create_result
17
+ CreateAclReport.new(rdkafka_response: self[:response], rdkafka_response_string: self[:response_string])
18
+ end
19
+
20
+ def raise_error
21
+ raise RdkafkaError.new(
22
+ self[:response],
23
+ broker_message: self[:response_string].read_string
24
+ )
25
+ end
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ class Admin
5
+ class CreateAclReport
6
+
7
+ # Upon successful creation of Acl RD_KAFKA_RESP_ERR_NO_ERROR - 0 is returned as rdkafka_response
8
+ # @return [Integer]
9
+ attr_reader :rdkafka_response
10
+
11
+
12
+ # Upon successful creation of Acl empty string will be returned as rdkafka_response_string
13
+ # @return [String]
14
+ attr_reader :rdkafka_response_string
15
+
16
+ def initialize(rdkafka_response:, rdkafka_response_string:)
17
+ @rdkafka_response = rdkafka_response
18
+ if rdkafka_response_string != FFI::Pointer::NULL
19
+ @rdkafka_response_string = rdkafka_response_string.read_string
20
+ end
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,27 @@
1
+ module Rdkafka
2
+ class Admin
3
+ class CreatePartitionsHandle < AbstractHandle
4
+ layout :pending, :bool,
5
+ :response, :int,
6
+ :error_string, :pointer,
7
+ :result_name, :pointer
8
+
9
+ # @return [String] the name of the operation
10
+ def operation_name
11
+ "create partitions"
12
+ end
13
+
14
+ # @return [Boolean] whether the create topic was successful
15
+ def create_result
16
+ CreatePartitionsReport.new(self[:error_string], self[:result_name])
17
+ end
18
+
19
+ def raise_error
20
+ raise RdkafkaError.new(
21
+ self[:response],
22
+ broker_message: CreateTopicReport.new(self[:error_string], self[:result_name]).error_string
23
+ )
24
+ end
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,6 @@
1
+ module Rdkafka
2
+ class Admin
3
+ class CreatePartitionsReport < CreateTopicReport
4
+ end
5
+ end
6
+ end
@@ -0,0 +1,30 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ class Admin
5
+ class DeleteAclHandle < AbstractHandle
6
+ layout :pending, :bool,
7
+ :response, :int,
8
+ :response_string, :pointer,
9
+ :matching_acls, :pointer,
10
+ :matching_acls_count, :int
11
+
12
+ # @return [String] the name of the operation
13
+ def operation_name
14
+ "delete acl"
15
+ end
16
+
17
+ # @return [DeleteAclReport] instance with an array of matching_acls
18
+ def create_result
19
+ DeleteAclReport.new(matching_acls: self[:matching_acls], matching_acls_count: self[:matching_acls_count])
20
+ end
21
+
22
+ def raise_error
23
+ raise RdkafkaError.new(
24
+ self[:response],
25
+ broker_message: self[:response_string].read_string
26
+ )
27
+ end
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ class Admin
5
+ class DeleteAclReport
6
+
7
+ # deleted acls
8
+ # @return [Rdkafka::Bindings::AclBindingResult]
9
+ attr_reader :deleted_acls
10
+
11
+ def initialize(matching_acls:, matching_acls_count:)
12
+ @deleted_acls=[]
13
+ if matching_acls != FFI::Pointer::NULL
14
+ acl_binding_result_pointers = matching_acls.read_array_of_pointer(matching_acls_count)
15
+ (1..matching_acls_count).map do |matching_acl_index|
16
+ acl_binding_result = AclBindingResult.new(acl_binding_result_pointers[matching_acl_index - 1])
17
+ @deleted_acls << acl_binding_result
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ class Admin
5
+ class DeleteGroupsHandle < AbstractHandle
6
+ layout :pending, :bool, # TODO: ???
7
+ :response, :int,
8
+ :error_string, :pointer,
9
+ :result_name, :pointer
10
+
11
+ # @return [String] the name of the operation
12
+ def operation_name
13
+ "delete groups"
14
+ end
15
+
16
+ def create_result
17
+ DeleteGroupsReport.new(self[:error_string], self[:result_name])
18
+ end
19
+
20
+ def raise_error
21
+ raise RdkafkaError.new(
22
+ self[:response],
23
+ broker_message: create_result.error_string
24
+ )
25
+ end
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ class Admin
5
+ class DeleteGroupsReport
6
+ # Any error message generated from the DeleteTopic
7
+ # @return [String]
8
+ attr_reader :error_string
9
+
10
+ # The name of the topic deleted
11
+ # @return [String]
12
+ attr_reader :result_name
13
+
14
+ def initialize(error_string, result_name)
15
+ if error_string != FFI::Pointer::NULL
16
+ @error_string = error_string.read_string
17
+ end
18
+ if result_name != FFI::Pointer::NULL
19
+ @result_name = @result_name = result_name.read_string
20
+ end
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,30 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ class Admin
5
+ class DescribeAclHandle < AbstractHandle
6
+ layout :pending, :bool,
7
+ :response, :int,
8
+ :response_string, :pointer,
9
+ :acls, :pointer,
10
+ :acls_count, :int
11
+
12
+ # @return [String] the name of the operation.
13
+ def operation_name
14
+ "describe acl"
15
+ end
16
+
17
+ # @return [DescribeAclReport] instance with an array of acls that matches the request filters.
18
+ def create_result
19
+ DescribeAclReport.new(acls: self[:acls], acls_count: self[:acls_count])
20
+ end
21
+
22
+ def raise_error
23
+ raise RdkafkaError.new(
24
+ self[:response],
25
+ broker_message: self[:response_string].read_string
26
+ )
27
+ end
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ class Admin
5
+ class DescribeAclReport
6
+
7
+ # acls that exists in the cluster for the resource_type, resource_name and pattern_type filters provided in the request.
8
+ # @return [Rdkafka::Bindings::AclBindingResult] array of matching acls.
9
+ attr_reader :acls
10
+
11
+ def initialize(acls:, acls_count:)
12
+ @acls=[]
13
+ if acls != FFI::Pointer::NULL
14
+ acl_binding_result_pointers = acls.read_array_of_pointer(acls_count)
15
+ (1..acls_count).map do |acl_index|
16
+ acl_binding_result = AclBindingResult.new(acl_binding_result_pointers[acl_index - 1])
17
+ @acls << acl_binding_result
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end