rdkafka 0.12.0 → 0.15.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (86) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +2 -0
  3. data/.github/FUNDING.yml +1 -0
  4. data/.github/workflows/ci.yml +58 -0
  5. data/.gitignore +4 -0
  6. data/.rspec +1 -0
  7. data/.ruby-gemset +1 -0
  8. data/.ruby-version +1 -0
  9. data/CHANGELOG.md +141 -93
  10. data/Gemfile +2 -0
  11. data/{LICENSE → MIT-LICENSE} +2 -1
  12. data/README.md +64 -29
  13. data/Rakefile +2 -0
  14. data/certs/cert_chain.pem +26 -0
  15. data/docker-compose.yml +18 -15
  16. data/ext/README.md +1 -1
  17. data/ext/Rakefile +3 -1
  18. data/lib/rdkafka/abstract_handle.rb +41 -25
  19. data/lib/rdkafka/admin/acl_binding_result.rb +37 -0
  20. data/lib/rdkafka/admin/create_acl_handle.rb +28 -0
  21. data/lib/rdkafka/admin/create_acl_report.rb +24 -0
  22. data/lib/rdkafka/admin/create_partitions_handle.rb +27 -0
  23. data/lib/rdkafka/admin/create_partitions_report.rb +6 -0
  24. data/lib/rdkafka/admin/create_topic_handle.rb +2 -0
  25. data/lib/rdkafka/admin/create_topic_report.rb +2 -0
  26. data/lib/rdkafka/admin/delete_acl_handle.rb +30 -0
  27. data/lib/rdkafka/admin/delete_acl_report.rb +23 -0
  28. data/lib/rdkafka/admin/delete_groups_handle.rb +28 -0
  29. data/lib/rdkafka/admin/delete_groups_report.rb +24 -0
  30. data/lib/rdkafka/admin/delete_topic_handle.rb +2 -0
  31. data/lib/rdkafka/admin/delete_topic_report.rb +2 -0
  32. data/lib/rdkafka/admin/describe_acl_handle.rb +30 -0
  33. data/lib/rdkafka/admin/describe_acl_report.rb +23 -0
  34. data/lib/rdkafka/admin.rb +494 -35
  35. data/lib/rdkafka/bindings.rb +175 -40
  36. data/lib/rdkafka/callbacks.rb +194 -1
  37. data/lib/rdkafka/config.rb +62 -25
  38. data/lib/rdkafka/consumer/headers.rb +24 -9
  39. data/lib/rdkafka/consumer/message.rb +3 -1
  40. data/lib/rdkafka/consumer/partition.rb +2 -0
  41. data/lib/rdkafka/consumer/topic_partition_list.rb +13 -8
  42. data/lib/rdkafka/consumer.rb +219 -102
  43. data/lib/rdkafka/error.rb +15 -0
  44. data/lib/rdkafka/helpers/time.rb +14 -0
  45. data/lib/rdkafka/metadata.rb +25 -2
  46. data/lib/rdkafka/native_kafka.rb +120 -0
  47. data/lib/rdkafka/producer/delivery_handle.rb +5 -2
  48. data/lib/rdkafka/producer/delivery_report.rb +9 -2
  49. data/lib/rdkafka/producer.rb +117 -17
  50. data/lib/rdkafka/version.rb +5 -3
  51. data/lib/rdkafka.rb +24 -2
  52. data/rdkafka.gemspec +19 -3
  53. data/renovate.json +6 -0
  54. data/spec/rdkafka/abstract_handle_spec.rb +1 -1
  55. data/spec/rdkafka/admin/create_acl_handle_spec.rb +56 -0
  56. data/spec/rdkafka/admin/create_acl_report_spec.rb +18 -0
  57. data/spec/rdkafka/admin/create_topic_handle_spec.rb +1 -1
  58. data/spec/rdkafka/admin/create_topic_report_spec.rb +1 -1
  59. data/spec/rdkafka/admin/delete_acl_handle_spec.rb +85 -0
  60. data/spec/rdkafka/admin/delete_acl_report_spec.rb +71 -0
  61. data/spec/rdkafka/admin/delete_topic_handle_spec.rb +1 -1
  62. data/spec/rdkafka/admin/delete_topic_report_spec.rb +1 -1
  63. data/spec/rdkafka/admin/describe_acl_handle_spec.rb +85 -0
  64. data/spec/rdkafka/admin/describe_acl_report_spec.rb +72 -0
  65. data/spec/rdkafka/admin_spec.rb +209 -5
  66. data/spec/rdkafka/bindings_spec.rb +2 -1
  67. data/spec/rdkafka/callbacks_spec.rb +1 -1
  68. data/spec/rdkafka/config_spec.rb +24 -3
  69. data/spec/rdkafka/consumer/headers_spec.rb +60 -0
  70. data/spec/rdkafka/consumer/message_spec.rb +1 -1
  71. data/spec/rdkafka/consumer/partition_spec.rb +1 -1
  72. data/spec/rdkafka/consumer/topic_partition_list_spec.rb +20 -1
  73. data/spec/rdkafka/consumer_spec.rb +332 -61
  74. data/spec/rdkafka/error_spec.rb +1 -1
  75. data/spec/rdkafka/metadata_spec.rb +4 -3
  76. data/spec/rdkafka/{producer/client_spec.rb → native_kafka_spec.rb} +13 -35
  77. data/spec/rdkafka/producer/delivery_handle_spec.rb +4 -1
  78. data/spec/rdkafka/producer/delivery_report_spec.rb +7 -3
  79. data/spec/rdkafka/producer_spec.rb +208 -20
  80. data/spec/spec_helper.rb +20 -2
  81. data.tar.gz.sig +3 -0
  82. metadata +79 -16
  83. metadata.gz.sig +3 -0
  84. data/.semaphore/semaphore.yml +0 -23
  85. data/bin/console +0 -11
  86. data/lib/rdkafka/producer/client.rb +0 -47
@@ -1,26 +1,37 @@
1
- require "ffi"
1
+ # frozen_string_literal: true
2
2
 
3
3
  module Rdkafka
4
+ # This class serves as an abstract base class to represent handles within the Rdkafka module.
5
+ # As a subclass of `FFI::Struct`, this class provides a blueprint for other specific handle
6
+ # classes to inherit from, ensuring they adhere to a particular structure and behavior.
7
+ #
8
+ # Subclasses must define their own layout, and the layout must start with:
9
+ #
10
+ # layout :pending, :bool,
11
+ # :response, :int
4
12
  class AbstractHandle < FFI::Struct
5
- # Subclasses must define their own layout, and the layout must start with:
6
- #
7
- # layout :pending, :bool,
8
- # :response, :int
13
+ include Helpers::Time
9
14
 
15
+ # Registry for registering all the handles.
10
16
  REGISTRY = {}
11
17
 
12
- CURRENT_TIME = -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }.freeze
13
-
14
- private_constant :CURRENT_TIME
18
+ class << self
19
+ # Adds handle to the register
20
+ #
21
+ # @param handle [AbstractHandle] any handle we want to register
22
+ def register(handle)
23
+ address = handle.to_ptr.address
24
+ REGISTRY[address] = handle
25
+ end
15
26
 
16
- def self.register(handle)
17
- address = handle.to_ptr.address
18
- REGISTRY[address] = handle
27
+ # Removes handle from the register based on the handle address
28
+ #
29
+ # @param address [Integer] address of the registered handle we want to remove
30
+ def remove(address)
31
+ REGISTRY.delete(address)
32
+ end
19
33
  end
20
34
 
21
- def self.remove(address)
22
- REGISTRY.delete(address)
23
- end
24
35
 
25
36
  # Whether the handle is still pending.
26
37
  #
@@ -30,29 +41,34 @@ module Rdkafka
30
41
  end
31
42
 
32
43
  # Wait for the operation to complete or raise an error if this takes longer than the timeout.
33
- # If there is a timeout this does not mean the operation failed, rdkafka might still be working on the operation.
34
- # In this case it is possible to call wait again.
44
+ # If there is a timeout this does not mean the operation failed, rdkafka might still be working
45
+ # on the operation. In this case it is possible to call wait again.
35
46
  #
36
- # @param max_wait_timeout [Numeric, nil] Amount of time to wait before timing out. If this is nil it does not time out.
37
- # @param wait_timeout [Numeric] Amount of time we should wait before we recheck if the operation has completed
47
+ # @param max_wait_timeout [Numeric, nil] Amount of time to wait before timing out.
48
+ # If this is nil it does not time out.
49
+ # @param wait_timeout [Numeric] Amount of time we should wait before we recheck if the
50
+ # operation has completed
51
+ # @param raise_response_error [Boolean] should we raise error when waiting finishes
52
+ #
53
+ # @return [Object] Operation-specific result
38
54
  #
39
55
  # @raise [RdkafkaError] When the operation failed
40
56
  # @raise [WaitTimeoutError] When the timeout has been reached and the handle is still pending
41
- #
42
- # @return [Object] Operation-specific result
43
- def wait(max_wait_timeout: 60, wait_timeout: 0.1)
57
+ def wait(max_wait_timeout: 60, wait_timeout: 0.1, raise_response_error: true)
44
58
  timeout = if max_wait_timeout
45
- CURRENT_TIME.call + max_wait_timeout
59
+ monotonic_now + max_wait_timeout
46
60
  else
47
61
  nil
48
62
  end
49
63
  loop do
50
64
  if pending?
51
- if timeout && timeout <= CURRENT_TIME.call
52
- raise WaitTimeoutError.new("Waiting for #{operation_name} timed out after #{max_wait_timeout} seconds")
65
+ if timeout && timeout <= monotonic_now
66
+ raise WaitTimeoutError.new(
67
+ "Waiting for #{operation_name} timed out after #{max_wait_timeout} seconds"
68
+ )
53
69
  end
54
70
  sleep wait_timeout
55
- elsif self[:response] != 0
71
+ elsif self[:response] != 0 && raise_response_error
56
72
  raise_error
57
73
  else
58
74
  return create_result
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ class Admin
5
+
6
+ # Extracts attributes of rd_kafka_AclBinding_t
7
+ #
8
+ class AclBindingResult
9
+ attr_reader :result_error, :error_string, :matching_acl_resource_type, :matching_acl_resource_name, :matching_acl_pattern_type, :matching_acl_principal, :matching_acl_host, :matching_acl_operation, :matching_acl_permission_type
10
+
11
+ def initialize(matching_acl)
12
+ rd_kafka_error_pointer = Rdkafka::Bindings.rd_kafka_AclBinding_error(matching_acl)
13
+ @result_error = Rdkafka::Bindings.rd_kafka_error_code(rd_kafka_error_pointer)
14
+ error_string = Rdkafka::Bindings.rd_kafka_error_string(rd_kafka_error_pointer)
15
+ if error_string != FFI::Pointer::NULL
16
+ @error_string = error_string.read_string
17
+ end
18
+ @matching_acl_resource_type = Rdkafka::Bindings.rd_kafka_AclBinding_restype(matching_acl)
19
+ matching_acl_resource_name = Rdkafka::Bindings.rd_kafka_AclBinding_name(matching_acl)
20
+ if matching_acl_resource_name != FFI::Pointer::NULL
21
+ @matching_acl_resource_name = matching_acl_resource_name.read_string
22
+ end
23
+ @matching_acl_pattern_type = Rdkafka::Bindings.rd_kafka_AclBinding_resource_pattern_type(matching_acl)
24
+ matching_acl_principal = Rdkafka::Bindings.rd_kafka_AclBinding_principal(matching_acl)
25
+ if matching_acl_principal != FFI::Pointer::NULL
26
+ @matching_acl_principal = matching_acl_principal.read_string
27
+ end
28
+ matching_acl_host = Rdkafka::Bindings.rd_kafka_AclBinding_host(matching_acl)
29
+ if matching_acl_host != FFI::Pointer::NULL
30
+ @matching_acl_host = matching_acl_host.read_string
31
+ end
32
+ @matching_acl_operation = Rdkafka::Bindings.rd_kafka_AclBinding_operation(matching_acl)
33
+ @matching_acl_permission_type = Rdkafka::Bindings.rd_kafka_AclBinding_permission_type(matching_acl)
34
+ end
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ class Admin
5
+ class CreateAclHandle < AbstractHandle
6
+ layout :pending, :bool,
7
+ :response, :int,
8
+ :response_string, :pointer
9
+
10
+ # @return [String] the name of the operation
11
+ def operation_name
12
+ "create acl"
13
+ end
14
+
15
+ # @return [CreateAclReport] instance with rdkafka_response value as 0 and rdkafka_response_string value as empty string if the acl creation was successful
16
+ def create_result
17
+ CreateAclReport.new(rdkafka_response: self[:response], rdkafka_response_string: self[:response_string])
18
+ end
19
+
20
+ def raise_error
21
+ raise RdkafkaError.new(
22
+ self[:response],
23
+ broker_message: self[:response_string].read_string
24
+ )
25
+ end
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ class Admin
5
+ class CreateAclReport
6
+
7
+ # Upon successful creation of Acl RD_KAFKA_RESP_ERR_NO_ERROR - 0 is returned as rdkafka_response
8
+ # @return [Integer]
9
+ attr_reader :rdkafka_response
10
+
11
+
12
+ # Upon successful creation of Acl empty string will be returned as rdkafka_response_string
13
+ # @return [String]
14
+ attr_reader :rdkafka_response_string
15
+
16
+ def initialize(rdkafka_response:, rdkafka_response_string:)
17
+ @rdkafka_response = rdkafka_response
18
+ if rdkafka_response_string != FFI::Pointer::NULL
19
+ @rdkafka_response_string = rdkafka_response_string.read_string
20
+ end
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,27 @@
1
+ module Rdkafka
2
+ class Admin
3
+ class CreatePartitionsHandle < AbstractHandle
4
+ layout :pending, :bool,
5
+ :response, :int,
6
+ :error_string, :pointer,
7
+ :result_name, :pointer
8
+
9
+ # @return [String] the name of the operation
10
+ def operation_name
11
+ "create partitions"
12
+ end
13
+
14
+ # @return [Boolean] whether the create topic was successful
15
+ def create_result
16
+ CreatePartitionsReport.new(self[:error_string], self[:result_name])
17
+ end
18
+
19
+ def raise_error
20
+ raise RdkafkaError.new(
21
+ self[:response],
22
+ broker_message: CreateTopicReport.new(self[:error_string], self[:result_name]).error_string
23
+ )
24
+ end
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,6 @@
1
+ module Rdkafka
2
+ class Admin
3
+ class CreatePartitionsReport < CreateTopicReport
4
+ end
5
+ end
6
+ end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Rdkafka
2
4
  class Admin
3
5
  class CreateTopicHandle < AbstractHandle
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Rdkafka
2
4
  class Admin
3
5
  class CreateTopicReport
@@ -0,0 +1,30 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ class Admin
5
+ class DeleteAclHandle < AbstractHandle
6
+ layout :pending, :bool,
7
+ :response, :int,
8
+ :response_string, :pointer,
9
+ :matching_acls, :pointer,
10
+ :matching_acls_count, :int
11
+
12
+ # @return [String] the name of the operation
13
+ def operation_name
14
+ "delete acl"
15
+ end
16
+
17
+ # @return [DeleteAclReport] instance with an array of matching_acls
18
+ def create_result
19
+ DeleteAclReport.new(matching_acls: self[:matching_acls], matching_acls_count: self[:matching_acls_count])
20
+ end
21
+
22
+ def raise_error
23
+ raise RdkafkaError.new(
24
+ self[:response],
25
+ broker_message: self[:response_string].read_string
26
+ )
27
+ end
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ class Admin
5
+ class DeleteAclReport
6
+
7
+ # deleted acls
8
+ # @return [Rdkafka::Bindings::AclBindingResult]
9
+ attr_reader :deleted_acls
10
+
11
+ def initialize(matching_acls:, matching_acls_count:)
12
+ @deleted_acls=[]
13
+ if matching_acls != FFI::Pointer::NULL
14
+ acl_binding_result_pointers = matching_acls.read_array_of_pointer(matching_acls_count)
15
+ (1..matching_acls_count).map do |matching_acl_index|
16
+ acl_binding_result = AclBindingResult.new(acl_binding_result_pointers[matching_acl_index - 1])
17
+ @deleted_acls << acl_binding_result
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ class Admin
5
+ class DeleteGroupsHandle < AbstractHandle
6
+ layout :pending, :bool, # TODO: ???
7
+ :response, :int,
8
+ :error_string, :pointer,
9
+ :result_name, :pointer
10
+
11
+ # @return [String] the name of the operation
12
+ def operation_name
13
+ "delete groups"
14
+ end
15
+
16
+ def create_result
17
+ DeleteGroupsReport.new(self[:error_string], self[:result_name])
18
+ end
19
+
20
+ def raise_error
21
+ raise RdkafkaError.new(
22
+ self[:response],
23
+ broker_message: create_result.error_string
24
+ )
25
+ end
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ class Admin
5
+ class DeleteGroupsReport
6
+ # Any error message generated from the DeleteTopic
7
+ # @return [String]
8
+ attr_reader :error_string
9
+
10
+ # The name of the topic deleted
11
+ # @return [String]
12
+ attr_reader :result_name
13
+
14
+ def initialize(error_string, result_name)
15
+ if error_string != FFI::Pointer::NULL
16
+ @error_string = error_string.read_string
17
+ end
18
+ if result_name != FFI::Pointer::NULL
19
+ @result_name = @result_name = result_name.read_string
20
+ end
21
+ end
22
+ end
23
+ end
24
+ end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Rdkafka
2
4
  class Admin
3
5
  class DeleteTopicHandle < AbstractHandle
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Rdkafka
2
4
  class Admin
3
5
  class DeleteTopicReport
@@ -0,0 +1,30 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ class Admin
5
+ class DescribeAclHandle < AbstractHandle
6
+ layout :pending, :bool,
7
+ :response, :int,
8
+ :response_string, :pointer,
9
+ :acls, :pointer,
10
+ :acls_count, :int
11
+
12
+ # @return [String] the name of the operation.
13
+ def operation_name
14
+ "describe acl"
15
+ end
16
+
17
+ # @return [DescribeAclReport] instance with an array of acls that matches the request filters.
18
+ def create_result
19
+ DescribeAclReport.new(acls: self[:acls], acls_count: self[:acls_count])
20
+ end
21
+
22
+ def raise_error
23
+ raise RdkafkaError.new(
24
+ self[:response],
25
+ broker_message: self[:response_string].read_string
26
+ )
27
+ end
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ class Admin
5
+ class DescribeAclReport
6
+
7
+ # acls that exists in the cluster for the resource_type, resource_name and pattern_type filters provided in the request.
8
+ # @return [Rdkafka::Bindings::AclBindingResult] array of matching acls.
9
+ attr_reader :acls
10
+
11
+ def initialize(acls:, acls_count:)
12
+ @acls=[]
13
+ if acls != FFI::Pointer::NULL
14
+ acl_binding_result_pointers = acls.read_array_of_pointer(acls_count)
15
+ (1..acls_count).map do |acl_index|
16
+ acl_binding_result = AclBindingResult.new(acl_binding_result_pointers[acl_index - 1])
17
+ @acls << acl_binding_result
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end