rdkafka 0.22.2 → 0.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +63 -3
  3. data/Gemfile +8 -0
  4. data/Gemfile.lint +14 -0
  5. data/Gemfile.lint.lock +123 -0
  6. data/README.md +19 -14
  7. data/Rakefile +21 -21
  8. data/bin/verify_kafka_warnings +39 -0
  9. data/dist/{librdkafka-2.8.0.tar.gz → librdkafka-2.14.0.tar.gz} +0 -0
  10. data/docker-compose-ssl.yml +35 -0
  11. data/docker-compose.yml +2 -2
  12. data/ext/Rakefile +27 -27
  13. data/lib/rdkafka/abstract_handle.rb +23 -5
  14. data/lib/rdkafka/admin/acl_binding_result.rb +5 -5
  15. data/lib/rdkafka/admin/config_resource_binding_result.rb +1 -0
  16. data/lib/rdkafka/admin/create_acl_handle.rb +7 -4
  17. data/lib/rdkafka/admin/create_acl_report.rb +3 -2
  18. data/lib/rdkafka/admin/create_partitions_handle.rb +8 -5
  19. data/lib/rdkafka/admin/create_partitions_report.rb +1 -0
  20. data/lib/rdkafka/admin/create_topic_handle.rb +8 -5
  21. data/lib/rdkafka/admin/create_topic_report.rb +3 -0
  22. data/lib/rdkafka/admin/delete_acl_handle.rb +9 -6
  23. data/lib/rdkafka/admin/delete_acl_report.rb +5 -3
  24. data/lib/rdkafka/admin/delete_groups_handle.rb +10 -5
  25. data/lib/rdkafka/admin/delete_groups_report.rb +3 -0
  26. data/lib/rdkafka/admin/delete_topic_handle.rb +8 -5
  27. data/lib/rdkafka/admin/delete_topic_report.rb +3 -0
  28. data/lib/rdkafka/admin/describe_acl_handle.rb +9 -6
  29. data/lib/rdkafka/admin/describe_acl_report.rb +5 -3
  30. data/lib/rdkafka/admin/describe_configs_handle.rb +7 -4
  31. data/lib/rdkafka/admin/describe_configs_report.rb +7 -1
  32. data/lib/rdkafka/admin/incremental_alter_configs_handle.rb +7 -4
  33. data/lib/rdkafka/admin/incremental_alter_configs_report.rb +7 -1
  34. data/lib/rdkafka/admin/list_offsets_handle.rb +36 -0
  35. data/lib/rdkafka/admin/list_offsets_report.rb +51 -0
  36. data/lib/rdkafka/admin.rb +301 -135
  37. data/lib/rdkafka/bindings.rb +199 -110
  38. data/lib/rdkafka/callbacks.rb +124 -21
  39. data/lib/rdkafka/config.rb +81 -33
  40. data/lib/rdkafka/consumer/headers.rb +3 -2
  41. data/lib/rdkafka/consumer/message.rb +12 -11
  42. data/lib/rdkafka/consumer/partition.rb +8 -4
  43. data/lib/rdkafka/consumer/topic_partition_list.rb +21 -17
  44. data/lib/rdkafka/consumer.rb +397 -45
  45. data/lib/rdkafka/defaults.rb +106 -0
  46. data/lib/rdkafka/error.rb +40 -14
  47. data/lib/rdkafka/helpers/oauth.rb +45 -13
  48. data/lib/rdkafka/helpers/time.rb +5 -0
  49. data/lib/rdkafka/metadata.rb +45 -21
  50. data/lib/rdkafka/native_kafka.rb +89 -4
  51. data/lib/rdkafka/producer/delivery_handle.rb +5 -5
  52. data/lib/rdkafka/producer/delivery_report.rb +10 -6
  53. data/lib/rdkafka/producer/partitions_count_cache.rb +29 -19
  54. data/lib/rdkafka/producer.rb +168 -82
  55. data/lib/rdkafka/version.rb +6 -3
  56. data/lib/rdkafka.rb +3 -0
  57. data/package-lock.json +331 -0
  58. data/package.json +9 -0
  59. data/rdkafka.gemspec +57 -36
  60. data/renovate.json +29 -24
  61. metadata +29 -124
  62. data/.github/CODEOWNERS +0 -3
  63. data/.github/FUNDING.yml +0 -1
  64. data/.github/workflows/ci_linux_x86_64_gnu.yml +0 -271
  65. data/.github/workflows/ci_linux_x86_64_musl.yml +0 -194
  66. data/.github/workflows/ci_macos_arm64.yml +0 -284
  67. data/.github/workflows/push_linux_x86_64_gnu.yml +0 -65
  68. data/.github/workflows/push_linux_x86_64_musl.yml +0 -79
  69. data/.github/workflows/push_macos_arm64.yml +0 -54
  70. data/.github/workflows/push_ruby.yml +0 -37
  71. data/.github/workflows/verify-action-pins.yml +0 -16
  72. data/.gitignore +0 -14
  73. data/.rspec +0 -2
  74. data/.ruby-gemset +0 -1
  75. data/.ruby-version +0 -1
  76. data/.yardopts +0 -2
  77. data/ext/README.md +0 -19
  78. data/ext/build_common.sh +0 -361
  79. data/ext/build_linux_x86_64_gnu.sh +0 -306
  80. data/ext/build_linux_x86_64_musl.sh +0 -763
  81. data/ext/build_macos_arm64.sh +0 -550
  82. data/spec/rdkafka/abstract_handle_spec.rb +0 -117
  83. data/spec/rdkafka/admin/create_acl_handle_spec.rb +0 -56
  84. data/spec/rdkafka/admin/create_acl_report_spec.rb +0 -18
  85. data/spec/rdkafka/admin/create_topic_handle_spec.rb +0 -52
  86. data/spec/rdkafka/admin/create_topic_report_spec.rb +0 -16
  87. data/spec/rdkafka/admin/delete_acl_handle_spec.rb +0 -85
  88. data/spec/rdkafka/admin/delete_acl_report_spec.rb +0 -72
  89. data/spec/rdkafka/admin/delete_topic_handle_spec.rb +0 -52
  90. data/spec/rdkafka/admin/delete_topic_report_spec.rb +0 -16
  91. data/spec/rdkafka/admin/describe_acl_handle_spec.rb +0 -85
  92. data/spec/rdkafka/admin/describe_acl_report_spec.rb +0 -73
  93. data/spec/rdkafka/admin_spec.rb +0 -971
  94. data/spec/rdkafka/bindings_spec.rb +0 -199
  95. data/spec/rdkafka/callbacks_spec.rb +0 -20
  96. data/spec/rdkafka/config_spec.rb +0 -258
  97. data/spec/rdkafka/consumer/headers_spec.rb +0 -73
  98. data/spec/rdkafka/consumer/message_spec.rb +0 -139
  99. data/spec/rdkafka/consumer/partition_spec.rb +0 -57
  100. data/spec/rdkafka/consumer/topic_partition_list_spec.rb +0 -248
  101. data/spec/rdkafka/consumer_spec.rb +0 -1274
  102. data/spec/rdkafka/error_spec.rb +0 -89
  103. data/spec/rdkafka/metadata_spec.rb +0 -79
  104. data/spec/rdkafka/native_kafka_spec.rb +0 -130
  105. data/spec/rdkafka/producer/delivery_handle_spec.rb +0 -45
  106. data/spec/rdkafka/producer/delivery_report_spec.rb +0 -25
  107. data/spec/rdkafka/producer/partitions_count_cache_spec.rb +0 -359
  108. data/spec/rdkafka/producer_spec.rb +0 -1345
  109. data/spec/spec_helper.rb +0 -195
@@ -28,10 +28,11 @@ module Rdkafka
28
28
  # @return [Time, nil]
29
29
  attr_reader :timestamp
30
30
 
31
- # @return [Hash<String, String>] a message headers
31
+ # @return [Hash{String => String}] message headers
32
32
  attr_reader :headers
33
33
 
34
34
  # @private
35
+ # @param native_message [Rdkafka::Bindings::Message] native message struct from librdkafka
35
36
  def initialize(native_message)
36
37
  # Set topic
37
38
  unless native_message[:rkt].null?
@@ -52,13 +53,11 @@ module Rdkafka
52
53
  # Set timestamp
53
54
  raw_timestamp = Rdkafka::Bindings.rd_kafka_message_timestamp(native_message, nil)
54
55
  @timestamp = if raw_timestamp && raw_timestamp > -1
55
- # Calculate seconds and microseconds
56
- seconds = raw_timestamp / 1000
57
- milliseconds = (raw_timestamp - seconds * 1000) * 1000
58
- Time.at(seconds, milliseconds)
59
- else
60
- nil
61
- end
56
+ # Calculate seconds and microseconds
57
+ seconds = raw_timestamp / 1000
58
+ milliseconds = (raw_timestamp - seconds * 1000) * 1000
59
+ Time.at(seconds, milliseconds)
60
+ end
62
61
 
63
62
  @headers = Headers.from_native(native_message)
64
63
  end
@@ -71,6 +70,11 @@ module Rdkafka
71
70
  "<Message in '#{topic}' with key '#{truncate(key)}', payload '#{truncate(payload)}', partition #{partition}, offset #{offset}, timestamp #{timestamp}#{is_headers}>"
72
71
  end
73
72
 
73
+ private
74
+
75
+ # Truncates a string for display purposes
76
+ # @param string [String, nil] the string to truncate
77
+ # @return [String, nil] truncated string or nil
74
78
  def truncate(string)
75
79
  if string && string.length > 40
76
80
  "#{string[0..39]}..."
@@ -78,9 +82,6 @@ module Rdkafka
78
82
  string
79
83
  end
80
84
  end
81
-
82
- private
83
-
84
85
  end
85
86
  end
86
87
  end
@@ -17,7 +17,10 @@ module Rdkafka
17
17
  attr_reader :err
18
18
 
19
19
  # @private
20
- def initialize(partition, offset, err = 0)
20
+ # @param partition [Integer] partition number
21
+ # @param offset [Integer, nil] partition offset
22
+ # @param err [Integer] error code from librdkafka
23
+ def initialize(partition, offset, err = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR)
21
24
  @partition = partition
22
25
  @offset = offset
23
26
  @err = err
@@ -28,7 +31,7 @@ module Rdkafka
28
31
  def to_s
29
32
  message = "<Partition #{partition}"
30
33
  message += " offset=#{offset}" if offset
31
- message += " err=#{err}" if err != 0
34
+ message += " err=#{err}" if err != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
32
35
  message += ">"
33
36
  message
34
37
  end
@@ -40,11 +43,12 @@ module Rdkafka
40
43
  end
41
44
 
42
45
  # Whether another partition is equal to this
46
+ # @param other [Object] object to compare with
43
47
  # @return [Boolean]
44
48
  def ==(other)
45
49
  self.class == other.class &&
46
- self.partition == other.partition &&
47
- self.offset == other.offset
50
+ partition == other.partition &&
51
+ offset == other.offset
48
52
  end
49
53
  end
50
54
  end
@@ -9,7 +9,7 @@ module Rdkafka
9
9
  # @param data [Hash{String => nil,Partition}] The topic and partition data or nil to create an empty list
10
10
  #
11
11
  # @return [TopicPartitionList]
12
- def initialize(data=nil)
12
+ def initialize(data = nil)
13
13
  @data = data || {}
14
14
  end
15
15
 
@@ -18,10 +18,10 @@ module Rdkafka
18
18
  def count
19
19
  i = 0
20
20
  @data.each do |_topic, partitions|
21
- if partitions
22
- i += partitions.count
21
+ i += if partitions
22
+ partitions.count
23
23
  else
24
- i+= 1
24
+ 1
25
25
  end
26
26
  end
27
27
  i
@@ -49,15 +49,14 @@ module Rdkafka
49
49
  #
50
50
  # @example Add a topic with all topics up to a count
51
51
  # tpl.add_topic("topic", 9)
52
- #
53
- def add_topic(topic, partitions=nil)
52
+ def add_topic(topic, partitions = nil)
54
53
  if partitions.nil?
55
54
  @data[topic.to_s] = nil
56
55
  else
57
56
  if partitions.is_a? Integer
58
57
  partitions = (0..partitions - 1)
59
58
  end
60
- @data[topic.to_s] = partitions.map { |p| Partition.new(p, nil, 0) }
59
+ @data[topic.to_s] = partitions.map { |p| Partition.new(p, nil, Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR) }
61
60
  end
62
61
  end
63
62
 
@@ -65,11 +64,13 @@ module Rdkafka
65
64
  # Calling this method multiple times for the same topic will overwrite the previous configuraton.
66
65
  #
67
66
  # @param topic [String] The topic's name
68
- # @param partitions_with_offsets [Hash<Integer, Integer>] The topic's partitions and offsets
69
- #
67
+ # @param partitions_with_offsets [Hash{Integer => Integer}, Array<Consumer::Partition>] The topic's
68
+ # partitions and offsets (Hash) or partitions with offsets and metadata (Array)
70
69
  # @return [nil]
71
70
  def add_topic_and_partitions_with_offsets(topic, partitions_with_offsets)
72
- @data[topic.to_s] = partitions_with_offsets.map { |p, o| Partition.new(p, o) }
71
+ @data[topic.to_s] = partitions_with_offsets.map do |p, o|
72
+ p.is_a?(Partition) ? p : Partition.new(p, o)
73
+ end
73
74
  end
74
75
 
75
76
  # Return a `Hash` with the topics as keys and and an array of partition information as the value if present.
@@ -85,8 +86,11 @@ module Rdkafka
85
86
  "<TopicPartitionList: #{to_h}>"
86
87
  end
87
88
 
89
+ # Check equality with another TopicPartitionList
90
+ # @param other [TopicPartitionList] object to compare with
91
+ # @return [Boolean]
88
92
  def ==(other)
89
- self.to_h == other.to_h
93
+ to_h == other.to_h
90
94
  end
91
95
 
92
96
  # Create a new topic partition list based of a native one.
@@ -105,15 +109,15 @@ module Rdkafka
105
109
  native_tpl[:cnt].times do |i|
106
110
  ptr = native_tpl[:elems] + (i * Rdkafka::Bindings::TopicPartition.size)
107
111
  elem = Rdkafka::Bindings::TopicPartition.new(ptr)
108
- if elem[:partition] == -1
112
+ if elem[:partition] == Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
109
113
  data[elem[:topic]] = nil
110
114
  else
111
115
  partitions = data[elem[:topic]] || []
112
116
  offset = if elem[:offset] == Rdkafka::Bindings::RD_KAFKA_OFFSET_INVALID
113
- nil
114
- else
115
- elem[:offset]
116
- end
117
+ nil
118
+ else
119
+ elem[:offset]
120
+ end
117
121
  partition = Partition.new(elem[:partition], offset, elem[:err])
118
122
  partitions.push(partition)
119
123
  data[elem[:topic]] = partitions
@@ -157,7 +161,7 @@ module Rdkafka
157
161
  Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
158
162
  tpl,
159
163
  topic,
160
- -1
164
+ Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
161
165
  )
162
166
  end
163
167
  end