rdkafka 0.22.0.beta1-x86_64-linux-gnu

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. checksums.yaml +7 -0
  2. data/.github/CODEOWNERS +3 -0
  3. data/.github/FUNDING.yml +1 -0
  4. data/.github/workflows/ci_linux_x86_64_gnu.yml +249 -0
  5. data/.github/workflows/ci_linux_x86_64_musl.yml +205 -0
  6. data/.github/workflows/ci_macos_arm64.yml +306 -0
  7. data/.github/workflows/push_linux_x86_64_gnu.yml +64 -0
  8. data/.github/workflows/push_linux_x86_64_musl.yml +77 -0
  9. data/.github/workflows/push_macos_arm64.yml +54 -0
  10. data/.github/workflows/push_ruby.yml +37 -0
  11. data/.github/workflows/verify-action-pins.yml +16 -0
  12. data/.gitignore +14 -0
  13. data/.rspec +2 -0
  14. data/.ruby-gemset +1 -0
  15. data/.ruby-version +1 -0
  16. data/.yardopts +2 -0
  17. data/CHANGELOG.md +247 -0
  18. data/Gemfile +5 -0
  19. data/MIT-LICENSE +22 -0
  20. data/README.md +178 -0
  21. data/Rakefile +96 -0
  22. data/docker-compose.yml +25 -0
  23. data/ext/README.md +19 -0
  24. data/ext/Rakefile +131 -0
  25. data/ext/build_common.sh +361 -0
  26. data/ext/build_linux_x86_64_gnu.sh +306 -0
  27. data/ext/build_linux_x86_64_musl.sh +763 -0
  28. data/ext/build_macos_arm64.sh +550 -0
  29. data/ext/librdkafka.so +0 -0
  30. data/lib/rdkafka/abstract_handle.rb +116 -0
  31. data/lib/rdkafka/admin/acl_binding_result.rb +51 -0
  32. data/lib/rdkafka/admin/config_binding_result.rb +30 -0
  33. data/lib/rdkafka/admin/config_resource_binding_result.rb +18 -0
  34. data/lib/rdkafka/admin/create_acl_handle.rb +28 -0
  35. data/lib/rdkafka/admin/create_acl_report.rb +24 -0
  36. data/lib/rdkafka/admin/create_partitions_handle.rb +27 -0
  37. data/lib/rdkafka/admin/create_partitions_report.rb +6 -0
  38. data/lib/rdkafka/admin/create_topic_handle.rb +29 -0
  39. data/lib/rdkafka/admin/create_topic_report.rb +24 -0
  40. data/lib/rdkafka/admin/delete_acl_handle.rb +30 -0
  41. data/lib/rdkafka/admin/delete_acl_report.rb +23 -0
  42. data/lib/rdkafka/admin/delete_groups_handle.rb +28 -0
  43. data/lib/rdkafka/admin/delete_groups_report.rb +24 -0
  44. data/lib/rdkafka/admin/delete_topic_handle.rb +29 -0
  45. data/lib/rdkafka/admin/delete_topic_report.rb +24 -0
  46. data/lib/rdkafka/admin/describe_acl_handle.rb +30 -0
  47. data/lib/rdkafka/admin/describe_acl_report.rb +24 -0
  48. data/lib/rdkafka/admin/describe_configs_handle.rb +33 -0
  49. data/lib/rdkafka/admin/describe_configs_report.rb +54 -0
  50. data/lib/rdkafka/admin/incremental_alter_configs_handle.rb +33 -0
  51. data/lib/rdkafka/admin/incremental_alter_configs_report.rb +54 -0
  52. data/lib/rdkafka/admin.rb +833 -0
  53. data/lib/rdkafka/bindings.rb +566 -0
  54. data/lib/rdkafka/callbacks.rb +415 -0
  55. data/lib/rdkafka/config.rb +398 -0
  56. data/lib/rdkafka/consumer/headers.rb +79 -0
  57. data/lib/rdkafka/consumer/message.rb +86 -0
  58. data/lib/rdkafka/consumer/partition.rb +51 -0
  59. data/lib/rdkafka/consumer/topic_partition_list.rb +169 -0
  60. data/lib/rdkafka/consumer.rb +653 -0
  61. data/lib/rdkafka/error.rb +101 -0
  62. data/lib/rdkafka/helpers/oauth.rb +58 -0
  63. data/lib/rdkafka/helpers/time.rb +14 -0
  64. data/lib/rdkafka/metadata.rb +115 -0
  65. data/lib/rdkafka/native_kafka.rb +139 -0
  66. data/lib/rdkafka/producer/delivery_handle.rb +40 -0
  67. data/lib/rdkafka/producer/delivery_report.rb +46 -0
  68. data/lib/rdkafka/producer/partitions_count_cache.rb +216 -0
  69. data/lib/rdkafka/producer.rb +430 -0
  70. data/lib/rdkafka/version.rb +7 -0
  71. data/lib/rdkafka.rb +54 -0
  72. data/rdkafka.gemspec +65 -0
  73. data/renovate.json +92 -0
  74. data/spec/rdkafka/abstract_handle_spec.rb +117 -0
  75. data/spec/rdkafka/admin/create_acl_handle_spec.rb +56 -0
  76. data/spec/rdkafka/admin/create_acl_report_spec.rb +18 -0
  77. data/spec/rdkafka/admin/create_topic_handle_spec.rb +52 -0
  78. data/spec/rdkafka/admin/create_topic_report_spec.rb +16 -0
  79. data/spec/rdkafka/admin/delete_acl_handle_spec.rb +85 -0
  80. data/spec/rdkafka/admin/delete_acl_report_spec.rb +72 -0
  81. data/spec/rdkafka/admin/delete_topic_handle_spec.rb +52 -0
  82. data/spec/rdkafka/admin/delete_topic_report_spec.rb +16 -0
  83. data/spec/rdkafka/admin/describe_acl_handle_spec.rb +85 -0
  84. data/spec/rdkafka/admin/describe_acl_report_spec.rb +73 -0
  85. data/spec/rdkafka/admin_spec.rb +770 -0
  86. data/spec/rdkafka/bindings_spec.rb +223 -0
  87. data/spec/rdkafka/callbacks_spec.rb +20 -0
  88. data/spec/rdkafka/config_spec.rb +258 -0
  89. data/spec/rdkafka/consumer/headers_spec.rb +73 -0
  90. data/spec/rdkafka/consumer/message_spec.rb +139 -0
  91. data/spec/rdkafka/consumer/partition_spec.rb +57 -0
  92. data/spec/rdkafka/consumer/topic_partition_list_spec.rb +248 -0
  93. data/spec/rdkafka/consumer_spec.rb +1274 -0
  94. data/spec/rdkafka/error_spec.rb +89 -0
  95. data/spec/rdkafka/metadata_spec.rb +79 -0
  96. data/spec/rdkafka/native_kafka_spec.rb +130 -0
  97. data/spec/rdkafka/producer/delivery_handle_spec.rb +45 -0
  98. data/spec/rdkafka/producer/delivery_report_spec.rb +25 -0
  99. data/spec/rdkafka/producer/partitions_count_cache_spec.rb +359 -0
  100. data/spec/rdkafka/producer_spec.rb +1052 -0
  101. data/spec/spec_helper.rb +195 -0
  102. metadata +276 -0
@@ -0,0 +1,169 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ class Consumer
5
+ # A list of topics with their partition information
6
+ class TopicPartitionList
7
+ # Create a topic partition list.
8
+ #
9
+ # @param data [Hash{String => nil,Partition}] The topic and partition data or nil to create an empty list
10
+ #
11
+ # @return [TopicPartitionList]
12
+ def initialize(data=nil)
13
+ @data = data || {}
14
+ end
15
+
16
+ # Number of items in the list
17
+ # @return [Integer]
18
+ def count
19
+ i = 0
20
+ @data.each do |_topic, partitions|
21
+ if partitions
22
+ i += partitions.count
23
+ else
24
+ i+= 1
25
+ end
26
+ end
27
+ i
28
+ end
29
+
30
+ # Whether this list is empty
31
+ # @return [Boolean]
32
+ def empty?
33
+ @data.empty?
34
+ end
35
+
36
+ # Add a topic with optionally partitions to the list.
37
+ # Calling this method multiple times for the same topic will overwrite the previous configuraton.
38
+ #
39
+ # @param topic [String] The topic's name
40
+ # @param partitions [Array<Integer>, Range<Integer>, Integer] The topic's partitions or partition count
41
+ #
42
+ # @return [nil]
43
+ #
44
+ # @example Add a topic with unassigned partitions
45
+ # tpl.add_topic("topic")
46
+ #
47
+ # @example Add a topic with assigned partitions
48
+ # tpl.add_topic("topic", (0..8))
49
+ #
50
+ # @example Add a topic with all topics up to a count
51
+ # tpl.add_topic("topic", 9)
52
+ #
53
+ def add_topic(topic, partitions=nil)
54
+ if partitions.nil?
55
+ @data[topic.to_s] = nil
56
+ else
57
+ if partitions.is_a? Integer
58
+ partitions = (0..partitions - 1)
59
+ end
60
+ @data[topic.to_s] = partitions.map { |p| Partition.new(p, nil, 0) }
61
+ end
62
+ end
63
+
64
+ # Add a topic with partitions and offsets set to the list
65
+ # Calling this method multiple times for the same topic will overwrite the previous configuraton.
66
+ #
67
+ # @param topic [String] The topic's name
68
+ # @param partitions_with_offsets [Hash<Integer, Integer>] The topic's partitions and offsets
69
+ #
70
+ # @return [nil]
71
+ def add_topic_and_partitions_with_offsets(topic, partitions_with_offsets)
72
+ @data[topic.to_s] = partitions_with_offsets.map { |p, o| Partition.new(p, o) }
73
+ end
74
+
75
+ # Return a `Hash` with the topics as keys and and an array of partition information as the value if present.
76
+ #
77
+ # @return [Hash{String => Array<Partition>,nil}]
78
+ def to_h
79
+ @data
80
+ end
81
+
82
+ # Human readable representation of this list.
83
+ # @return [String]
84
+ def to_s
85
+ "<TopicPartitionList: #{to_h}>"
86
+ end
87
+
88
+ def ==(other)
89
+ self.to_h == other.to_h
90
+ end
91
+
92
+ # Create a new topic partition list based of a native one.
93
+ #
94
+ # @private
95
+ #
96
+ # @param pointer [FFI::Pointer] Optional pointer to an existing native list. Its contents will be copied.
97
+ #
98
+ # @return [TopicPartitionList]
99
+ def self.from_native_tpl(pointer)
100
+ # Data to be moved into the tpl
101
+ data = {}
102
+
103
+ # Create struct and copy its contents
104
+ native_tpl = Rdkafka::Bindings::TopicPartitionList.new(pointer)
105
+ native_tpl[:cnt].times do |i|
106
+ ptr = native_tpl[:elems] + (i * Rdkafka::Bindings::TopicPartition.size)
107
+ elem = Rdkafka::Bindings::TopicPartition.new(ptr)
108
+ if elem[:partition] == -1
109
+ data[elem[:topic]] = nil
110
+ else
111
+ partitions = data[elem[:topic]] || []
112
+ offset = if elem[:offset] == Rdkafka::Bindings::RD_KAFKA_OFFSET_INVALID
113
+ nil
114
+ else
115
+ elem[:offset]
116
+ end
117
+ partition = Partition.new(elem[:partition], offset, elem[:err])
118
+ partitions.push(partition)
119
+ data[elem[:topic]] = partitions
120
+ end
121
+ end
122
+
123
+ # Return the created object
124
+ TopicPartitionList.new(data)
125
+ end
126
+
127
+ # Create a native tpl with the contents of this object added.
128
+ #
129
+ # The pointer will be cleaned by `rd_kafka_topic_partition_list_destroy` when GC releases it.
130
+ #
131
+ # @private
132
+ # @return [FFI::Pointer]
133
+ def to_native_tpl
134
+ tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(count)
135
+
136
+ @data.each do |topic, partitions|
137
+ if partitions
138
+ partitions.each do |p|
139
+ Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
140
+ tpl,
141
+ topic,
142
+ p.partition
143
+ )
144
+
145
+ if p.offset
146
+ offset = p.offset.is_a?(Time) ? p.offset.to_f * 1_000 : p.offset
147
+
148
+ Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(
149
+ tpl,
150
+ topic,
151
+ p.partition,
152
+ offset
153
+ )
154
+ end
155
+ end
156
+ else
157
+ Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
158
+ tpl,
159
+ topic,
160
+ -1
161
+ )
162
+ end
163
+ end
164
+
165
+ tpl
166
+ end
167
+ end
168
+ end
169
+ end