ruby-kafka 0.7.5 → 0.7.6.beta2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +36 -3
- data/CHANGELOG.md +7 -0
- data/README.md +32 -0
- data/lib/kafka.rb +4 -0
- data/lib/kafka/async_producer.rb +27 -9
- data/lib/kafka/broker.rb +1 -1
- data/lib/kafka/broker_pool.rb +1 -1
- data/lib/kafka/client.rb +10 -3
- data/lib/kafka/cluster.rb +1 -1
- data/lib/kafka/connection.rb +4 -1
- data/lib/kafka/connection_builder.rb +1 -1
- data/lib/kafka/consumer.rb +46 -13
- data/lib/kafka/consumer_group.rb +10 -1
- data/lib/kafka/fetch_operation.rb +1 -1
- data/lib/kafka/fetched_batch_generator.rb +1 -1
- data/lib/kafka/fetched_offset_resolver.rb +1 -1
- data/lib/kafka/fetcher.rb +5 -2
- data/lib/kafka/offset_manager.rb +1 -1
- data/lib/kafka/produce_operation.rb +1 -1
- data/lib/kafka/producer.rb +13 -7
- data/lib/kafka/protocol/record_batch.rb +3 -1
- data/lib/kafka/protocol/sasl_handshake_request.rb +1 -1
- data/lib/kafka/sasl/gssapi.rb +1 -1
- data/lib/kafka/sasl/oauth.rb +64 -0
- data/lib/kafka/sasl/plain.rb +1 -1
- data/lib/kafka/sasl/scram.rb +1 -1
- data/lib/kafka/sasl_authenticator.rb +10 -3
- data/lib/kafka/tagged_logger.rb +72 -0
- data/lib/kafka/transaction_manager.rb +1 -1
- data/lib/kafka/transaction_state_machine.rb +1 -1
- data/lib/kafka/version.rb +1 -1
- metadata +6 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b14f3fd396d495fc5c240cd5451b7806154b0fda6dbae72764cfa8087a4b778d
|
4
|
+
data.tar.gz: e283a412d4bcdfd7b6ac8f8c0ba7d6ef9b1bbc94163796ca23cbe6d884710076
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1bbe81c129b203d3a4f7f64853cdd21ab99b602844fc426726a74878b40bf9266a9ed14a77b05bab5e500118aa35ea8a0e39bdbf466018d746e744580a525c66
|
7
|
+
data.tar.gz: 359e818723d15663dc6d1de2b83b70aad4f27bd293b0bb95426be16a440f9c9c48cd58d53a754d8e62d541b8043ef8109ab7549173e2232732ebe94673acf301
|
data/.circleci/config.yml
CHANGED
@@ -113,21 +113,53 @@ jobs:
|
|
113
113
|
environment:
|
114
114
|
LOG_LEVEL: DEBUG
|
115
115
|
- image: wurstmeister/zookeeper
|
116
|
-
- image: wurstmeister/kafka:2.11-2.0.
|
116
|
+
- image: wurstmeister/kafka:2.11-2.0.1
|
117
117
|
environment:
|
118
118
|
KAFKA_ADVERTISED_HOST_NAME: localhost
|
119
119
|
KAFKA_ADVERTISED_PORT: 9092
|
120
120
|
KAFKA_PORT: 9092
|
121
121
|
KAFKA_ZOOKEEPER_CONNECT: localhost:2181
|
122
122
|
KAFKA_DELETE_TOPIC_ENABLE: true
|
123
|
-
- image: wurstmeister/kafka:2.11-2.0.
|
123
|
+
- image: wurstmeister/kafka:2.11-2.0.1
|
124
124
|
environment:
|
125
125
|
KAFKA_ADVERTISED_HOST_NAME: localhost
|
126
126
|
KAFKA_ADVERTISED_PORT: 9093
|
127
127
|
KAFKA_PORT: 9093
|
128
128
|
KAFKA_ZOOKEEPER_CONNECT: localhost:2181
|
129
129
|
KAFKA_DELETE_TOPIC_ENABLE: true
|
130
|
-
- image: wurstmeister/kafka:2.11-2.0.
|
130
|
+
- image: wurstmeister/kafka:2.11-2.0.1
|
131
|
+
environment:
|
132
|
+
KAFKA_ADVERTISED_HOST_NAME: localhost
|
133
|
+
KAFKA_ADVERTISED_PORT: 9094
|
134
|
+
KAFKA_PORT: 9094
|
135
|
+
KAFKA_ZOOKEEPER_CONNECT: localhost:2181
|
136
|
+
KAFKA_DELETE_TOPIC_ENABLE: true
|
137
|
+
steps:
|
138
|
+
- checkout
|
139
|
+
- run: bundle install --path vendor/bundle
|
140
|
+
- run: bundle exec rspec --profile --tag functional spec/functional
|
141
|
+
|
142
|
+
kafka-2.1:
|
143
|
+
docker:
|
144
|
+
- image: circleci/ruby:2.5.1-node
|
145
|
+
environment:
|
146
|
+
LOG_LEVEL: DEBUG
|
147
|
+
- image: wurstmeister/zookeeper
|
148
|
+
- image: wurstmeister/kafka:2.12-2.1.0
|
149
|
+
environment:
|
150
|
+
KAFKA_ADVERTISED_HOST_NAME: localhost
|
151
|
+
KAFKA_ADVERTISED_PORT: 9092
|
152
|
+
KAFKA_PORT: 9092
|
153
|
+
KAFKA_ZOOKEEPER_CONNECT: localhost:2181
|
154
|
+
KAFKA_DELETE_TOPIC_ENABLE: true
|
155
|
+
- image: wurstmeister/kafka:2.12-2.1.0
|
156
|
+
environment:
|
157
|
+
KAFKA_ADVERTISED_HOST_NAME: localhost
|
158
|
+
KAFKA_ADVERTISED_PORT: 9093
|
159
|
+
KAFKA_PORT: 9093
|
160
|
+
KAFKA_ZOOKEEPER_CONNECT: localhost:2181
|
161
|
+
KAFKA_DELETE_TOPIC_ENABLE: true
|
162
|
+
- image: wurstmeister/kafka:2.12-2.1.0
|
131
163
|
environment:
|
132
164
|
KAFKA_ADVERTISED_HOST_NAME: localhost
|
133
165
|
KAFKA_ADVERTISED_PORT: 9094
|
@@ -148,3 +180,4 @@ workflows:
|
|
148
180
|
- kafka-1.0.0
|
149
181
|
- kafka-1.1
|
150
182
|
- kafka-2.0
|
183
|
+
- kafka-2.1
|
data/CHANGELOG.md
CHANGED
@@ -4,6 +4,13 @@ Changes and additions to the library will be listed here.
|
|
4
4
|
|
5
5
|
## Unreleased
|
6
6
|
|
7
|
+
## 0.7.6
|
8
|
+
- Introduce regex matching in `Consumer#subscribe` (#700)
|
9
|
+
- Only rejoin group on error if we're not in shutdown mode (#711)
|
10
|
+
- Use `maxTimestamp` for `logAppendTime` timestamps (#706)
|
11
|
+
- Async producer limit number of retries (#708)
|
12
|
+
- Support SASL OAuthBearer Authentication (#710)
|
13
|
+
|
7
14
|
## 0.7.5
|
8
15
|
- Distribute partitions across consumer groups when there are few partitions per topic (#681)
|
9
16
|
- Fix an issue where a consumer would fail to fetch any messages (#689)
|
data/README.md
CHANGED
@@ -98,6 +98,16 @@ Or install it yourself as:
|
|
98
98
|
<td>Limited support</td>
|
99
99
|
<td>Limited support</td>
|
100
100
|
</tr>
|
101
|
+
<tr>
|
102
|
+
<th>Kafka 2.0</th>
|
103
|
+
<td>Limited support</td>
|
104
|
+
<td>Limited support</td>
|
105
|
+
</tr>
|
106
|
+
<tr>
|
107
|
+
<th>Kafka 2.1</th>
|
108
|
+
<td>Limited support</td>
|
109
|
+
<td>Limited support</td>
|
110
|
+
</tr>
|
101
111
|
</table>
|
102
112
|
|
103
113
|
This library is targeting Kafka 0.9 with the v0.4.x series and Kafka 0.10 with the v0.5.x series. There's limited support for Kafka 0.8, and things should work with Kafka 0.11, although there may be performance issues due to changes in the protocol.
|
@@ -107,6 +117,8 @@ This library is targeting Kafka 0.9 with the v0.4.x series and Kafka 0.10 with t
|
|
107
117
|
- **Kafka 0.10:** Full support for the Producer and Consumer API in ruby-kafka v0.5.x. Note that you _must_ run version 0.10.1 or higher of Kafka due to limitations in 0.10.0.
|
108
118
|
- **Kafka 0.11:** Full support for Producer API, limited support for Consumer API in ruby-kafka v0.7.x. New features in 0.11.x includes new Record Batch format, idempotent and transactional production. The missing feature is dirty reading of Consumer API.
|
109
119
|
- **Kafka 1.0:** Everything that works with Kafka 0.11 should still work, but so far no features specific to Kafka 1.0 have been added.
|
120
|
+
- **Kafka 2.0:** Everything that works with Kafka 1.0 should still work, but so far no features specific to Kafka 2.0 have been added.
|
121
|
+
- **Kafka 2.1:** Everything that works with Kafka 2.0 should still work, but so far no features specific to Kafka 2.1 have been added.
|
110
122
|
|
111
123
|
This library requires Ruby 2.1 or higher.
|
112
124
|
|
@@ -976,6 +988,26 @@ kafka = Kafka.new(
|
|
976
988
|
)
|
977
989
|
```
|
978
990
|
|
991
|
+
##### OAUTHBEARER
|
992
|
+
This mechanism is supported in kafka >= 2.0.0 as of [KIP-255](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=75968876)
|
993
|
+
|
994
|
+
In order to authenticate using OAUTHBEARER, you must set the client with an instance of a class that implements a `token` method (the interface is described in [Kafka::Sasl::OAuth](lib/kafka/sasl/oauth.rb)) which returns an ID/Access token.
|
995
|
+
|
996
|
+
Optionally, the client may implement an `extensions` method that returns a map of key-value pairs. These can be sent with the SASL/OAUTHBEARER initial client response. This is only supported in kafka >= 2.1.0.
|
997
|
+
|
998
|
+
```ruby
|
999
|
+
class TokenProvider
|
1000
|
+
def token
|
1001
|
+
"some_id_token"
|
1002
|
+
end
|
1003
|
+
end
|
1004
|
+
# ...
|
1005
|
+
client = Kafka.new(
|
1006
|
+
["kafka1:9092"],
|
1007
|
+
sasl_oauth_token_provider: TokenProvider.new
|
1008
|
+
)
|
1009
|
+
```
|
1010
|
+
|
979
1011
|
### Topic management
|
980
1012
|
|
981
1013
|
In addition to producing and consuming messages, ruby-kafka supports managing Kafka topics and their configurations. See [the Kafka documentation](https://kafka.apache.org/documentation/#topicconfigs) for a full list of topic configuration keys.
|
data/lib/kafka.rb
CHANGED
@@ -351,6 +351,10 @@ module Kafka
|
|
351
351
|
class FailedScramAuthentication < SaslScramError
|
352
352
|
end
|
353
353
|
|
354
|
+
# The Token Provider object used for SASL OAuthBearer does not implement the method `token`
|
355
|
+
class TokenMethodNotImplementedError < Error
|
356
|
+
end
|
357
|
+
|
354
358
|
# Initializes a new Kafka client.
|
355
359
|
#
|
356
360
|
# @see Client#initialize
|
data/lib/kafka/async_producer.rb
CHANGED
@@ -72,7 +72,7 @@ module Kafka
|
|
72
72
|
# @param delivery_interval [Integer] if greater than zero, the number of
|
73
73
|
# seconds between automatic message deliveries.
|
74
74
|
#
|
75
|
-
def initialize(sync_producer:, max_queue_size: 1000, delivery_threshold: 0, delivery_interval: 0, instrumenter:, logger:)
|
75
|
+
def initialize(sync_producer:, max_queue_size: 1000, delivery_threshold: 0, delivery_interval: 0, max_retries: -1, retry_backoff: 0, instrumenter:, logger:)
|
76
76
|
raise ArgumentError unless max_queue_size > 0
|
77
77
|
raise ArgumentError unless delivery_threshold >= 0
|
78
78
|
raise ArgumentError unless delivery_interval >= 0
|
@@ -80,14 +80,16 @@ module Kafka
|
|
80
80
|
@queue = Queue.new
|
81
81
|
@max_queue_size = max_queue_size
|
82
82
|
@instrumenter = instrumenter
|
83
|
-
@logger = logger
|
83
|
+
@logger = TaggedLogger.new(logger)
|
84
84
|
|
85
85
|
@worker = Worker.new(
|
86
86
|
queue: @queue,
|
87
87
|
producer: sync_producer,
|
88
88
|
delivery_threshold: delivery_threshold,
|
89
|
+
max_retries: max_retries,
|
90
|
+
retry_backoff: retry_backoff,
|
89
91
|
instrumenter: instrumenter,
|
90
|
-
logger: logger
|
92
|
+
logger: logger
|
91
93
|
)
|
92
94
|
|
93
95
|
# The timer will no-op if the delivery interval is zero.
|
@@ -184,15 +186,18 @@ module Kafka
|
|
184
186
|
end
|
185
187
|
|
186
188
|
class Worker
|
187
|
-
def initialize(queue:, producer:, delivery_threshold:, instrumenter:, logger:)
|
189
|
+
def initialize(queue:, producer:, delivery_threshold:, max_retries: -1, retry_backoff: 0, instrumenter:, logger:)
|
188
190
|
@queue = queue
|
189
191
|
@producer = producer
|
190
192
|
@delivery_threshold = delivery_threshold
|
193
|
+
@max_retries = max_retries
|
194
|
+
@retry_backoff = retry_backoff
|
191
195
|
@instrumenter = instrumenter
|
192
|
-
@logger = logger
|
196
|
+
@logger = TaggedLogger.new(logger)
|
193
197
|
end
|
194
198
|
|
195
199
|
def run
|
200
|
+
@logger.push_tags(@producer.to_s)
|
196
201
|
@logger.info "Starting async producer in the background..."
|
197
202
|
|
198
203
|
loop do
|
@@ -233,15 +238,28 @@ module Kafka
|
|
233
238
|
@logger.error "Async producer crashed!"
|
234
239
|
ensure
|
235
240
|
@producer.shutdown
|
241
|
+
@logger.pop_tags
|
236
242
|
end
|
237
243
|
|
238
244
|
private
|
239
245
|
|
240
246
|
def produce(*args)
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
247
|
+
retries = 0
|
248
|
+
begin
|
249
|
+
@producer.produce(*args)
|
250
|
+
rescue BufferOverflow => e
|
251
|
+
deliver_messages
|
252
|
+
if @max_retries == -1
|
253
|
+
retry
|
254
|
+
elsif retries < @max_retries
|
255
|
+
retries += 1
|
256
|
+
sleep @retry_backoff**retries
|
257
|
+
retry
|
258
|
+
else
|
259
|
+
@logger.error("Failed to asynchronously produce messages due to BufferOverflow")
|
260
|
+
@instrumenter.instrument("error.async_producer", { error: e })
|
261
|
+
end
|
262
|
+
end
|
245
263
|
end
|
246
264
|
|
247
265
|
def deliver_messages
|
data/lib/kafka/broker.rb
CHANGED
data/lib/kafka/broker_pool.rb
CHANGED
data/lib/kafka/client.rb
CHANGED
@@ -14,6 +14,7 @@ require "kafka/fetch_operation"
|
|
14
14
|
require "kafka/connection_builder"
|
15
15
|
require "kafka/instrumenter"
|
16
16
|
require "kafka/sasl_authenticator"
|
17
|
+
require "kafka/tagged_logger"
|
17
18
|
|
18
19
|
module Kafka
|
19
20
|
class Client
|
@@ -61,14 +62,17 @@ module Kafka
|
|
61
62
|
#
|
62
63
|
# @param sasl_over_ssl [Boolean] whether to enforce SSL with SASL
|
63
64
|
#
|
65
|
+
# @param sasl_oauth_token_provider [Object, nil] OAuthBearer Token Provider instance that
|
66
|
+
# implements method token. See {Sasl::OAuth#initialize}
|
67
|
+
#
|
64
68
|
# @return [Client]
|
65
69
|
def initialize(seed_brokers:, client_id: "ruby-kafka", logger: nil, connect_timeout: nil, socket_timeout: nil,
|
66
70
|
ssl_ca_cert_file_path: nil, ssl_ca_cert: nil, ssl_client_cert: nil, ssl_client_cert_key: nil,
|
67
71
|
ssl_client_cert_key_password: nil, ssl_client_cert_chain: nil, sasl_gssapi_principal: nil,
|
68
72
|
sasl_gssapi_keytab: nil, sasl_plain_authzid: '', sasl_plain_username: nil, sasl_plain_password: nil,
|
69
73
|
sasl_scram_username: nil, sasl_scram_password: nil, sasl_scram_mechanism: nil,
|
70
|
-
sasl_over_ssl: true, ssl_ca_certs_from_system: false)
|
71
|
-
@logger =
|
74
|
+
sasl_over_ssl: true, ssl_ca_certs_from_system: false, sasl_oauth_token_provider: nil)
|
75
|
+
@logger = TaggedLogger.new(logger)
|
72
76
|
@instrumenter = Instrumenter.new(client_id: client_id)
|
73
77
|
@seed_brokers = normalize_seed_brokers(seed_brokers)
|
74
78
|
|
@@ -91,6 +95,7 @@ module Kafka
|
|
91
95
|
sasl_scram_username: sasl_scram_username,
|
92
96
|
sasl_scram_password: sasl_scram_password,
|
93
97
|
sasl_scram_mechanism: sasl_scram_mechanism,
|
98
|
+
sasl_oauth_token_provider: sasl_oauth_token_provider,
|
94
99
|
logger: @logger
|
95
100
|
)
|
96
101
|
|
@@ -295,7 +300,7 @@ module Kafka
|
|
295
300
|
#
|
296
301
|
# @see AsyncProducer
|
297
302
|
# @return [AsyncProducer]
|
298
|
-
def async_producer(delivery_interval: 0, delivery_threshold: 0, max_queue_size: 1000, **options)
|
303
|
+
def async_producer(delivery_interval: 0, delivery_threshold: 0, max_queue_size: 1000, max_retries: -1, retry_backoff: 0, **options)
|
299
304
|
sync_producer = producer(**options)
|
300
305
|
|
301
306
|
AsyncProducer.new(
|
@@ -303,6 +308,8 @@ module Kafka
|
|
303
308
|
delivery_interval: delivery_interval,
|
304
309
|
delivery_threshold: delivery_threshold,
|
305
310
|
max_queue_size: max_queue_size,
|
311
|
+
max_retries: max_retries,
|
312
|
+
retry_backoff: retry_backoff,
|
306
313
|
instrumenter: @instrumenter,
|
307
314
|
logger: @logger,
|
308
315
|
)
|
data/lib/kafka/cluster.rb
CHANGED
data/lib/kafka/connection.rb
CHANGED
@@ -52,7 +52,7 @@ module Kafka
|
|
52
52
|
# @return [Connection] a new connection.
|
53
53
|
def initialize(host:, port:, client_id:, logger:, instrumenter:, connect_timeout: nil, socket_timeout: nil, ssl_context: nil)
|
54
54
|
@host, @port, @client_id = host, port, client_id
|
55
|
-
@logger = logger
|
55
|
+
@logger = TaggedLogger.new(logger)
|
56
56
|
@instrumenter = instrumenter
|
57
57
|
|
58
58
|
@connect_timeout = connect_timeout || CONNECT_TIMEOUT
|
@@ -93,6 +93,7 @@ module Kafka
|
|
93
93
|
|
94
94
|
raise IdleConnection if idle?
|
95
95
|
|
96
|
+
@logger.push_tags(api_name)
|
96
97
|
@instrumenter.instrument("request.connection", notification) do
|
97
98
|
open unless open?
|
98
99
|
|
@@ -113,6 +114,8 @@ module Kafka
|
|
113
114
|
close
|
114
115
|
|
115
116
|
raise ConnectionError, "Connection error #{e.class}: #{e}"
|
117
|
+
ensure
|
118
|
+
@logger.pop_tags
|
116
119
|
end
|
117
120
|
|
118
121
|
private
|
@@ -4,7 +4,7 @@ module Kafka
|
|
4
4
|
class ConnectionBuilder
|
5
5
|
def initialize(client_id:, logger:, instrumenter:, connect_timeout:, socket_timeout:, ssl_context:, sasl_authenticator:)
|
6
6
|
@client_id = client_id
|
7
|
-
@logger = logger
|
7
|
+
@logger = TaggedLogger.new(logger)
|
8
8
|
@instrumenter = instrumenter
|
9
9
|
@connect_timeout = connect_timeout
|
10
10
|
@socket_timeout = socket_timeout
|
data/lib/kafka/consumer.rb
CHANGED
@@ -46,7 +46,7 @@ module Kafka
|
|
46
46
|
|
47
47
|
def initialize(cluster:, logger:, instrumenter:, group:, fetcher:, offset_manager:, session_timeout:, heartbeat:)
|
48
48
|
@cluster = cluster
|
49
|
-
@logger = logger
|
49
|
+
@logger = TaggedLogger.new(logger)
|
50
50
|
@instrumenter = instrumenter
|
51
51
|
@group = group
|
52
52
|
@offset_manager = offset_manager
|
@@ -82,7 +82,8 @@ module Kafka
|
|
82
82
|
# messages to be written. In the former case, set `start_from_beginning`
|
83
83
|
# to true (the default); in the latter, set it to false.
|
84
84
|
#
|
85
|
-
# @param
|
85
|
+
# @param topic_or_regex [String, Regexp] subscribe to single topic with a string
|
86
|
+
# or multiple topics matching a regex.
|
86
87
|
# @param default_offset [Symbol] whether to start from the beginning or the
|
87
88
|
# end of the topic's partitions. Deprecated.
|
88
89
|
# @param start_from_beginning [Boolean] whether to start from the beginning
|
@@ -93,12 +94,16 @@ module Kafka
|
|
93
94
|
# @param max_bytes_per_partition [Integer] the maximum amount of data fetched
|
94
95
|
# from a single partition at a time.
|
95
96
|
# @return [nil]
|
96
|
-
def subscribe(
|
97
|
+
def subscribe(topic_or_regex, default_offset: nil, start_from_beginning: true, max_bytes_per_partition: 1048576)
|
97
98
|
default_offset ||= start_from_beginning ? :earliest : :latest
|
98
99
|
|
99
|
-
|
100
|
-
|
101
|
-
|
100
|
+
if topic_or_regex.is_a?(Regexp)
|
101
|
+
cluster_topics.select { |topic| topic =~ topic_or_regex }.each do |topic|
|
102
|
+
subscribe_to_topic(topic, default_offset, start_from_beginning, max_bytes_per_partition)
|
103
|
+
end
|
104
|
+
else
|
105
|
+
subscribe_to_topic(topic_or_regex, default_offset, start_from_beginning, max_bytes_per_partition)
|
106
|
+
end
|
102
107
|
|
103
108
|
nil
|
104
109
|
end
|
@@ -241,7 +246,7 @@ module Kafka
|
|
241
246
|
|
242
247
|
trigger_heartbeat
|
243
248
|
|
244
|
-
return if
|
249
|
+
return if shutting_down?
|
245
250
|
end
|
246
251
|
|
247
252
|
# We've successfully processed a batch from the partition, so we can clear
|
@@ -336,7 +341,7 @@ module Kafka
|
|
336
341
|
|
337
342
|
trigger_heartbeat
|
338
343
|
|
339
|
-
return if
|
344
|
+
return if shutting_down?
|
340
345
|
end
|
341
346
|
|
342
347
|
# We may not have received any messages, but it's still a good idea to
|
@@ -386,22 +391,23 @@ module Kafka
|
|
386
391
|
|
387
392
|
def consumer_loop
|
388
393
|
@running = true
|
394
|
+
@logger.push_tags(@group.to_s)
|
389
395
|
|
390
396
|
@fetcher.start
|
391
397
|
|
392
|
-
while
|
398
|
+
while running?
|
393
399
|
begin
|
394
400
|
@instrumenter.instrument("loop.consumer") do
|
395
401
|
yield
|
396
402
|
end
|
397
403
|
rescue HeartbeatError
|
398
404
|
make_final_offsets_commit!
|
399
|
-
join_group
|
405
|
+
join_group if running?
|
400
406
|
rescue OffsetCommitError
|
401
|
-
join_group
|
407
|
+
join_group if running?
|
402
408
|
rescue RebalanceInProgress
|
403
409
|
@logger.warn "Group rebalance in progress, re-joining..."
|
404
|
-
join_group
|
410
|
+
join_group if running?
|
405
411
|
rescue FetchError, NotLeaderForPartition, UnknownTopicOrPartition
|
406
412
|
@cluster.mark_as_stale!
|
407
413
|
rescue LeaderNotAvailable => e
|
@@ -424,6 +430,7 @@ module Kafka
|
|
424
430
|
make_final_offsets_commit!
|
425
431
|
@group.leave rescue nil
|
426
432
|
@running = false
|
433
|
+
@logger.pop_tags
|
427
434
|
end
|
428
435
|
|
429
436
|
def make_final_offsets_commit!(attempts = 3)
|
@@ -505,7 +512,7 @@ module Kafka
|
|
505
512
|
|
506
513
|
def fetch_batches
|
507
514
|
# Return early if the consumer has been stopped.
|
508
|
-
return [] if
|
515
|
+
return [] if shutting_down?
|
509
516
|
|
510
517
|
join_group unless @group.member?
|
511
518
|
|
@@ -545,6 +552,14 @@ module Kafka
|
|
545
552
|
@pauses[topic][partition]
|
546
553
|
end
|
547
554
|
|
555
|
+
def running?
|
556
|
+
@running
|
557
|
+
end
|
558
|
+
|
559
|
+
def shutting_down?
|
560
|
+
!running?
|
561
|
+
end
|
562
|
+
|
548
563
|
def clear_current_offsets(excluding: {})
|
549
564
|
@current_offsets.each do |topic, partitions|
|
550
565
|
partitions.keep_if do |partition, _|
|
@@ -552,5 +567,23 @@ module Kafka
|
|
552
567
|
end
|
553
568
|
end
|
554
569
|
end
|
570
|
+
|
571
|
+
def subscribe_to_topic(topic, default_offset, start_from_beginning, max_bytes_per_partition)
|
572
|
+
@group.subscribe(topic)
|
573
|
+
@offset_manager.set_default_offset(topic, default_offset)
|
574
|
+
@fetcher.subscribe(topic, max_bytes_per_partition: max_bytes_per_partition)
|
575
|
+
end
|
576
|
+
|
577
|
+
def cluster_topics
|
578
|
+
attempts = 0
|
579
|
+
begin
|
580
|
+
attempts += 1
|
581
|
+
@cluster.list_topics
|
582
|
+
rescue Kafka::ConnectionError
|
583
|
+
@cluster.mark_as_stale!
|
584
|
+
retry unless attempts > 1
|
585
|
+
raise
|
586
|
+
end
|
587
|
+
end
|
555
588
|
end
|
556
589
|
end
|
data/lib/kafka/consumer_group.rb
CHANGED
@@ -9,7 +9,7 @@ module Kafka
|
|
9
9
|
|
10
10
|
def initialize(cluster:, logger:, group_id:, session_timeout:, retention_time:, instrumenter:)
|
11
11
|
@cluster = cluster
|
12
|
-
@logger = logger
|
12
|
+
@logger = TaggedLogger.new(logger)
|
13
13
|
@group_id = group_id
|
14
14
|
@session_timeout = session_timeout
|
15
15
|
@instrumenter = instrumenter
|
@@ -122,6 +122,15 @@ module Kafka
|
|
122
122
|
retry
|
123
123
|
end
|
124
124
|
|
125
|
+
def to_s
|
126
|
+
"[#{@group_id}] {" + assigned_partitions.map { |topic, partitions|
|
127
|
+
partition_str = partitions.size > 5 ?
|
128
|
+
"#{partitions[0..4].join(', ')}..." :
|
129
|
+
partitions.join(', ')
|
130
|
+
"#{topic}: #{partition_str}"
|
131
|
+
}.join('; ') + '}:'
|
132
|
+
end
|
133
|
+
|
125
134
|
private
|
126
135
|
|
127
136
|
def join_group
|
@@ -23,7 +23,7 @@ module Kafka
|
|
23
23
|
class FetchOperation
|
24
24
|
def initialize(cluster:, logger:, min_bytes: 1, max_bytes: 10485760, max_wait_time: 5)
|
25
25
|
@cluster = cluster
|
26
|
-
@logger = logger
|
26
|
+
@logger = TaggedLogger.new(logger)
|
27
27
|
@min_bytes = min_bytes
|
28
28
|
@max_bytes = max_bytes
|
29
29
|
@max_wait_time = max_wait_time
|
data/lib/kafka/fetcher.rb
CHANGED
@@ -8,7 +8,7 @@ module Kafka
|
|
8
8
|
|
9
9
|
def initialize(cluster:, logger:, instrumenter:, max_queue_size:, group:)
|
10
10
|
@cluster = cluster
|
11
|
-
@logger = logger
|
11
|
+
@logger = TaggedLogger.new(logger)
|
12
12
|
@instrumenter = instrumenter
|
13
13
|
@max_queue_size = max_queue_size
|
14
14
|
@group = group
|
@@ -55,7 +55,7 @@ module Kafka
|
|
55
55
|
while @running
|
56
56
|
loop
|
57
57
|
end
|
58
|
-
@logger.info "Fetcher thread exited."
|
58
|
+
@logger.info "#{@group} Fetcher thread exited."
|
59
59
|
end
|
60
60
|
@thread.abort_on_exception = true
|
61
61
|
end
|
@@ -94,6 +94,7 @@ module Kafka
|
|
94
94
|
attr_reader :current_reset_counter
|
95
95
|
|
96
96
|
def loop
|
97
|
+
@logger.push_tags(@group.to_s)
|
97
98
|
@instrumenter.instrument("loop.fetcher", {
|
98
99
|
queue_size: @queue.size,
|
99
100
|
})
|
@@ -112,6 +113,8 @@ module Kafka
|
|
112
113
|
@logger.warn "Reached max fetcher queue size (#{@max_queue_size}), sleeping 1s"
|
113
114
|
sleep 1
|
114
115
|
end
|
116
|
+
ensure
|
117
|
+
@logger.pop_tags
|
115
118
|
end
|
116
119
|
|
117
120
|
def handle_configure(min_bytes, max_bytes, max_wait_time)
|
data/lib/kafka/offset_manager.rb
CHANGED
data/lib/kafka/producer.rb
CHANGED
@@ -130,7 +130,7 @@ module Kafka
|
|
130
130
|
def initialize(cluster:, transaction_manager:, logger:, instrumenter:, compressor:, ack_timeout:, required_acks:, max_retries:, retry_backoff:, max_buffer_size:, max_buffer_bytesize:)
|
131
131
|
@cluster = cluster
|
132
132
|
@transaction_manager = transaction_manager
|
133
|
-
@logger = logger
|
133
|
+
@logger = TaggedLogger.new(logger)
|
134
134
|
@instrumenter = instrumenter
|
135
135
|
@required_acks = required_acks == :all ? -1 : required_acks
|
136
136
|
@ack_timeout = ack_timeout
|
@@ -150,6 +150,10 @@ module Kafka
|
|
150
150
|
@pending_message_queue = PendingMessageQueue.new
|
151
151
|
end
|
152
152
|
|
153
|
+
def to_s
|
154
|
+
"Producer #{@target_topics.to_a.join(', ')}"
|
155
|
+
end
|
156
|
+
|
153
157
|
# Produces a message to the specified topic. Note that messages are buffered in
|
154
158
|
# the producer until {#deliver_messages} is called.
|
155
159
|
#
|
@@ -205,7 +209,7 @@ module Kafka
|
|
205
209
|
# If the producer is in transactional mode, all the message production
|
206
210
|
# must be used when the producer is currently in transaction
|
207
211
|
if @transaction_manager.transactional? && !@transaction_manager.in_transaction?
|
208
|
-
raise
|
212
|
+
raise "Cannot produce to #{topic}: You must trigger begin_transaction before producing messages"
|
209
213
|
end
|
210
214
|
|
211
215
|
@target_topics.add(topic)
|
@@ -391,11 +395,11 @@ module Kafka
|
|
391
395
|
if buffer_size.zero?
|
392
396
|
break
|
393
397
|
elsif attempt <= @max_retries
|
394
|
-
@logger.warn "Failed to send all messages; attempting retry #{attempt} of #{@max_retries} after #{@retry_backoff}s"
|
398
|
+
@logger.warn "Failed to send all messages to #{pretty_partitions}; attempting retry #{attempt} of #{@max_retries} after #{@retry_backoff}s"
|
395
399
|
|
396
400
|
sleep @retry_backoff
|
397
401
|
else
|
398
|
-
@logger.error "Failed to send all messages; keeping remaining messages in buffer"
|
402
|
+
@logger.error "Failed to send all messages to #{pretty_partitions}; keeping remaining messages in buffer"
|
399
403
|
break
|
400
404
|
end
|
401
405
|
end
|
@@ -407,12 +411,14 @@ module Kafka
|
|
407
411
|
end
|
408
412
|
|
409
413
|
unless @buffer.empty?
|
410
|
-
|
411
|
-
|
412
|
-
raise DeliveryFailed.new("Failed to send messages to #{partitions}", buffer_messages)
|
414
|
+
raise DeliveryFailed.new("Failed to send messages to #{pretty_partitions}", buffer_messages)
|
413
415
|
end
|
414
416
|
end
|
415
417
|
|
418
|
+
def pretty_partitions
|
419
|
+
@buffer.map {|topic, partition, _| "#{topic}/#{partition}" }.join(", ")
|
420
|
+
end
|
421
|
+
|
416
422
|
def assign_partitions!
|
417
423
|
failed_messages = []
|
418
424
|
topics_with_failures = Set.new
|
@@ -11,6 +11,7 @@ module Kafka
|
|
11
11
|
CODEC_ID_MASK = 0b00000111
|
12
12
|
IN_TRANSACTION_MASK = 0b00010000
|
13
13
|
IS_CONTROL_BATCH_MASK = 0b00100000
|
14
|
+
TIMESTAMP_TYPE_MASK = 0b001000
|
14
15
|
|
15
16
|
attr_reader :records, :first_offset, :first_timestamp, :partition_leader_epoch, :in_transaction, :is_control_batch, :last_offset_delta, :max_timestamp, :producer_id, :producer_epoch, :first_sequence
|
16
17
|
|
@@ -163,6 +164,7 @@ module Kafka
|
|
163
164
|
codec_id = attributes & CODEC_ID_MASK
|
164
165
|
in_transaction = (attributes & IN_TRANSACTION_MASK) > 0
|
165
166
|
is_control_batch = (attributes & IS_CONTROL_BATCH_MASK) > 0
|
167
|
+
log_append_time = (attributes & TIMESTAMP_TYPE_MASK) != 0
|
166
168
|
|
167
169
|
last_offset_delta = record_batch_decoder.int32
|
168
170
|
first_timestamp = Time.at(record_batch_decoder.int64 / 1000)
|
@@ -186,7 +188,7 @@ module Kafka
|
|
186
188
|
until records_array_decoder.eof?
|
187
189
|
record = Record.decode(records_array_decoder)
|
188
190
|
record.offset = first_offset + record.offset_delta
|
189
|
-
record.create_time = first_timestamp + record.timestamp_delta
|
191
|
+
record.create_time = log_append_time && max_timestamp ? max_timestamp : first_timestamp + record.timestamp_delta
|
190
192
|
records_array << record
|
191
193
|
end
|
192
194
|
|
@@ -8,7 +8,7 @@ module Kafka
|
|
8
8
|
|
9
9
|
class SaslHandshakeRequest
|
10
10
|
|
11
|
-
SUPPORTED_MECHANISMS = %w(GSSAPI PLAIN SCRAM-SHA-256 SCRAM-SHA-512)
|
11
|
+
SUPPORTED_MECHANISMS = %w(GSSAPI PLAIN SCRAM-SHA-256 SCRAM-SHA-512 OAUTHBEARER)
|
12
12
|
|
13
13
|
def initialize(mechanism)
|
14
14
|
unless SUPPORTED_MECHANISMS.include?(mechanism)
|
data/lib/kafka/sasl/gssapi.rb
CHANGED
@@ -0,0 +1,64 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Kafka
|
4
|
+
module Sasl
|
5
|
+
class OAuth
|
6
|
+
OAUTH_IDENT = "OAUTHBEARER"
|
7
|
+
|
8
|
+
# token_provider: THE FOLLOWING INTERFACE MUST BE FULFILLED:
|
9
|
+
#
|
10
|
+
# [REQUIRED] TokenProvider#token - Returns an ID/Access Token to be sent to the Kafka client.
|
11
|
+
# The implementation should ensure token reuse so that multiple calls at connect time do not
|
12
|
+
# create multiple tokens. The implementation should also periodically refresh the token in
|
13
|
+
# order to guarantee that each call returns an unexpired token. A timeout error should
|
14
|
+
# be returned after a short period of inactivity so that the broker can log debugging
|
15
|
+
# info and retry.
|
16
|
+
#
|
17
|
+
# [OPTIONAL] TokenProvider#extensions - Returns a map of key-value pairs that can be sent with the
|
18
|
+
# SASL/OAUTHBEARER initial client response. If not provided, the values are ignored. This feature
|
19
|
+
# is only available in Kafka >= 2.1.0.
|
20
|
+
#
|
21
|
+
def initialize(logger:, token_provider:)
|
22
|
+
@logger = TaggedLogger.new(logger)
|
23
|
+
@token_provider = token_provider
|
24
|
+
end
|
25
|
+
|
26
|
+
def ident
|
27
|
+
OAUTH_IDENT
|
28
|
+
end
|
29
|
+
|
30
|
+
def configured?
|
31
|
+
@token_provider
|
32
|
+
end
|
33
|
+
|
34
|
+
def authenticate!(host, encoder, decoder)
|
35
|
+
# Send SASLOauthBearerClientResponse with token
|
36
|
+
@logger.debug "Authenticating to #{host} with SASL #{OAUTH_IDENT}"
|
37
|
+
|
38
|
+
encoder.write_bytes(initial_client_response)
|
39
|
+
|
40
|
+
begin
|
41
|
+
# receive SASL OAuthBearer Server Response
|
42
|
+
msg = decoder.bytes
|
43
|
+
raise Kafka::Error, "SASL #{OAUTH_IDENT} authentication failed: unknown error" unless msg
|
44
|
+
rescue Errno::ETIMEDOUT, EOFError => e
|
45
|
+
raise Kafka::Error, "SASL #{OAUTH_IDENT} authentication failed: #{e.message}"
|
46
|
+
end
|
47
|
+
|
48
|
+
@logger.debug "SASL #{OAUTH_IDENT} authentication successful."
|
49
|
+
end
|
50
|
+
|
51
|
+
private
|
52
|
+
|
53
|
+
def initial_client_response
|
54
|
+
raise Kafka::TokenMethodNotImplementedError, "Token provider doesn't define 'token'" unless @token_provider.respond_to? :token
|
55
|
+
"n,,\x01auth=Bearer #{@token_provider.token}#{token_extensions}\x01\x01"
|
56
|
+
end
|
57
|
+
|
58
|
+
def token_extensions
|
59
|
+
return nil unless @token_provider.respond_to? :extensions
|
60
|
+
"\x01#{@token_provider.extensions.map {|e| e.join("=")}.join("\x01")}"
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
data/lib/kafka/sasl/plain.rb
CHANGED
data/lib/kafka/sasl/scram.rb
CHANGED
@@ -3,13 +3,15 @@
|
|
3
3
|
require 'kafka/sasl/plain'
|
4
4
|
require 'kafka/sasl/gssapi'
|
5
5
|
require 'kafka/sasl/scram'
|
6
|
+
require 'kafka/sasl/oauth'
|
6
7
|
|
7
8
|
module Kafka
|
8
9
|
class SaslAuthenticator
|
9
10
|
def initialize(logger:, sasl_gssapi_principal:, sasl_gssapi_keytab:,
|
10
11
|
sasl_plain_authzid:, sasl_plain_username:, sasl_plain_password:,
|
11
|
-
sasl_scram_username:, sasl_scram_password:, sasl_scram_mechanism
|
12
|
-
|
12
|
+
sasl_scram_username:, sasl_scram_password:, sasl_scram_mechanism:,
|
13
|
+
sasl_oauth_token_provider:)
|
14
|
+
@logger = TaggedLogger.new(logger)
|
13
15
|
|
14
16
|
@plain = Sasl::Plain.new(
|
15
17
|
authzid: sasl_plain_authzid,
|
@@ -31,7 +33,12 @@ module Kafka
|
|
31
33
|
logger: @logger,
|
32
34
|
)
|
33
35
|
|
34
|
-
@
|
36
|
+
@oauth = Sasl::OAuth.new(
|
37
|
+
token_provider: sasl_oauth_token_provider,
|
38
|
+
logger: @logger,
|
39
|
+
)
|
40
|
+
|
41
|
+
@mechanism = [@gssapi, @plain, @scram, @oauth].find(&:configured?)
|
35
42
|
end
|
36
43
|
|
37
44
|
def enabled?
|
@@ -0,0 +1,72 @@
|
|
1
|
+
require 'forwardable'
|
2
|
+
|
3
|
+
# Basic implementation of a tagged logger that matches the API of
|
4
|
+
# ActiveSupport::TaggedLogging.
|
5
|
+
|
6
|
+
module Kafka
|
7
|
+
module TaggedFormatter
|
8
|
+
|
9
|
+
def call(severity, timestamp, progname, msg)
|
10
|
+
super(severity, timestamp, progname, "#{tags_text}#{msg}")
|
11
|
+
end
|
12
|
+
|
13
|
+
def tagged(*tags)
|
14
|
+
new_tags = push_tags(*tags)
|
15
|
+
yield self
|
16
|
+
ensure
|
17
|
+
pop_tags(new_tags.size)
|
18
|
+
end
|
19
|
+
|
20
|
+
def push_tags(*tags)
|
21
|
+
tags.flatten.reject { |t| t.nil? || t.empty? }.tap do |new_tags|
|
22
|
+
current_tags.concat new_tags
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def pop_tags(size = 1)
|
27
|
+
current_tags.pop size
|
28
|
+
end
|
29
|
+
|
30
|
+
def clear_tags!
|
31
|
+
current_tags.clear
|
32
|
+
end
|
33
|
+
|
34
|
+
def current_tags
|
35
|
+
# We use our object ID here to avoid conflicting with other instances
|
36
|
+
thread_key = @thread_key ||= "kafka_tagged_logging_tags:#{object_id}".freeze
|
37
|
+
Thread.current[thread_key] ||= []
|
38
|
+
end
|
39
|
+
|
40
|
+
def tags_text
|
41
|
+
tags = current_tags
|
42
|
+
if tags.any?
|
43
|
+
tags.collect { |tag| "[#{tag}] " }.join
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
end
|
48
|
+
|
49
|
+
module TaggedLogger
|
50
|
+
extend Forwardable
|
51
|
+
delegate [:push_tags, :pop_tags, :clear_tags!] => :formatter
|
52
|
+
|
53
|
+
def self.new(logger)
|
54
|
+
logger ||= Logger.new(nil)
|
55
|
+
return logger if logger.respond_to?(:push_tags) # already included
|
56
|
+
# Ensure we set a default formatter so we aren't extending nil!
|
57
|
+
logger.formatter ||= Logger::Formatter.new
|
58
|
+
logger.formatter.extend TaggedFormatter
|
59
|
+
logger.extend(self)
|
60
|
+
end
|
61
|
+
|
62
|
+
def tagged(*tags)
|
63
|
+
formatter.tagged(*tags) { yield self }
|
64
|
+
end
|
65
|
+
|
66
|
+
def flush
|
67
|
+
clear_tags!
|
68
|
+
super if defined?(super)
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
end
|
data/lib/kafka/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ruby-kafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.7.
|
4
|
+
version: 0.7.6.beta2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Schierbeck
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-02-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: digest-crc
|
@@ -418,6 +418,7 @@ files:
|
|
418
418
|
- lib/kafka/protocol/sync_group_response.rb
|
419
419
|
- lib/kafka/round_robin_assignment_strategy.rb
|
420
420
|
- lib/kafka/sasl/gssapi.rb
|
421
|
+
- lib/kafka/sasl/oauth.rb
|
421
422
|
- lib/kafka/sasl/plain.rb
|
422
423
|
- lib/kafka/sasl/scram.rb
|
423
424
|
- lib/kafka/sasl_authenticator.rb
|
@@ -426,6 +427,7 @@ files:
|
|
426
427
|
- lib/kafka/ssl_context.rb
|
427
428
|
- lib/kafka/ssl_socket_with_timeout.rb
|
428
429
|
- lib/kafka/statsd.rb
|
430
|
+
- lib/kafka/tagged_logger.rb
|
429
431
|
- lib/kafka/transaction_manager.rb
|
430
432
|
- lib/kafka/transaction_state_machine.rb
|
431
433
|
- lib/kafka/version.rb
|
@@ -446,9 +448,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
446
448
|
version: 2.1.0
|
447
449
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
448
450
|
requirements:
|
449
|
-
- - "
|
451
|
+
- - ">"
|
450
452
|
- !ruby/object:Gem::Version
|
451
|
-
version:
|
453
|
+
version: 1.3.1
|
452
454
|
requirements: []
|
453
455
|
rubyforge_project:
|
454
456
|
rubygems_version: 2.7.6
|