ruby-kafka 1.1.0.beta1 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +33 -0
- data/CHANGELOG.md +2 -0
- data/README.md +9 -0
- data/lib/kafka/client.rb +4 -0
- data/lib/kafka/consumer_group.rb +6 -0
- data/lib/kafka/datadog.rb +10 -1
- data/lib/kafka/offset_manager.rb +12 -1
- data/lib/kafka/ssl_context.rb +4 -3
- data/lib/kafka/version.rb +1 -1
- data/ruby-kafka.gemspec +1 -1
- metadata +5 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 22b958f3d452f35c3c6084a5b8b790f370b96069e87ef5974838e5ea4b6945e1
|
4
|
+
data.tar.gz: eb6d704dbaace9c13a99bc93924d8a6b84ee5eb686c9c863a81d085ffeb7e92d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 22c1d59bcdd42849849122b559f0d161653a5cfa492ffb36f28a875b109444d670a76bc62ba0856ae2ebce95fd10abce9e16834928fc7fe6eb2ee006827d307c
|
7
|
+
data.tar.gz: 3a51df6b1d40e1edbd96c06f3473319f0ac1b072b040b676a8e1d980fbaad114ad0248a7670cb7ae0a51a2c79ac390f2f4cfe8f65eedb5dd55be020c6011bc18
|
data/.circleci/config.yml
CHANGED
@@ -267,6 +267,38 @@ jobs:
|
|
267
267
|
- run: bundle install --path vendor/bundle
|
268
268
|
- run: bundle exec rspec --profile --tag functional spec/functional
|
269
269
|
|
270
|
+
kafka-2.5:
|
271
|
+
docker:
|
272
|
+
- image: circleci/ruby:2.5.1-node
|
273
|
+
environment:
|
274
|
+
LOG_LEVEL: DEBUG
|
275
|
+
- image: wurstmeister/zookeeper
|
276
|
+
- image: wurstmeister/kafka:2.12-2.5.0
|
277
|
+
environment:
|
278
|
+
KAFKA_ADVERTISED_HOST_NAME: localhost
|
279
|
+
KAFKA_ADVERTISED_PORT: 9092
|
280
|
+
KAFKA_PORT: 9092
|
281
|
+
KAFKA_ZOOKEEPER_CONNECT: localhost:2181
|
282
|
+
KAFKA_DELETE_TOPIC_ENABLE: true
|
283
|
+
- image: wurstmeister/kafka:2.12-2.5.0
|
284
|
+
environment:
|
285
|
+
KAFKA_ADVERTISED_HOST_NAME: localhost
|
286
|
+
KAFKA_ADVERTISED_PORT: 9093
|
287
|
+
KAFKA_PORT: 9093
|
288
|
+
KAFKA_ZOOKEEPER_CONNECT: localhost:2181
|
289
|
+
KAFKA_DELETE_TOPIC_ENABLE: true
|
290
|
+
- image: wurstmeister/kafka:2.12-2.5.0
|
291
|
+
environment:
|
292
|
+
KAFKA_ADVERTISED_HOST_NAME: localhost
|
293
|
+
KAFKA_ADVERTISED_PORT: 9094
|
294
|
+
KAFKA_PORT: 9094
|
295
|
+
KAFKA_ZOOKEEPER_CONNECT: localhost:2181
|
296
|
+
KAFKA_DELETE_TOPIC_ENABLE: true
|
297
|
+
steps:
|
298
|
+
- checkout
|
299
|
+
- run: bundle install --path vendor/bundle
|
300
|
+
- run: bundle exec rspec --profile --tag functional spec/functional
|
301
|
+
|
270
302
|
workflows:
|
271
303
|
version: 2
|
272
304
|
test:
|
@@ -280,3 +312,4 @@ workflows:
|
|
280
312
|
- kafka-2.2
|
281
313
|
- kafka-2.3
|
282
314
|
- kafka-2.4
|
315
|
+
- kafka-2.5
|
data/CHANGELOG.md
CHANGED
@@ -6,6 +6,8 @@ Changes and additions to the library will be listed here.
|
|
6
6
|
|
7
7
|
## 1.1.0
|
8
8
|
|
9
|
+
- Extra sanity checking when marking offsets as processed (#824).
|
10
|
+
- Make `verify_hostname` settable for SSL contexts (#828).
|
9
11
|
- Instrument `create_time` from last message in batch (#811).
|
10
12
|
- Add client function for fetching topic replica count (#822).
|
11
13
|
- Allow consumers to refresh the topic lists (#818).
|
data/README.md
CHANGED
@@ -118,10 +118,16 @@ Or install it yourself as:
|
|
118
118
|
<td>Limited support</td>
|
119
119
|
<td>Limited support</td>
|
120
120
|
</tr>
|
121
|
+
<tr>
|
121
122
|
<th>Kafka 2.4</th>
|
122
123
|
<td>Limited support</td>
|
123
124
|
<td>Limited support</td>
|
124
125
|
</tr>
|
126
|
+
<tr>
|
127
|
+
<th>Kafka 2.5</th>
|
128
|
+
<td>Limited support</td>
|
129
|
+
<td>Limited support</td>
|
130
|
+
</tr>
|
125
131
|
</table>
|
126
132
|
|
127
133
|
This library is targeting Kafka 0.9 with the v0.4.x series and Kafka 0.10 with the v0.5.x series. There's limited support for Kafka 0.8, and things should work with Kafka 0.11, although there may be performance issues due to changes in the protocol.
|
@@ -136,6 +142,7 @@ This library is targeting Kafka 0.9 with the v0.4.x series and Kafka 0.10 with t
|
|
136
142
|
- **Kafka 2.2:** Everything that works with Kafka 2.1 should still work, but so far no features specific to Kafka 2.2 have been added.
|
137
143
|
- **Kafka 2.3:** Everything that works with Kafka 2.2 should still work, but so far no features specific to Kafka 2.3 have been added.
|
138
144
|
- **Kafka 2.4:** Everything that works with Kafka 2.3 should still work, but so far no features specific to Kafka 2.4 have been added.
|
145
|
+
- **Kafka 2.5:** Everything that works with Kafka 2.4 should still work, but so far no features specific to Kafka 2.5 have been added.
|
139
146
|
|
140
147
|
This library requires Ruby 2.1 or higher.
|
141
148
|
|
@@ -945,6 +952,8 @@ This configures the store to look up CA certificates from the system default cer
|
|
945
952
|
|
946
953
|
In order to authenticate the client to the cluster, you need to pass in a certificate and key created for the client and trusted by the brokers.
|
947
954
|
|
955
|
+
**NOTE**: You can disable hostname validation by passing `verify_hostname: false`.
|
956
|
+
|
948
957
|
```ruby
|
949
958
|
kafka = Kafka.new(
|
950
959
|
["kafka1:9092"],
|
data/lib/kafka/client.rb
CHANGED
@@ -65,6 +65,10 @@ module Kafka
|
|
65
65
|
# @param sasl_oauth_token_provider [Object, nil] OAuthBearer Token Provider instance that
|
66
66
|
# implements method token. See {Sasl::OAuth#initialize}
|
67
67
|
#
|
68
|
+
# @param verify_hostname [Boolean, true] whether to verify that the host serving
|
69
|
+
# the SSL certificate and the signing chain of the certificate have the correct domains
|
70
|
+
# based on the CA certificate
|
71
|
+
#
|
68
72
|
# @return [Client]
|
69
73
|
def initialize(seed_brokers:, client_id: "ruby-kafka", logger: nil, connect_timeout: nil, socket_timeout: nil,
|
70
74
|
ssl_ca_cert_file_path: nil, ssl_ca_cert: nil, ssl_client_cert: nil, ssl_client_cert_key: nil,
|
data/lib/kafka/consumer_group.rb
CHANGED
data/lib/kafka/datadog.rb
CHANGED
@@ -31,7 +31,7 @@ module Kafka
|
|
31
31
|
|
32
32
|
class << self
|
33
33
|
def statsd
|
34
|
-
@statsd ||= ::Datadog::Statsd.new(host, port, namespace: namespace, tags: tags)
|
34
|
+
@statsd ||= ::Datadog::Statsd.new(host, port, namespace: namespace, tags: tags, socket_path: socket_path)
|
35
35
|
end
|
36
36
|
|
37
37
|
def statsd=(statsd)
|
@@ -57,6 +57,15 @@ module Kafka
|
|
57
57
|
clear
|
58
58
|
end
|
59
59
|
|
60
|
+
def socket_path
|
61
|
+
@socket_path
|
62
|
+
end
|
63
|
+
|
64
|
+
def socket_path=(socket_path)
|
65
|
+
@socket_path = socket_path
|
66
|
+
clear
|
67
|
+
end
|
68
|
+
|
60
69
|
def namespace
|
61
70
|
@namespace ||= STATSD_NAMESPACE
|
62
71
|
end
|
data/lib/kafka/offset_manager.rb
CHANGED
@@ -50,9 +50,20 @@ module Kafka
|
|
50
50
|
# @param offset [Integer] the offset of the message that should be marked as processed.
|
51
51
|
# @return [nil]
|
52
52
|
def mark_as_processed(topic, partition, offset)
|
53
|
-
@
|
53
|
+
unless @group.assigned_to?(topic, partition)
|
54
|
+
@logger.debug "Not marking #{topic}/#{partition}:#{offset} as processed for partition not assigned to this consumer."
|
55
|
+
return
|
56
|
+
end
|
54
57
|
@processed_offsets[topic] ||= {}
|
55
58
|
|
59
|
+
last_processed_offset = @processed_offsets[topic][partition] || -1
|
60
|
+
if last_processed_offset > offset + 1
|
61
|
+
@logger.debug "Not overwriting newer offset #{topic}/#{partition}:#{last_processed_offset - 1} with older #{offset}"
|
62
|
+
return
|
63
|
+
end
|
64
|
+
|
65
|
+
@uncommitted_offsets += 1
|
66
|
+
|
56
67
|
# The committed offset should always be the offset of the next message that the
|
57
68
|
# application will read, thus adding one to the last message processed.
|
58
69
|
@processed_offsets[topic][partition] = offset + 1
|
data/lib/kafka/ssl_context.rb
CHANGED
@@ -54,11 +54,12 @@ module Kafka
|
|
54
54
|
store.set_default_paths
|
55
55
|
end
|
56
56
|
ssl_context.cert_store = store
|
57
|
-
ssl_context.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
58
|
-
# Verify certificate hostname if supported (ruby >= 2.4.0)
|
59
|
-
ssl_context.verify_hostname = verify_hostname if ssl_context.respond_to?(:verify_hostname=)
|
60
57
|
end
|
61
58
|
|
59
|
+
ssl_context.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
60
|
+
# Verify certificate hostname if supported (ruby >= 2.4.0)
|
61
|
+
ssl_context.verify_hostname = verify_hostname if ssl_context.respond_to?(:verify_hostname=)
|
62
|
+
|
62
63
|
ssl_context
|
63
64
|
end
|
64
65
|
end
|
data/lib/kafka/version.rb
CHANGED
data/ruby-kafka.gemspec
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ruby-kafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.1.0
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Schierbeck
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-06-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: digest-crc
|
@@ -476,7 +476,7 @@ files:
|
|
476
476
|
- ruby-kafka.gemspec
|
477
477
|
homepage: https://github.com/zendesk/ruby-kafka
|
478
478
|
licenses:
|
479
|
-
- Apache
|
479
|
+
- Apache-2.0
|
480
480
|
metadata: {}
|
481
481
|
post_install_message:
|
482
482
|
rdoc_options: []
|
@@ -489,9 +489,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
489
489
|
version: 2.1.0
|
490
490
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
491
491
|
requirements:
|
492
|
-
- - "
|
492
|
+
- - ">="
|
493
493
|
- !ruby/object:Gem::Version
|
494
|
-
version:
|
494
|
+
version: '0'
|
495
495
|
requirements: []
|
496
496
|
rubyforge_project:
|
497
497
|
rubygems_version: 2.7.6
|