racecar 0.3.7 → 0.3.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/CHANGELOG.md +6 -0
- data/README.md +13 -2
- data/lib/racecar/config.rb +6 -2
- data/lib/racecar/consumer.rb +1 -0
- data/lib/racecar/runner.rb +6 -0
- data/lib/racecar/version.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 2e43d9ec7a2fe43e035cb42d1ab21a250072d26bd5dc1b6dde34600870193152
|
4
|
+
data.tar.gz: 4a3dc623057a7c3c78ab5a009a85f9b1bf839d49334578894575a93cb0a06ea9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2676b9e5adaa2280b7865b3cc636603ec4a6c827c648b4d9df21796f1891da1f998863c370ebd674ecbf8b0f5b3cf1953e2e521ba09a422c9eed7bfde3d93808
|
7
|
+
data.tar.gz: 59b529314b4b3b13a5ebad8f9a53c51aa24ed4ef8ae22b30f17dfac3474a8dc7714776fcacda29b04b649df4ffee2754c1bd0d1f9da1048b43e3eff864ff1bb1
|
data/CHANGELOG.md
CHANGED
@@ -2,6 +2,12 @@
|
|
2
2
|
|
3
3
|
## Unreleased
|
4
4
|
|
5
|
+
## racecar v0.3.8
|
6
|
+
|
7
|
+
* Change the default `max_wait_time` to 1 second.
|
8
|
+
* Allow setting the `offset_retention_time` for consumers.
|
9
|
+
* Allow pausing partitions indefinitely (#63).
|
10
|
+
|
5
11
|
## racecar v0.3.7
|
6
12
|
|
7
13
|
* Allow setting the key and/or partition key when producing messages.
|
data/README.md
CHANGED
@@ -57,7 +57,7 @@ In Kafka, _consumer groups_ are sets of processes that collaboratively process m
|
|
57
57
|
|
58
58
|
### Creating consumers
|
59
59
|
|
60
|
-
A Racecar consumer is a simple
|
60
|
+
A Racecar consumer is a simple Ruby class that inherits from `Racecar::Consumer`:
|
61
61
|
|
62
62
|
```ruby
|
63
63
|
class UserBanConsumer < Racecar::Consumer
|
@@ -231,6 +231,7 @@ The consumers will checkpoint their positions from time to time in order to be a
|
|
231
231
|
|
232
232
|
* `offset_commit_interval` – How often to save the consumer's position in Kafka. Default is every 10 seconds.
|
233
233
|
* `offset_commit_threshold` – How many messages to process before forcing a checkpoint. Default is 0, which means there's no limit. Setting this to e.g. 100 makes the consumer stop every 100 messages to checkpoint its position.
|
234
|
+
* `offset_retention_time` - How long committed offsets will be retained. Defaults to the broker setting.
|
234
235
|
|
235
236
|
#### Timeouts & intervals
|
236
237
|
|
@@ -241,7 +242,7 @@ All timeouts are defined in number of seconds.
|
|
241
242
|
* `pause_timeout` – How long to pause a partition for if the consumer raises an exception while processing a message. Default is to pause for 10 seconds. Set this to zero in order to disable automatic pausing of partitions.
|
242
243
|
* `connect_timeout` – How long to wait when trying to connect to a Kafka broker. Default is 10 seconds.
|
243
244
|
* `socket_timeout` – How long to wait when trying to communicate with a Kafka broker. Default is 30 seconds.
|
244
|
-
* `max_wait_time` – How long to allow the Kafka brokers to wait before returning messages. A higher number means larger batches, at the cost of higher latency. Default is
|
245
|
+
* `max_wait_time` – How long to allow the Kafka brokers to wait before returning messages. A higher number means larger batches, at the cost of higher latency. Default is 1 second.
|
245
246
|
|
246
247
|
#### SSL encryption, authentication & authorization
|
247
248
|
|
@@ -265,6 +266,16 @@ If using PLAIN:
|
|
265
266
|
* `sasl_plain_username` – The username used to authenticate.
|
266
267
|
* `sasl_plain_password` – The password used to authenticate.
|
267
268
|
|
269
|
+
#### Datadog monitoring
|
270
|
+
|
271
|
+
Racecar supports configuring ruby-kafka's [Datadog](https://www.datadoghq.com/) monitoring integration. If you're running a normal Datadog agent on your host, you just need to set `datadog_enabled` to `true`, as the rest of the settings come with sane defaults.
|
272
|
+
|
273
|
+
* `datadog_enabled` – Whether Datadog monitoring is enabled (defaults to `false`).
|
274
|
+
* `datadog_host` – The host running the Datadog agent.
|
275
|
+
* `datadog_port` – The port of the Datadog agent.
|
276
|
+
* `datadog_namespace` – The namespace to use for Datadog metrics.
|
277
|
+
* `datadog_tags` – Tags that should always be set on Datadog metrics.
|
278
|
+
|
268
279
|
|
269
280
|
### Testing consumers
|
270
281
|
|
data/lib/racecar/config.rb
CHANGED
@@ -19,7 +19,10 @@ module Racecar
|
|
19
19
|
desc "How often to send a heartbeat message to Kafka"
|
20
20
|
float :heartbeat_interval, default: 10
|
21
21
|
|
22
|
-
desc "How long
|
22
|
+
desc "How long committed offsets will be retained."
|
23
|
+
integer :offset_retention_time
|
24
|
+
|
25
|
+
desc "How long to pause a partition for if the consumer raises an exception while processing a message -- set to -1 to pause indefinitely"
|
23
26
|
float :pause_timeout, default: 10
|
24
27
|
|
25
28
|
desc "The idle timeout after which a consumer is kicked out of the group"
|
@@ -32,7 +35,7 @@ module Racecar
|
|
32
35
|
float :socket_timeout, default: 30
|
33
36
|
|
34
37
|
desc "How long to allow the Kafka brokers to wait before returning messages"
|
35
|
-
float :max_wait_time, default:
|
38
|
+
float :max_wait_time, default: 1
|
36
39
|
|
37
40
|
desc "A prefix used when generating consumer group names"
|
38
41
|
string :group_id_prefix
|
@@ -139,6 +142,7 @@ module Racecar
|
|
139
142
|
|
140
143
|
self.subscriptions = consumer_class.subscriptions
|
141
144
|
self.max_wait_time = consumer_class.max_wait_time || self.max_wait_time
|
145
|
+
self.offset_retention_time = consumer_class.offset_retention_time || self.offset_retention_time
|
142
146
|
self.pidfile ||= "#{group_id}.pid"
|
143
147
|
end
|
144
148
|
|
data/lib/racecar/consumer.rb
CHANGED
data/lib/racecar/runner.rb
CHANGED
@@ -35,6 +35,7 @@ module Racecar
|
|
35
35
|
offset_commit_threshold: config.offset_commit_threshold,
|
36
36
|
session_timeout: config.session_timeout,
|
37
37
|
heartbeat_interval: config.heartbeat_interval,
|
38
|
+
offset_retention_time: config.offset_retention_time,
|
38
39
|
)
|
39
40
|
|
40
41
|
# Stop the consumer on SIGINT, SIGQUIT or SIGTERM.
|
@@ -107,6 +108,11 @@ module Racecar
|
|
107
108
|
# left off.
|
108
109
|
@logger.warn "Pausing partition #{e.topic}/#{e.partition} for #{config.pause_timeout} seconds"
|
109
110
|
consumer.pause(e.topic, e.partition, timeout: config.pause_timeout)
|
111
|
+
elsif config.pause_timeout == -1
|
112
|
+
# A pause timeout of -1 means indefinite pausing, which in ruby-kafka is done by passing nil as
|
113
|
+
# the timeout.
|
114
|
+
@logger.warn "Pausing partition #{e.topic}/#{e.partition} indefinitely, or until the process is restarted"
|
115
|
+
consumer.pause(e.topic, e.partition, timeout: nil)
|
110
116
|
end
|
111
117
|
|
112
118
|
config.error_handler.call(e.cause, {
|
data/lib/racecar/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: racecar
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Schierbeck
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: exe
|
11
11
|
cert_chain: []
|
12
|
-
date: 2018-
|
12
|
+
date: 2018-03-02 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: king_konf
|
@@ -140,7 +140,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
140
140
|
version: '0'
|
141
141
|
requirements: []
|
142
142
|
rubyforge_project:
|
143
|
-
rubygems_version: 2.6
|
143
|
+
rubygems_version: 2.7.6
|
144
144
|
signing_key:
|
145
145
|
specification_version: 4
|
146
146
|
summary: A framework for running Kafka consumers
|