racecar 2.3.0.alpha1 → 2.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +1 -1
- data/CHANGELOG.md +16 -0
- data/Gemfile +4 -0
- data/Gemfile.lock +14 -16
- data/README.md +87 -73
- data/lib/racecar/config.rb +1 -0
- data/lib/racecar/consumer.rb +22 -20
- data/lib/racecar/consumer_set.rb +1 -1
- data/lib/racecar/datadog.rb +5 -1
- data/lib/racecar/heroku.rb +49 -0
- data/lib/racecar/runner.rb +1 -0
- data/lib/racecar/version.rb +1 -1
- data/racecar.gemspec +4 -2
- metadata +10 -9
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 91b2364f47ab248a2311ddd8164e41c6f2749a66a0acb02c8f5b97e4d33f8f4f
|
4
|
+
data.tar.gz: 447eef9ccabae8fd2ecdccd79a910c9445556f2ca1cd29064d68b4e2aa70654e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2d9bf1559b276e14050f74f709ac461a5241fa2c558e260c9eb6d2e722061d8a7032e39b30aaed091f7db6ae9d8202750ec6ba0e74f4d91f6d510078af803568
|
7
|
+
data.tar.gz: 52830f9b015384a91b337964157570e159afffe8a14d792420af45d30f97494209703aca2460f61960920988f66bd0a3771a036b0e9ae83528860510bf0eef1b
|
data/.github/workflows/ci.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -2,6 +2,22 @@
|
|
2
2
|
|
3
3
|
## Unreleased
|
4
4
|
|
5
|
+
## v2.5.0
|
6
|
+
|
7
|
+
* `fetch_messages` can be configured per consumer, just as the maximum timeout to wait for a full batch.
|
8
|
+
|
9
|
+
## v2.4.0
|
10
|
+
|
11
|
+
* Update librdkafka version from 1.4.0 to 1.5.0 by upgrading from rdkafka 0.8.0 to 0.10.0. ([#263](https://github.com/zendesk/racecar/pull/263))
|
12
|
+
* Restore support for Ruby 2.4 (#258)
|
13
|
+
|
14
|
+
## racecar v2.3.1
|
15
|
+
|
16
|
+
* Handle `ERR_NOT_COORDINATOR` (#209)
|
17
|
+
|
18
|
+
## racecar v2.3.0
|
19
|
+
|
20
|
+
* Add native support for Heroku (#248)
|
5
21
|
* [Racecar::Consumer] When messages fail to deliver, an extended error with hints is now raised. Instead of `Rdkafka::RdkafkaError` you'll get a `Racecar::MessageDeliveryError` instead. ([#219](https://github.com/zendesk/racecar/pull/219)). If you have set a `Racecar.config.error_handler`, it might need to be updated.
|
6
22
|
* [Racecar::Consumer] When message delivery times out, Racecar will reset the producer in an attempt to fix some of the potential causes for this error. ([#219](https://github.com/zendesk/racecar/pull/219))
|
7
23
|
* Validate the `process` and `process_batch` method signature on consumer classes when initializing (#236)
|
data/Gemfile
CHANGED
@@ -4,3 +4,7 @@ source 'https://rubygems.org'
|
|
4
4
|
|
5
5
|
# Specify your gem's dependencies in racecar.gemspec
|
6
6
|
gemspec
|
7
|
+
|
8
|
+
# We actually support version 6.0 (see gemspec); this extra restriction is added just for running the test suite also
|
9
|
+
# on Ruby 2.4, which activesupport 6.0 no longer supports
|
10
|
+
gem 'activesupport', '< 6.0'
|
data/Gemfile.lock
CHANGED
@@ -1,35 +1,34 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
racecar (2.
|
4
|
+
racecar (2.4.0)
|
5
5
|
king_konf (~> 1.0.0)
|
6
|
-
rdkafka (~> 0.
|
6
|
+
rdkafka (~> 0.10.0)
|
7
7
|
|
8
8
|
GEM
|
9
9
|
remote: https://rubygems.org/
|
10
10
|
specs:
|
11
|
-
activesupport (
|
11
|
+
activesupport (5.2.6)
|
12
12
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
13
13
|
i18n (>= 0.7, < 2)
|
14
14
|
minitest (~> 5.1)
|
15
15
|
tzinfo (~> 1.1)
|
16
|
-
zeitwerk (~> 2.2, >= 2.2.2)
|
17
16
|
coderay (1.1.3)
|
18
|
-
concurrent-ruby (1.1.
|
17
|
+
concurrent-ruby (1.1.9)
|
19
18
|
diff-lcs (1.4.4)
|
20
|
-
dogstatsd-ruby (
|
21
|
-
ffi (1.
|
22
|
-
i18n (1.8.
|
19
|
+
dogstatsd-ruby (5.2.0)
|
20
|
+
ffi (1.15.4)
|
21
|
+
i18n (1.8.10)
|
23
22
|
concurrent-ruby (~> 1.0)
|
24
23
|
king_konf (1.0.0)
|
25
24
|
method_source (1.0.0)
|
26
|
-
mini_portile2 (2.
|
27
|
-
minitest (5.14.
|
25
|
+
mini_portile2 (2.7.0)
|
26
|
+
minitest (5.14.4)
|
28
27
|
pry (0.13.1)
|
29
28
|
coderay (~> 1.1)
|
30
29
|
method_source (~> 1.0)
|
31
30
|
rake (13.0.1)
|
32
|
-
rdkafka (0.
|
31
|
+
rdkafka (0.10.0)
|
33
32
|
ffi (~> 1.9)
|
34
33
|
mini_portile2 (~> 2.1)
|
35
34
|
rake (>= 12.3)
|
@@ -48,17 +47,16 @@ GEM
|
|
48
47
|
rspec-support (3.10.2)
|
49
48
|
thread_safe (0.3.6)
|
50
49
|
timecop (0.9.2)
|
51
|
-
tzinfo (1.2.
|
50
|
+
tzinfo (1.2.9)
|
52
51
|
thread_safe (~> 0.1)
|
53
|
-
zeitwerk (2.4.2)
|
54
52
|
|
55
53
|
PLATFORMS
|
56
54
|
ruby
|
57
55
|
|
58
56
|
DEPENDENCIES
|
59
|
-
activesupport (
|
57
|
+
activesupport (< 6.0)
|
60
58
|
bundler (>= 1.13, < 3)
|
61
|
-
dogstatsd-ruby (>= 4.0.0, <
|
59
|
+
dogstatsd-ruby (>= 4.0.0, < 6.0.0)
|
62
60
|
pry
|
63
61
|
racecar!
|
64
62
|
rake (> 10.0)
|
@@ -66,4 +64,4 @@ DEPENDENCIES
|
|
66
64
|
timecop
|
67
65
|
|
68
66
|
BUNDLED WITH
|
69
|
-
2.
|
67
|
+
2.2.26
|
data/README.md
CHANGED
@@ -10,22 +10,21 @@ The framework is based on [rdkafka-ruby](https://github.com/appsignal/rdkafka-ru
|
|
10
10
|
|
11
11
|
1. [Installation](#installation)
|
12
12
|
2. [Usage](#usage)
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
13
|
+
1. [Creating consumers](#creating-consumers)
|
14
|
+
2. [Running consumers](#running-consumers)
|
15
|
+
3. [Producing messages](#producing-messages)
|
16
|
+
4. [Configuration](#configuration)
|
17
|
+
5. [Testing consumers](#testing-consumers)
|
18
|
+
6. [Deploying consumers](#deploying-consumers)
|
19
|
+
7. [Handling errors](#handling-errors)
|
20
|
+
8. [Logging](#logging)
|
21
|
+
9. [Operations](#operations)
|
22
|
+
10. [Upgrading from v1 to v2](#upgrading-from-v1-to-v2)
|
23
23
|
3. [Development](#development)
|
24
24
|
4. [Contributing](#contributing)
|
25
25
|
5. [Support and Discussion](#support-and-discussion)
|
26
26
|
6. [Copyright and license](#copyright-and-license)
|
27
27
|
|
28
|
-
|
29
28
|
## Installation
|
30
29
|
|
31
30
|
Add this line to your application's Gemfile:
|
@@ -77,7 +76,7 @@ In order to create your own consumer, run the Rails generator `racecar:consumer`
|
|
77
76
|
|
78
77
|
$ bundle exec rails generate racecar:consumer TapDance
|
79
78
|
|
80
|
-
This will create a file at `app/consumers/tap_dance_consumer.rb` which you can modify to your liking. Add one or more calls to
|
79
|
+
This will create a file at `app/consumers/tap_dance_consumer.rb` which you can modify to your liking. Add one or more calls to `subscribes_to` in order to have the consumer subscribe to Kafka topics.
|
81
80
|
|
82
81
|
Now run your consumer with `bundle exec racecar TapDanceConsumer`.
|
83
82
|
|
@@ -88,13 +87,15 @@ Note: if you're not using Rails, you'll have to add the file yourself. No-one wi
|
|
88
87
|
Warning - limited battle testing in production environments; use at your own risk!
|
89
88
|
|
90
89
|
If you want to process different partitions in parallel, and don't want to deploy a number of instances matching the total partitions of the topic, you can specify the number of workers to spin up - that number of processes will be forked, and each will register its own consumer in the group. Some things to note:
|
90
|
+
|
91
91
|
- This would make no difference on a single partitioned topic - only one consumer would ever be assigned a partition. A couple of example configurations to process all partitions in parallel (we'll assume a 15 partition topic):
|
92
92
|
- Parallel workers set to 3, 5 separate instances / replicas running in your container orchestrator
|
93
93
|
- Parallel workers set to 5, 3 separate instances / replicas running in your container orchestrator
|
94
94
|
- Since we're forking new processes, the memory demands are a little higher
|
95
|
-
- From some initial testing, running 5 parallel workers requires no more than double the memory of running a Racecar consumer without parallelism.
|
95
|
+
- From some initial testing, running 5 parallel workers requires no more than double the memory of running a Racecar consumer without parallelism.
|
96
96
|
|
97
97
|
The number of parallel workers is configured per consumer class; you may only want to take advantage of this for busier consumers:
|
98
|
+
|
98
99
|
```ruby
|
99
100
|
class ParallelProcessingConsumer < Racecar::Consumer
|
100
101
|
subscribes_to "some-topic"
|
@@ -182,9 +183,9 @@ message.headers #=> { "Header-A" => 42, ... }
|
|
182
183
|
|
183
184
|
In order to avoid your consumer being kicked out of its group during long-running message processing operations, you'll need to let Kafka regularly know that the consumer is still healthy. There's two mechanisms in place to ensure that:
|
184
185
|
|
185
|
-
|
186
|
+
_Heartbeats:_ They are automatically sent in the background and ensure the broker can still talk to the consumer. This will detect network splits, ungraceful shutdowns, etc.
|
186
187
|
|
187
|
-
|
188
|
+
_Message Fetch Interval:_ Kafka expects the consumer to query for new messages within this time limit. This will detect situations with slow IO or the consumer being stuck in an infinite loop without making actual progress. This limit applies to a whole batch if you do batch processing. Use `max_poll_interval` to increase the default 5 minute timeout, or reduce batching with `fetch_messages`.
|
188
189
|
|
189
190
|
#### Tearing down resources when stopping
|
190
191
|
|
@@ -263,35 +264,39 @@ end
|
|
263
264
|
|
264
265
|
#### Basic configuration
|
265
266
|
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
267
|
+
- `brokers` – A list of Kafka brokers in the cluster that you're consuming from. Defaults to `localhost` on port 9092, the default Kafka port.
|
268
|
+
- `client_id` – A string used to identify the client in logs and metrics.
|
269
|
+
- `group_id` – The group id to use for a given group of consumers. Note that this _must_ be different for each consumer class. If left blank a group id is generated based on the consumer class name such that (for example) a consumer with the class name `BaconConsumer` would default to a group id of `bacon-consumer`.
|
270
|
+
- `group_id_prefix` – A prefix used when generating consumer group names. For instance, if you set the prefix to be `kevin.` and your consumer class is named `BaconConsumer`, the resulting consumer group will be named `kevin.bacon-consumer`.
|
271
|
+
|
272
|
+
#### Batches
|
273
|
+
|
274
|
+
- `fetch_messages` - The number of messages to fetch in a single batch. This can be set on a per consumer basis.
|
270
275
|
|
271
276
|
#### Logging
|
272
277
|
|
273
|
-
|
274
|
-
|
278
|
+
- `logfile` – A filename that log messages should be written to. Default is `nil`, which means logs will be written to standard output.
|
279
|
+
- `log_level` – The log level for the Racecar logs, one of `debug`, `info`, `warn`, or `error`. Default is `info`.
|
275
280
|
|
276
281
|
#### Consumer checkpointing
|
277
282
|
|
278
283
|
The consumers will checkpoint their positions from time to time in order to be able to recover from failures. This is called _committing offsets_, since it's done by tracking the offset reached in each partition being processed, and committing those offset numbers to the Kafka offset storage API. If you can tolerate more double-processing after a failure, you can increase the interval between commits in order to better performance. You can also do the opposite if you prefer less chance of double-processing.
|
279
284
|
|
280
|
-
|
285
|
+
- `offset_commit_interval` – How often to save the consumer's position in Kafka. Default is every 10 seconds.
|
281
286
|
|
282
287
|
#### Timeouts & intervals
|
283
288
|
|
284
289
|
All timeouts are defined in number of seconds.
|
285
290
|
|
286
|
-
|
287
|
-
|
288
|
-
|
289
|
-
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
291
|
+
- `session_timeout` – The idle timeout after which a consumer is kicked out of the group. Consumers must send heartbeats with at least this frequency.
|
292
|
+
- `heartbeat_interval` – How often to send a heartbeat message to Kafka.
|
293
|
+
- `max_poll_interval` – The maximum time between two message fetches before the consumer is kicked out of the group. Put differently, your (batch) processing must finish earlier than this.
|
294
|
+
- `pause_timeout` – How long to pause a partition for if the consumer raises an exception while processing a message. Default is to pause for 10 seconds. Set this to `0` in order to disable automatic pausing of partitions or to `-1` to pause indefinitely.
|
295
|
+
- `pause_with_exponential_backoff` – Set to `true` if you want to double the `pause_timeout` on each consecutive failure of a particular partition.
|
296
|
+
- `socket_timeout` – How long to wait when trying to communicate with a Kafka broker. Default is 30 seconds.
|
297
|
+
- `max_wait_time` – How long to allow the Kafka brokers to wait before returning messages. A higher number means larger batches, at the cost of higher latency. Default is 1 second.
|
298
|
+
- `message_timeout` – How long to try to deliver a produced message before finally giving up. Default is 5 minutes. Transient errors are automatically retried. If a message delivery fails, the current read message batch is retried.
|
299
|
+
- `statistics_interval` – How frequently librdkafka should publish statistics about its consumers and producers; you must also add a `statistics_callback` method to your processor, otherwise the stats are disabled. The default is 1 second, however this can be quite memory hungry, so you may want to tune this and monitor.
|
295
300
|
|
296
301
|
#### Memory & network usage
|
297
302
|
|
@@ -299,55 +304,55 @@ Kafka is _really_ good at throwing data at consumers, so you may want to tune th
|
|
299
304
|
|
300
305
|
Racecar uses [rdkafka-ruby](https://github.com/appsignal/rdkafka-ruby) under the hood, which fetches messages from the Kafka brokers in a background thread. This thread pushes fetch responses, possible containing messages from many partitions, into a queue that is read by the processing thread (AKA your code). The main way to control the fetcher thread is to control the size of those responses and the size of the queue.
|
301
306
|
|
302
|
-
|
303
|
-
|
307
|
+
- `max_bytes` — Maximum amount of data the broker shall return for a Fetch request.
|
308
|
+
- `min_message_queue_size` — The minimum number of messages in the local consumer queue.
|
304
309
|
|
305
310
|
The memory usage limit is roughly estimated as `max_bytes * min_message_queue_size`, plus whatever your application uses.
|
306
311
|
|
307
312
|
#### SSL encryption, authentication & authorization
|
308
313
|
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
314
|
+
- `security_protocol` – Protocol used to communicate with brokers (`:ssl`)
|
315
|
+
- `ssl_ca_location` – File or directory path to CA certificate(s) for verifying the broker's key
|
316
|
+
- `ssl_crl_location` – Path to CRL for verifying broker's certificate validity
|
317
|
+
- `ssl_keystore_location` – Path to client's keystore (PKCS#12) used for authentication
|
318
|
+
- `ssl_keystore_password` – Client's keystore (PKCS#12) password
|
319
|
+
- `ssl_certificate_location` – Path to the certificate
|
320
|
+
- `ssl_key_location` – Path to client's certificate used for authentication
|
321
|
+
- `ssl_key_password` – Client's certificate password
|
317
322
|
|
318
323
|
#### SASL encryption, authentication & authorization
|
319
324
|
|
320
325
|
Racecar has support for using SASL to authenticate clients using either the GSSAPI or PLAIN mechanism either via plaintext or SSL connection.
|
321
326
|
|
322
|
-
|
323
|
-
|
327
|
+
- `security_protocol` – Protocol used to communicate with brokers (`:sasl_plaintext` `:sasl_ssl`)
|
328
|
+
- `sasl_mechanism` – SASL mechanism to use for authentication (`GSSAPI` `PLAIN` `SCRAM-SHA-256` `SCRAM-SHA-512`)
|
324
329
|
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
330
|
+
- `sasl_kerberos_principal` – This client's Kerberos principal name
|
331
|
+
- `sasl_kerberos_kinit_cmd` – Full kerberos kinit command string, `%{config.prop.name}` is replaced by corresponding config object value, `%{broker.name}` returns the broker's hostname
|
332
|
+
- `sasl_kerberos_keytab` – Path to Kerberos keytab file. Uses system default if not set
|
333
|
+
- `sasl_kerberos_min_time_before_relogin` – Minimum time in milliseconds between key refresh attempts
|
334
|
+
- `sasl_username` – SASL username for use with the PLAIN and SASL-SCRAM-.. mechanism
|
335
|
+
- `sasl_password` – SASL password for use with the PLAIN and SASL-SCRAM-.. mechanism
|
331
336
|
|
332
337
|
#### Producing messages
|
333
338
|
|
334
339
|
These settings are related to consumers that _produce messages to Kafka_.
|
335
340
|
|
336
|
-
|
341
|
+
- `producer_compression_codec` – If defined, Racecar will compress messages before writing them to Kafka. The codec needs to be one of `gzip`, `lz4`, or `snappy`, either as a Symbol or a String.
|
337
342
|
|
338
343
|
#### Datadog monitoring
|
339
344
|
|
340
345
|
Racecar supports [Datadog](https://www.datadoghq.com/) monitoring integration. If you're running a normal Datadog agent on your host, you just need to set `datadog_enabled` to `true`, as the rest of the settings come with sane defaults.
|
341
346
|
|
342
|
-
|
343
|
-
|
344
|
-
|
345
|
-
|
346
|
-
|
347
|
+
- `datadog_enabled` – Whether Datadog monitoring is enabled (defaults to `false`).
|
348
|
+
- `datadog_host` – The host running the Datadog agent.
|
349
|
+
- `datadog_port` – The port of the Datadog agent.
|
350
|
+
- `datadog_namespace` – The namespace to use for Datadog metrics.
|
351
|
+
- `datadog_tags` – Tags that should always be set on Datadog metrics.
|
347
352
|
|
348
353
|
Furthermore, there's a [standard Datadog dashboard configution file](https://raw.githubusercontent.com/zendesk/racecar/master/extra/datadog-dashboard.json) that you can import to get started with a Racecar dashboard for all of your consumers.
|
349
354
|
|
350
|
-
#### Consumers Without Rails
|
355
|
+
#### Consumers Without Rails
|
351
356
|
|
352
357
|
By default, if Rails is detected, it will be automatically started when the consumer is started. There are cases where you might not want or need Rails. You can pass the `--without-rails` option when starting the consumer and Rails won't be started.
|
353
358
|
|
@@ -385,7 +390,6 @@ describe CreateContactsConsumer do
|
|
385
390
|
end
|
386
391
|
```
|
387
392
|
|
388
|
-
|
389
393
|
### Deploying consumers
|
390
394
|
|
391
395
|
If you're already deploying your Rails application using e.g. [Capistrano](http://capistranorb.com/), all you need to do to run your Racecar consumers in production is to have some _process supervisor_ start the processes and manage them for you.
|
@@ -397,7 +401,7 @@ racecar-process-payments: bundle exec racecar ProcessPaymentsConsumer
|
|
397
401
|
racecar-resize-images: bundle exec racecar ResizeImagesConsumer
|
398
402
|
```
|
399
403
|
|
400
|
-
If you've ever used Heroku you'll recognize the format – indeed, deploying to Heroku should just work if you add Racecar invocations to your Procfile
|
404
|
+
If you've ever used Heroku you'll recognize the format – indeed, deploying to Heroku should just work if you add Racecar invocations to your Procfile and [enable the Heroku integration](#deploying-to-heroku)
|
401
405
|
|
402
406
|
With Foreman, you can easily run these processes locally by executing `foreman run`; in production you'll want to _export_ to another process management format such as Upstart or Runit. [capistrano-foreman](https://github.com/hyperoslo/capistrano-foreman) allows you to do this with Capistrano.
|
403
407
|
|
@@ -425,20 +429,37 @@ spec:
|
|
425
429
|
app: my-racecar
|
426
430
|
spec:
|
427
431
|
containers:
|
428
|
-
|
429
|
-
|
430
|
-
|
431
|
-
|
432
|
-
|
433
|
-
|
434
|
-
|
435
|
-
|
432
|
+
- name: my-racecar
|
433
|
+
image: my-racecar-image
|
434
|
+
command: ["bundle", "exec", "racecar", "MyConsumer"]
|
435
|
+
env: # <-- you can configure the consumer using environment variables!
|
436
|
+
- name: RACECAR_BROKERS
|
437
|
+
value: kafka1,kafka2,kafka3
|
438
|
+
- name: RACECAR_OFFSET_COMMIT_INTERVAL
|
439
|
+
value: 5
|
436
440
|
```
|
437
441
|
|
438
442
|
The important part is the `strategy.type` value, which tells Kubernetes how to upgrade from one version of your Deployment to another. Many services use so-called _rolling updates_, where some but not all containers are replaced with the new version. This is done so that, if the new version doesn't work, the old version is still there to serve most of the requests. For Kafka consumers, this doesn't work well. The reason is that every time a consumer joins or leaves a group, every other consumer in the group needs to stop and synchronize the list of partitions assigned to each group member. So if the group is updated in a rolling fashion, this synchronization would occur over and over again, causing undesirable double-processing of messages as consumers would start only to be synchronized shortly after.
|
439
443
|
|
440
444
|
Instead, the `Recreate` update strategy should be used. It completely tears down the existing containers before starting all of the new containers simultaneously, allowing for a single synchronization stage and a much faster, more stable deployment update.
|
441
445
|
|
446
|
+
#### Deploying to Heroku
|
447
|
+
|
448
|
+
If you run your applications in Heroku and/or use the Heroku Kafka add-on, you application will be provided with 4 ENV variables that allow connecting to the cluster: `KAFKA_URL`, `KAFKA_TRUSTED_CERT`, `KAFKA_CLIENT_CERT`, and `KAFKA_CLIENT_CERT_KEY`.
|
449
|
+
|
450
|
+
Racecar has a built-in helper for configuring your application based on these variables – just add `require "racecar/heroku"` and everything should just work.
|
451
|
+
|
452
|
+
Please note aliasing the Heroku Kafka add-on will break this integration. If you have a need to do that, please ask on [the discussion board](https://github.com/zendesk/racecar/discussions).
|
453
|
+
|
454
|
+
```ruby
|
455
|
+
# This takes care of setting up your consumer based on the ENV
|
456
|
+
# variables provided by Heroku.
|
457
|
+
require "racecar/heroku"
|
458
|
+
|
459
|
+
class SomeConsumer < Racecar::Consumer
|
460
|
+
# ...
|
461
|
+
end
|
462
|
+
```
|
442
463
|
|
443
464
|
#### Running consumers in the background
|
444
465
|
|
@@ -456,7 +477,6 @@ Since the process is daemonized, you need to know the process id (PID) in order
|
|
456
477
|
|
457
478
|
Again, the recommended approach is to manage the processes using process managers. Only do this if you have to.
|
458
479
|
|
459
|
-
|
460
480
|
### Handling errors
|
461
481
|
|
462
482
|
When processing messages from a Kafka topic, your code may encounter an error and raise an exception. The cause is typically one of two things:
|
@@ -494,26 +514,22 @@ end
|
|
494
514
|
|
495
515
|
It is highly recommended that you set up an error handler. Please note that the `info` object contains different keys and values depending on whether you are using `process` or `process_batch`. See the `instrumentation_payload` object in the `process` and `process_batch` methods in the `Runner` class for the complete list.
|
496
516
|
|
497
|
-
|
498
517
|
### Logging
|
499
518
|
|
500
519
|
By default, Racecar will log to `STDOUT`. If you're using Rails, your application code will use whatever logger you've configured there.
|
501
520
|
|
502
521
|
In order to make Racecar log its own operations to a log file, set the `logfile` configuration variable or pass `--log filename.log` to the `racecar` command.
|
503
522
|
|
504
|
-
|
505
523
|
### Operations
|
506
524
|
|
507
525
|
In order to gracefully shut down a Racecar consumer process, send it the `SIGTERM` signal. Most process supervisors such as Runit and Kubernetes send this signal when shutting down a process, so using those systems will make things easier.
|
508
526
|
|
509
527
|
In order to introspect the configuration of a consumer process, send it the `SIGUSR1` signal. This will make Racecar print its configuration to the standard error file descriptor associated with the consumer process, so you'll need to know where that is written to.
|
510
528
|
|
511
|
-
|
512
529
|
### Upgrading from v1 to v2
|
513
530
|
|
514
531
|
In order to safely upgrade from Racecar v1 to v2, you need to completely shut down your consumer group before starting it up again with the v2 Racecar dependency. In general, you should avoid rolling deploys for consumers groups, so it is likely the case that this will just work for you, but it's a good idea to check first.
|
515
532
|
|
516
|
-
|
517
533
|
## Development
|
518
534
|
|
519
535
|
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rspec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
@@ -535,14 +551,12 @@ Please note - your code directory is mounted as a volume, so you can make code c
|
|
535
551
|
|
536
552
|
Bug reports and pull requests are welcome on [GitHub](https://github.com/zendesk/racecar). Feel free to [join our Slack team](https://ruby-kafka-slack.herokuapp.com/) and ask how best to contribute!
|
537
553
|
|
538
|
-
|
539
554
|
## Support and Discussion
|
540
555
|
|
541
556
|
If you've discovered a bug, please file a [Github issue](https://github.com/zendesk/racecar/issues/new), and make sure to include all the relevant information, including the version of Racecar, rdkafka-ruby, and Kafka that you're using.
|
542
557
|
|
543
558
|
If you have other questions, or would like to discuss best practises, or how to contribute to the project, [join our Slack team](https://ruby-kafka-slack.herokuapp.com/)!
|
544
559
|
|
545
|
-
|
546
560
|
## Copyright and license
|
547
561
|
|
548
562
|
Copyright 2017 Daniel Schierbeck & Zendesk
|
data/lib/racecar/config.rb
CHANGED
@@ -223,6 +223,7 @@ module Racecar
|
|
223
223
|
self.parallel_workers = consumer_class.parallel_workers
|
224
224
|
self.subscriptions = consumer_class.subscriptions
|
225
225
|
self.max_wait_time = consumer_class.max_wait_time || self.max_wait_time
|
226
|
+
self.fetch_messages = consumer_class.fetch_messages || self.fetch_messages
|
226
227
|
self.pidfile ||= "#{group_id}.pid"
|
227
228
|
end
|
228
229
|
|
data/lib/racecar/consumer.rb
CHANGED
@@ -9,7 +9,7 @@ module Racecar
|
|
9
9
|
class << self
|
10
10
|
attr_accessor :max_wait_time
|
11
11
|
attr_accessor :group_id
|
12
|
-
attr_accessor :producer, :consumer, :parallel_workers
|
12
|
+
attr_accessor :producer, :consumer, :parallel_workers, :fetch_messages
|
13
13
|
|
14
14
|
def subscriptions
|
15
15
|
@subscriptions ||= []
|
@@ -63,25 +63,27 @@ module Racecar
|
|
63
63
|
|
64
64
|
@instrumenter.instrument('deliver_messages', instrumentation_payload) do
|
65
65
|
@delivery_handles.each do |handle|
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
66
|
+
begin
|
67
|
+
# rdkafka-ruby checks every wait_timeout seconds if the message was
|
68
|
+
# successfully delivered, up to max_wait_timeout seconds before raising
|
69
|
+
# Rdkafka::AbstractHandle::WaitTimeoutError. librdkafka will (re)try to
|
70
|
+
# deliver all messages in the background, until "config.message_timeout"
|
71
|
+
# (message.timeout.ms) is exceeded. Phrased differently, rdkafka-ruby's
|
72
|
+
# WaitTimeoutError is just informative.
|
73
|
+
# The raising can be avoided if max_wait_timeout below is greater than
|
74
|
+
# config.message_timeout, but config is not available here (without
|
75
|
+
# changing the interface).
|
76
|
+
handle.wait(max_wait_timeout: 60, wait_timeout: 0.1)
|
77
|
+
rescue Rdkafka::AbstractHandle::WaitTimeoutError => e
|
78
|
+
partition = MessageDeliveryError.partition_from_delivery_handle(handle)
|
79
|
+
# ideally we could use the logger passed to the Runner, but it is not
|
80
|
+
# available here. The runner sets it for Rdkafka, though, so we can use
|
81
|
+
# that instead.
|
82
|
+
@config.logger.debug "Still trying to deliver message to (partition #{partition})... (will try up to Racecar.config.message_timeout)"
|
83
|
+
retry
|
84
|
+
rescue Rdkafka::RdkafkaError => e
|
85
|
+
raise MessageDeliveryError.new(e, handle)
|
86
|
+
end
|
85
87
|
end
|
86
88
|
end
|
87
89
|
end
|
data/lib/racecar/consumer_set.rb
CHANGED
@@ -156,7 +156,7 @@ module Racecar
|
|
156
156
|
msg = current.poll(max_wait_time_ms)
|
157
157
|
rescue Rdkafka::RdkafkaError => e
|
158
158
|
case e.code
|
159
|
-
when :max_poll_exceeded, :transport # -147, -195
|
159
|
+
when :max_poll_exceeded, :transport, :not_coordinator # -147, -195, 16
|
160
160
|
reset_current_consumer
|
161
161
|
end
|
162
162
|
raise
|
data/lib/racecar/datadog.rb
CHANGED
@@ -0,0 +1,49 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'tempfile'
|
4
|
+
|
5
|
+
# Heroku Kafka addon provides 4 ENVs to connect to their Kafka Broker
|
6
|
+
# KAFKA_TRUSTED_CERT, KAFKA_CLIENT_CERT, KAFKA_CLIENT_CERT_KEY, KAFKA_URL
|
7
|
+
# This will work only if the Heroku Kafka add-on is aliased to "KAFKA"
|
8
|
+
|
9
|
+
$stderr.puts "=> Loading configuration from Heroku Kafka ENVs"
|
10
|
+
|
11
|
+
module Racecar
|
12
|
+
module Heroku
|
13
|
+
def self.load_configuration!
|
14
|
+
[
|
15
|
+
"KAFKA_URL",
|
16
|
+
"KAFKA_TRUSTED_CERT",
|
17
|
+
"KAFKA_CLIENT_CERT",
|
18
|
+
"KAFKA_CLIENT_CERT_KEY"
|
19
|
+
]. each do |env_name|
|
20
|
+
if ENV[env_name].nil?
|
21
|
+
$stderr.puts "Error: ENV #{env_name} is not set"
|
22
|
+
exit 1
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
Racecar.configure do |config|
|
27
|
+
ca_cert = ENV["KAFKA_TRUSTED_CERT"]
|
28
|
+
client_cert = ENV["KAFKA_CLIENT_CERT"]
|
29
|
+
client_cert_key = ENV["KAFKA_CLIENT_CERT_KEY"]
|
30
|
+
|
31
|
+
tmp_file_path = lambda do |data|
|
32
|
+
tempfile = Tempfile.new(['', '.pem'])
|
33
|
+
tempfile << data
|
34
|
+
tempfile.close
|
35
|
+
tempfile.path
|
36
|
+
end
|
37
|
+
|
38
|
+
config.security_protocol = :ssl
|
39
|
+
config.ssl_ca_location = tmp_file_path.call(ca_cert)
|
40
|
+
config.ssl_certificate_location = tmp_file_path.call(client_cert)
|
41
|
+
config.ssl_key_location = tmp_file_path.call(client_cert_key)
|
42
|
+
|
43
|
+
config.brokers = ENV["KAFKA_URL"].to_s.gsub('kafka+ssl://', '').split(',')
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
Racecar::Heroku.load_configuration!
|
data/lib/racecar/runner.rb
CHANGED
data/lib/racecar/version.rb
CHANGED
data/racecar.gemspec
CHANGED
@@ -20,14 +20,16 @@ Gem::Specification.new do |spec|
|
|
20
20
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
21
21
|
spec.require_paths = ["lib"]
|
22
22
|
|
23
|
+
spec.required_ruby_version = '>= 2.4'
|
24
|
+
|
23
25
|
spec.add_runtime_dependency "king_konf", "~> 1.0.0"
|
24
|
-
spec.add_runtime_dependency "rdkafka", "~> 0.
|
26
|
+
spec.add_runtime_dependency "rdkafka", "~> 0.10.0"
|
25
27
|
|
26
28
|
spec.add_development_dependency "bundler", [">= 1.13", "< 3"]
|
27
29
|
spec.add_development_dependency "pry"
|
28
30
|
spec.add_development_dependency "rake", "> 10.0"
|
29
31
|
spec.add_development_dependency "rspec", "~> 3.0"
|
30
32
|
spec.add_development_dependency "timecop"
|
31
|
-
spec.add_development_dependency "dogstatsd-ruby", ">= 4.0.0", "<
|
33
|
+
spec.add_development_dependency "dogstatsd-ruby", ">= 4.0.0", "< 6.0.0"
|
32
34
|
spec.add_development_dependency "activesupport", ">= 4.0", "< 6.1"
|
33
35
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: racecar
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Schierbeck
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: exe
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2022-01-28 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: king_konf
|
@@ -31,14 +31,14 @@ dependencies:
|
|
31
31
|
requirements:
|
32
32
|
- - "~>"
|
33
33
|
- !ruby/object:Gem::Version
|
34
|
-
version: 0.
|
34
|
+
version: 0.10.0
|
35
35
|
type: :runtime
|
36
36
|
prerelease: false
|
37
37
|
version_requirements: !ruby/object:Gem::Requirement
|
38
38
|
requirements:
|
39
39
|
- - "~>"
|
40
40
|
- !ruby/object:Gem::Version
|
41
|
-
version: 0.
|
41
|
+
version: 0.10.0
|
42
42
|
- !ruby/object:Gem::Dependency
|
43
43
|
name: bundler
|
44
44
|
requirement: !ruby/object:Gem::Requirement
|
@@ -124,7 +124,7 @@ dependencies:
|
|
124
124
|
version: 4.0.0
|
125
125
|
- - "<"
|
126
126
|
- !ruby/object:Gem::Version
|
127
|
-
version:
|
127
|
+
version: 6.0.0
|
128
128
|
type: :development
|
129
129
|
prerelease: false
|
130
130
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -134,7 +134,7 @@ dependencies:
|
|
134
134
|
version: 4.0.0
|
135
135
|
- - "<"
|
136
136
|
- !ruby/object:Gem::Version
|
137
|
-
version:
|
137
|
+
version: 6.0.0
|
138
138
|
- !ruby/object:Gem::Dependency
|
139
139
|
name: activesupport
|
140
140
|
requirement: !ruby/object:Gem::Requirement
|
@@ -199,6 +199,7 @@ files:
|
|
199
199
|
- lib/racecar/ctl.rb
|
200
200
|
- lib/racecar/daemon.rb
|
201
201
|
- lib/racecar/datadog.rb
|
202
|
+
- lib/racecar/heroku.rb
|
202
203
|
- lib/racecar/instrumenter.rb
|
203
204
|
- lib/racecar/message.rb
|
204
205
|
- lib/racecar/message_delivery_error.rb
|
@@ -221,12 +222,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
221
222
|
requirements:
|
222
223
|
- - ">="
|
223
224
|
- !ruby/object:Gem::Version
|
224
|
-
version: '
|
225
|
+
version: '2.4'
|
225
226
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
226
227
|
requirements:
|
227
|
-
- - "
|
228
|
+
- - ">="
|
228
229
|
- !ruby/object:Gem::Version
|
229
|
-
version:
|
230
|
+
version: '0'
|
230
231
|
requirements: []
|
231
232
|
rubygems_version: 3.1.2
|
232
233
|
signing_key:
|