racecar 2.0.0 → 2.10.0.beta2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/dependabot.yml +17 -0
- data/.github/workflows/ci.yml +46 -0
- data/.github/workflows/publish.yml +12 -0
- data/.gitignore +1 -2
- data/CHANGELOG.md +83 -1
- data/Dockerfile +9 -0
- data/Gemfile +6 -0
- data/Gemfile.lock +72 -0
- data/README.md +303 -82
- data/Rakefile +5 -0
- data/docker-compose.yml +65 -0
- data/examples/batch_consumer.rb +4 -2
- data/examples/cat_consumer.rb +2 -0
- data/examples/producing_consumer.rb +2 -0
- data/exe/racecar +37 -14
- data/extra/datadog-dashboard.json +1 -0
- data/lib/ensure_hash_compact.rb +2 -0
- data/lib/generators/racecar/consumer_generator.rb +2 -0
- data/lib/generators/racecar/install_generator.rb +2 -0
- data/lib/racecar/cli.rb +26 -21
- data/lib/racecar/config.rb +80 -4
- data/lib/racecar/consumer.rb +51 -6
- data/lib/racecar/consumer_set.rb +113 -44
- data/lib/racecar/ctl.rb +31 -3
- data/lib/racecar/daemon.rb +4 -2
- data/lib/racecar/datadog.rb +83 -3
- data/lib/racecar/delivery_callback.rb +27 -0
- data/lib/racecar/erroneous_state_error.rb +34 -0
- data/lib/racecar/heroku.rb +49 -0
- data/lib/racecar/instrumenter.rb +4 -7
- data/lib/racecar/liveness_probe.rb +78 -0
- data/lib/racecar/message.rb +6 -1
- data/lib/racecar/message_delivery_error.rb +112 -0
- data/lib/racecar/null_instrumenter.rb +2 -0
- data/lib/racecar/parallel_runner.rb +110 -0
- data/lib/racecar/pause.rb +8 -4
- data/lib/racecar/producer.rb +139 -0
- data/lib/racecar/rails_config_file_loader.rb +7 -1
- data/lib/racecar/rebalance_listener.rb +58 -0
- data/lib/racecar/runner.rb +79 -37
- data/lib/racecar/version.rb +3 -1
- data/lib/racecar.rb +36 -8
- data/racecar.gemspec +7 -4
- metadata +47 -25
- data/.github/workflows/rspec.yml +0 -24
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 2277b7e88416d37c97f2467738360dcd92a52ea680a24e7a2aa6610883297917
|
|
4
|
+
data.tar.gz: fbd34f82fe0f4b75b4e1672073e8b92cd7f9b211b8361e6469af496aa9e6b338
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 6506e65b5380e1dbb71c91ece7065274f36c6f1afe880e529f9e811bd1c5dcb4ac0c81728194b07a92e4b125c0731d2d3a896bb61239d7a5977d33351ebaed4b
|
|
7
|
+
data.tar.gz: f4d19863c9cd0012a5542574d2f1c3b95a5dd8916a6176a6aff83712098da633b3fe43df4ef8c4e74ce2939510d169cbdf9fa4b58bdedcc8a8e686371cfd1b8b
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
version: 2
|
|
2
|
+
registries:
|
|
3
|
+
artifactory:
|
|
4
|
+
type: rubygems-server
|
|
5
|
+
url: ${{secrets.ARTIFACTORY_URL}}
|
|
6
|
+
username: ${{secrets.ARTIFACTORY_USERNAME}}
|
|
7
|
+
password: ${{secrets.ARTIFACTORY_API_KEY}}
|
|
8
|
+
updates:
|
|
9
|
+
- package-ecosystem: "bundler"
|
|
10
|
+
vendor: true
|
|
11
|
+
directory: "/"
|
|
12
|
+
registries:
|
|
13
|
+
- artifactory
|
|
14
|
+
schedule:
|
|
15
|
+
interval: "daily"
|
|
16
|
+
allow:
|
|
17
|
+
- dependency-type: "direct" # only explicitly defined dependencies.
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
pull_request:
|
|
5
|
+
branches: ["master"]
|
|
6
|
+
push:
|
|
7
|
+
branches: ["master"]
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
unit-specs:
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
|
|
13
|
+
strategy:
|
|
14
|
+
matrix:
|
|
15
|
+
ruby-version:
|
|
16
|
+
- "2.7"
|
|
17
|
+
- "3.0"
|
|
18
|
+
- "3.1"
|
|
19
|
+
- "3.2"
|
|
20
|
+
- "3.3.0-preview2"
|
|
21
|
+
|
|
22
|
+
steps:
|
|
23
|
+
- uses: zendesk/checkout@v3
|
|
24
|
+
- name: Set up Ruby
|
|
25
|
+
uses: zendesk/setup-ruby@v1
|
|
26
|
+
with:
|
|
27
|
+
ruby-version: ${{ matrix.ruby-version }}
|
|
28
|
+
bundler-cache: true
|
|
29
|
+
- name: Build and test with RSpec
|
|
30
|
+
run: bundle exec rspec --exclude-pattern='spec/integration/*_spec.rb'
|
|
31
|
+
|
|
32
|
+
integration-specs:
|
|
33
|
+
runs-on: ubuntu-latest
|
|
34
|
+
steps:
|
|
35
|
+
- uses: zendesk/checkout@v3
|
|
36
|
+
- name: Set up Ruby
|
|
37
|
+
uses: zendesk/setup-ruby@v1
|
|
38
|
+
with:
|
|
39
|
+
ruby-version: "2.7"
|
|
40
|
+
bundler-cache: true
|
|
41
|
+
- name: Bring up docker-compose stack
|
|
42
|
+
run: docker-compose up -d
|
|
43
|
+
- name: Build and test with RSpec
|
|
44
|
+
env:
|
|
45
|
+
RACECAR_BROKERS: localhost:9092
|
|
46
|
+
run: timeout --kill-after 180 150 bundle exec rspec spec/integration/*_spec.rb
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
name: Publish Gem
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags: v*
|
|
6
|
+
|
|
7
|
+
jobs:
|
|
8
|
+
call-workflow:
|
|
9
|
+
uses: zendesk/gw/.github/workflows/ruby-gem-publication.yml@main
|
|
10
|
+
secrets:
|
|
11
|
+
RUBY_GEMS_API_KEY: ${{ secrets.RUBY_GEMS_API_KEY }}
|
|
12
|
+
RUBY_GEMS_TOTP_DEVICE: ${{ secrets.RUBY_GEMS_TOTP_DEVICE }}
|
data/.gitignore
CHANGED
data/CHANGELOG.md
CHANGED
|
@@ -1,8 +1,88 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## Unreleased
|
|
4
|
+
|
|
5
|
+
## 2.10.0.beta2
|
|
6
|
+
|
|
7
|
+
* Don't load rails env for liveness probe
|
|
8
|
+
* Resolve Rails 7.1 logger incompatibility
|
|
9
|
+
* Test with Ruby 3.3 preview
|
|
10
|
+
|
|
11
|
+
## 2.10.0.beta1
|
|
12
|
+
|
|
13
|
+
* Bump rdkafka gem version to 0.13.0
|
|
14
|
+
* Support cooperative-sticky
|
|
15
|
+
* Instrument produce delivery errors
|
|
16
|
+
* Fix config load for liveness probe
|
|
17
|
+
* Send exceptions to `process_batch` instrumenter
|
|
18
|
+
* Docker test fixes
|
|
19
|
+
* Test with Ruby 3.2
|
|
20
|
+
|
|
21
|
+
## v2.9.0, v2.9.0.beta1
|
|
22
|
+
|
|
23
|
+
* Add `partitioner` producer config option to allow changing the strategy to
|
|
24
|
+
determine which topic partition a message is written to when racecar
|
|
25
|
+
produces a kafka message
|
|
26
|
+
* Add built-in liveness probe for Kubernetes deployments.
|
|
27
|
+
|
|
28
|
+
## v2.8.2
|
|
29
|
+
* Handles ErroneousStateError, in previous versions the consumer would do several unecessary group leave/joins. The log level is also changed to WARN instead of ERROR. ([#295](https://github.com/zendesk/racecar/pull/295))
|
|
30
|
+
|
|
31
|
+
## v2.8.1
|
|
32
|
+
* Adds new ErroneousStateError to racecar in order to give more information on this new possible exception.
|
|
33
|
+
|
|
34
|
+
## v2.8.0
|
|
35
|
+
* Update librdkafka version from 1.8.2 to 1.9.0 by upgrading from rdkafka 0.10.0 to 0.12.0. ([#293](https://github.com/zendesk/racecar/pull/293))
|
|
36
|
+
|
|
37
|
+
## v2.7.0
|
|
38
|
+
|
|
39
|
+
* Update librdkafka version from 1.5.0 to 1.8.2 by upgrading from rdkafka 0.10.0 to 0.11.1. ([#283](https://github.com/zendesk/racecar/pull/283))
|
|
40
|
+
* Bumps minimum required Ruby version to 2.6
|
|
41
|
+
|
|
42
|
+
## v2.6.0
|
|
43
|
+
|
|
44
|
+
* Add capability to specify partition number when producing messages
|
|
45
|
+
|
|
46
|
+
## v2.5.0
|
|
47
|
+
|
|
48
|
+
* `fetch_messages` can be configured per consumer, just as the maximum timeout to wait for a full batch.
|
|
49
|
+
|
|
50
|
+
## v2.4.0
|
|
51
|
+
|
|
52
|
+
* Update librdkafka version from 1.4.0 to 1.5.0 by upgrading from rdkafka 0.8.0 to 0.10.0. ([#263](https://github.com/zendesk/racecar/pull/263))
|
|
53
|
+
* Restore support for Ruby 2.4 (#258)
|
|
54
|
+
|
|
55
|
+
## racecar v2.3.1
|
|
56
|
+
|
|
57
|
+
* Handle `ERR_NOT_COORDINATOR` (#209)
|
|
58
|
+
|
|
59
|
+
## racecar v2.3.0
|
|
60
|
+
|
|
61
|
+
* Add native support for Heroku (#248)
|
|
62
|
+
* [Racecar::Consumer] When messages fail to deliver, an extended error with hints is now raised. Instead of `Rdkafka::RdkafkaError` you'll get a `Racecar::MessageDeliveryError` instead. ([#219](https://github.com/zendesk/racecar/pull/219)). If you have set a `Racecar.config.error_handler`, it might need to be updated.
|
|
63
|
+
* [Racecar::Consumer] When message delivery times out, Racecar will reset the producer in an attempt to fix some of the potential causes for this error. ([#219](https://github.com/zendesk/racecar/pull/219))
|
|
64
|
+
* Validate the `process` and `process_batch` method signature on consumer classes when initializing (#236)
|
|
65
|
+
* Add Ruby 3.0 compatibility (#237)
|
|
66
|
+
* Introduce parallel runner, which forks a number of independent consumers, allowing partitions to be processed in parallel. ([#222](https://github.com/zendesk/racecar/pull/222))
|
|
67
|
+
* [Racecar::Runner] Ensure producer is closed, whether it closes or errors. ([#222](https://github.com/zendesk/racecar/pull/222))
|
|
68
|
+
* Configure `statistics_interval` directly in the config. Disable statistics when no callback is defined ([#232](https://github.com/zendesk/racecar/pull/232))
|
|
69
|
+
|
|
70
|
+
## racecar v2.2.0
|
|
71
|
+
|
|
72
|
+
* [Racecar::ConsumerSet] **breaking change** `Racecar::ConsumerSet`'s functions `poll` and `batch_pall` expect the max wait values to be given in milliseconds. The defaults were using `config.max_wait_time`, which is in seconds. If you do not directly use `Racecar::ConsumerSet`, or always call its `poll` and `batch_poll` functions by specfiying the max wait time (the first argument), then this breaking change does not affect you. ([#214](https://github.com/zendesk/racecar/pull/214))
|
|
73
|
+
|
|
74
|
+
## racecar v2.1.1
|
|
75
|
+
|
|
76
|
+
* [Bugfix] Close RdKafka consumer in ConsumerSet#reset_current_consumer to prevent memory leak (#196)
|
|
77
|
+
* [Bugfix] `poll`/`batch_poll` would not retry in edge cases and raise immediately. They still honor the `max_wait_time` setting, but might return no messages instead and only retry on their next call. ([#177](https://github.com/zendesk/racecar/pull/177))
|
|
78
|
+
|
|
79
|
+
## racecar v2.1.0
|
|
80
|
+
|
|
81
|
+
* Bump rdkafka to 0.8.0 (#191)
|
|
82
|
+
|
|
3
83
|
## racecar v2.0.0
|
|
4
84
|
|
|
5
|
-
* Replace `ruby-kafka` with `rdkafka-ruby` as the low-level library underneath Racecar.
|
|
85
|
+
* Replace `ruby-kafka` with `rdkafka-ruby` as the low-level library underneath Racecar (#91).
|
|
6
86
|
* Fix `max_wait_time` usage (#179).
|
|
7
87
|
* Removed config option `sasl_over_ssl`.
|
|
8
88
|
* [Racecar::Consumer] Do not pause consuming partitions on exception.
|
|
@@ -26,6 +106,8 @@
|
|
|
26
106
|
* [Instrumentation] if processors define a `statistics_callback`, it will be called once every second for every subscription or producer connection. The first argument will be a Hash, for contents see [librdkafka STATISTICS.md](https://github.com/edenhill/librdkafka/blob/master/STATISTICS.md).
|
|
27
107
|
* Add current directory to `$LOAD_PATH` only when `--require` option is used (#117).
|
|
28
108
|
* Remove manual heartbeat support, see [Long-running message processing section in README](README.md#long-running-message-processing).
|
|
109
|
+
* Rescue exceptions--then log and pass to `on_error`--at the outermost level of `exe/racecar`, so that exceptions raised outside `Cli.run` are not silently discarded (#186).
|
|
110
|
+
* When exceptions with a `cause` are logged, recursively log the `cause` detail, separated by `--- Caused by: ---\n`.
|
|
29
111
|
|
|
30
112
|
## racecar v1.0.0
|
|
31
113
|
|
data/Dockerfile
ADDED
data/Gemfile
CHANGED
|
@@ -1,4 +1,10 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
source 'https://rubygems.org'
|
|
2
4
|
|
|
3
5
|
# Specify your gem's dependencies in racecar.gemspec
|
|
4
6
|
gemspec
|
|
7
|
+
|
|
8
|
+
# We actually support version 7.x (see gemspec); this extra restriction is added just for running the test suite also
|
|
9
|
+
# on Ruby 2.6, which activesupport 7.0 does not support.
|
|
10
|
+
gem 'activesupport', '~> 6.1.0'
|
data/Gemfile.lock
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
PATH
|
|
2
|
+
remote: .
|
|
3
|
+
specs:
|
|
4
|
+
racecar (2.10.0.beta2)
|
|
5
|
+
king_konf (~> 1.0.0)
|
|
6
|
+
rdkafka (~> 0.13.0)
|
|
7
|
+
|
|
8
|
+
GEM
|
|
9
|
+
remote: https://rubygems.org/
|
|
10
|
+
specs:
|
|
11
|
+
activesupport (6.1.7.3)
|
|
12
|
+
concurrent-ruby (~> 1.0, >= 1.0.2)
|
|
13
|
+
i18n (>= 1.6, < 2)
|
|
14
|
+
minitest (>= 5.1)
|
|
15
|
+
tzinfo (~> 2.0)
|
|
16
|
+
zeitwerk (~> 2.3)
|
|
17
|
+
byebug (11.1.3)
|
|
18
|
+
coderay (1.1.3)
|
|
19
|
+
concurrent-ruby (1.2.2)
|
|
20
|
+
diff-lcs (1.5.0)
|
|
21
|
+
dogstatsd-ruby (5.5.0)
|
|
22
|
+
ffi (1.15.5)
|
|
23
|
+
i18n (1.12.0)
|
|
24
|
+
concurrent-ruby (~> 1.0)
|
|
25
|
+
king_konf (1.0.1)
|
|
26
|
+
method_source (1.0.0)
|
|
27
|
+
mini_portile2 (2.8.1)
|
|
28
|
+
minitest (5.18.0)
|
|
29
|
+
pry (0.14.2)
|
|
30
|
+
coderay (~> 1.1)
|
|
31
|
+
method_source (~> 1.0)
|
|
32
|
+
pry-byebug (3.10.1)
|
|
33
|
+
byebug (~> 11.0)
|
|
34
|
+
pry (>= 0.13, < 0.15)
|
|
35
|
+
rake (13.0.6)
|
|
36
|
+
rdkafka (0.13.0)
|
|
37
|
+
ffi (~> 1.15)
|
|
38
|
+
mini_portile2 (~> 2.6)
|
|
39
|
+
rake (> 12)
|
|
40
|
+
rspec (3.12.0)
|
|
41
|
+
rspec-core (~> 3.12.0)
|
|
42
|
+
rspec-expectations (~> 3.12.0)
|
|
43
|
+
rspec-mocks (~> 3.12.0)
|
|
44
|
+
rspec-core (3.12.1)
|
|
45
|
+
rspec-support (~> 3.12.0)
|
|
46
|
+
rspec-expectations (3.12.2)
|
|
47
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
|
48
|
+
rspec-support (~> 3.12.0)
|
|
49
|
+
rspec-mocks (3.12.4)
|
|
50
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
|
51
|
+
rspec-support (~> 3.12.0)
|
|
52
|
+
rspec-support (3.12.0)
|
|
53
|
+
timecop (0.9.6)
|
|
54
|
+
tzinfo (2.0.6)
|
|
55
|
+
concurrent-ruby (~> 1.0)
|
|
56
|
+
zeitwerk (2.6.7)
|
|
57
|
+
|
|
58
|
+
PLATFORMS
|
|
59
|
+
ruby
|
|
60
|
+
|
|
61
|
+
DEPENDENCIES
|
|
62
|
+
activesupport (~> 6.1.0)
|
|
63
|
+
bundler (>= 1.13, < 3)
|
|
64
|
+
dogstatsd-ruby (>= 4.0.0, < 6.0.0)
|
|
65
|
+
pry-byebug
|
|
66
|
+
racecar!
|
|
67
|
+
rake (> 10.0)
|
|
68
|
+
rspec (~> 3.0)
|
|
69
|
+
timecop
|
|
70
|
+
|
|
71
|
+
BUNDLED WITH
|
|
72
|
+
2.4.9
|