racecar 2.0.0 → 2.10.0.beta2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/dependabot.yml +17 -0
- data/.github/workflows/ci.yml +46 -0
- data/.github/workflows/publish.yml +12 -0
- data/.gitignore +1 -2
- data/CHANGELOG.md +83 -1
- data/Dockerfile +9 -0
- data/Gemfile +6 -0
- data/Gemfile.lock +72 -0
- data/README.md +303 -82
- data/Rakefile +5 -0
- data/docker-compose.yml +65 -0
- data/examples/batch_consumer.rb +4 -2
- data/examples/cat_consumer.rb +2 -0
- data/examples/producing_consumer.rb +2 -0
- data/exe/racecar +37 -14
- data/extra/datadog-dashboard.json +1 -0
- data/lib/ensure_hash_compact.rb +2 -0
- data/lib/generators/racecar/consumer_generator.rb +2 -0
- data/lib/generators/racecar/install_generator.rb +2 -0
- data/lib/racecar/cli.rb +26 -21
- data/lib/racecar/config.rb +80 -4
- data/lib/racecar/consumer.rb +51 -6
- data/lib/racecar/consumer_set.rb +113 -44
- data/lib/racecar/ctl.rb +31 -3
- data/lib/racecar/daemon.rb +4 -2
- data/lib/racecar/datadog.rb +83 -3
- data/lib/racecar/delivery_callback.rb +27 -0
- data/lib/racecar/erroneous_state_error.rb +34 -0
- data/lib/racecar/heroku.rb +49 -0
- data/lib/racecar/instrumenter.rb +4 -7
- data/lib/racecar/liveness_probe.rb +78 -0
- data/lib/racecar/message.rb +6 -1
- data/lib/racecar/message_delivery_error.rb +112 -0
- data/lib/racecar/null_instrumenter.rb +2 -0
- data/lib/racecar/parallel_runner.rb +110 -0
- data/lib/racecar/pause.rb +8 -4
- data/lib/racecar/producer.rb +139 -0
- data/lib/racecar/rails_config_file_loader.rb +7 -1
- data/lib/racecar/rebalance_listener.rb +58 -0
- data/lib/racecar/runner.rb +79 -37
- data/lib/racecar/version.rb +3 -1
- data/lib/racecar.rb +36 -8
- data/racecar.gemspec +7 -4
- metadata +47 -25
- data/.github/workflows/rspec.yml +0 -24
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 2277b7e88416d37c97f2467738360dcd92a52ea680a24e7a2aa6610883297917
|
|
4
|
+
data.tar.gz: fbd34f82fe0f4b75b4e1672073e8b92cd7f9b211b8361e6469af496aa9e6b338
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 6506e65b5380e1dbb71c91ece7065274f36c6f1afe880e529f9e811bd1c5dcb4ac0c81728194b07a92e4b125c0731d2d3a896bb61239d7a5977d33351ebaed4b
|
|
7
|
+
data.tar.gz: f4d19863c9cd0012a5542574d2f1c3b95a5dd8916a6176a6aff83712098da633b3fe43df4ef8c4e74ce2939510d169cbdf9fa4b58bdedcc8a8e686371cfd1b8b
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
version: 2
|
|
2
|
+
registries:
|
|
3
|
+
artifactory:
|
|
4
|
+
type: rubygems-server
|
|
5
|
+
url: ${{secrets.ARTIFACTORY_URL}}
|
|
6
|
+
username: ${{secrets.ARTIFACTORY_USERNAME}}
|
|
7
|
+
password: ${{secrets.ARTIFACTORY_API_KEY}}
|
|
8
|
+
updates:
|
|
9
|
+
- package-ecosystem: "bundler"
|
|
10
|
+
vendor: true
|
|
11
|
+
directory: "/"
|
|
12
|
+
registries:
|
|
13
|
+
- artifactory
|
|
14
|
+
schedule:
|
|
15
|
+
interval: "daily"
|
|
16
|
+
allow:
|
|
17
|
+
- dependency-type: "direct" # only explicitly defined dependencies.
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
pull_request:
|
|
5
|
+
branches: ["master"]
|
|
6
|
+
push:
|
|
7
|
+
branches: ["master"]
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
unit-specs:
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
|
|
13
|
+
strategy:
|
|
14
|
+
matrix:
|
|
15
|
+
ruby-version:
|
|
16
|
+
- "2.7"
|
|
17
|
+
- "3.0"
|
|
18
|
+
- "3.1"
|
|
19
|
+
- "3.2"
|
|
20
|
+
- "3.3.0-preview2"
|
|
21
|
+
|
|
22
|
+
steps:
|
|
23
|
+
- uses: zendesk/checkout@v3
|
|
24
|
+
- name: Set up Ruby
|
|
25
|
+
uses: zendesk/setup-ruby@v1
|
|
26
|
+
with:
|
|
27
|
+
ruby-version: ${{ matrix.ruby-version }}
|
|
28
|
+
bundler-cache: true
|
|
29
|
+
- name: Build and test with RSpec
|
|
30
|
+
run: bundle exec rspec --exclude-pattern='spec/integration/*_spec.rb'
|
|
31
|
+
|
|
32
|
+
integration-specs:
|
|
33
|
+
runs-on: ubuntu-latest
|
|
34
|
+
steps:
|
|
35
|
+
- uses: zendesk/checkout@v3
|
|
36
|
+
- name: Set up Ruby
|
|
37
|
+
uses: zendesk/setup-ruby@v1
|
|
38
|
+
with:
|
|
39
|
+
ruby-version: "2.7"
|
|
40
|
+
bundler-cache: true
|
|
41
|
+
- name: Bring up docker-compose stack
|
|
42
|
+
run: docker-compose up -d
|
|
43
|
+
- name: Build and test with RSpec
|
|
44
|
+
env:
|
|
45
|
+
RACECAR_BROKERS: localhost:9092
|
|
46
|
+
run: timeout --kill-after 180 150 bundle exec rspec spec/integration/*_spec.rb
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
name: Publish Gem
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags: v*
|
|
6
|
+
|
|
7
|
+
jobs:
|
|
8
|
+
call-workflow:
|
|
9
|
+
uses: zendesk/gw/.github/workflows/ruby-gem-publication.yml@main
|
|
10
|
+
secrets:
|
|
11
|
+
RUBY_GEMS_API_KEY: ${{ secrets.RUBY_GEMS_API_KEY }}
|
|
12
|
+
RUBY_GEMS_TOTP_DEVICE: ${{ secrets.RUBY_GEMS_TOTP_DEVICE }}
|
data/.gitignore
CHANGED
data/CHANGELOG.md
CHANGED
|
@@ -1,8 +1,88 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## Unreleased
|
|
4
|
+
|
|
5
|
+
## 2.10.0.beta2
|
|
6
|
+
|
|
7
|
+
* Don't load rails env for liveness probe
|
|
8
|
+
* Resolve Rails 7.1 logger incompatibility
|
|
9
|
+
* Test with Ruby 3.3 preview
|
|
10
|
+
|
|
11
|
+
## 2.10.0.beta1
|
|
12
|
+
|
|
13
|
+
* Bump rdkafka gem version to 0.13.0
|
|
14
|
+
* Support cooperative-sticky
|
|
15
|
+
* Instrument produce delivery errors
|
|
16
|
+
* Fix config load for liveness probe
|
|
17
|
+
* Send exceptions to `process_batch` instrumenter
|
|
18
|
+
* Docker test fixes
|
|
19
|
+
* Test with Ruby 3.2
|
|
20
|
+
|
|
21
|
+
## v2.9.0, v2.9.0.beta1
|
|
22
|
+
|
|
23
|
+
* Add `partitioner` producer config option to allow changing the strategy to
|
|
24
|
+
determine which topic partition a message is written to when racecar
|
|
25
|
+
produces a kafka message
|
|
26
|
+
* Add built-in liveness probe for Kubernetes deployments.
|
|
27
|
+
|
|
28
|
+
## v2.8.2
|
|
29
|
+
* Handles ErroneousStateError, in previous versions the consumer would do several unecessary group leave/joins. The log level is also changed to WARN instead of ERROR. ([#295](https://github.com/zendesk/racecar/pull/295))
|
|
30
|
+
|
|
31
|
+
## v2.8.1
|
|
32
|
+
* Adds new ErroneousStateError to racecar in order to give more information on this new possible exception.
|
|
33
|
+
|
|
34
|
+
## v2.8.0
|
|
35
|
+
* Update librdkafka version from 1.8.2 to 1.9.0 by upgrading from rdkafka 0.10.0 to 0.12.0. ([#293](https://github.com/zendesk/racecar/pull/293))
|
|
36
|
+
|
|
37
|
+
## v2.7.0
|
|
38
|
+
|
|
39
|
+
* Update librdkafka version from 1.5.0 to 1.8.2 by upgrading from rdkafka 0.10.0 to 0.11.1. ([#283](https://github.com/zendesk/racecar/pull/283))
|
|
40
|
+
* Bumps minimum required Ruby version to 2.6
|
|
41
|
+
|
|
42
|
+
## v2.6.0
|
|
43
|
+
|
|
44
|
+
* Add capability to specify partition number when producing messages
|
|
45
|
+
|
|
46
|
+
## v2.5.0
|
|
47
|
+
|
|
48
|
+
* `fetch_messages` can be configured per consumer, just as the maximum timeout to wait for a full batch.
|
|
49
|
+
|
|
50
|
+
## v2.4.0
|
|
51
|
+
|
|
52
|
+
* Update librdkafka version from 1.4.0 to 1.5.0 by upgrading from rdkafka 0.8.0 to 0.10.0. ([#263](https://github.com/zendesk/racecar/pull/263))
|
|
53
|
+
* Restore support for Ruby 2.4 (#258)
|
|
54
|
+
|
|
55
|
+
## racecar v2.3.1
|
|
56
|
+
|
|
57
|
+
* Handle `ERR_NOT_COORDINATOR` (#209)
|
|
58
|
+
|
|
59
|
+
## racecar v2.3.0
|
|
60
|
+
|
|
61
|
+
* Add native support for Heroku (#248)
|
|
62
|
+
* [Racecar::Consumer] When messages fail to deliver, an extended error with hints is now raised. Instead of `Rdkafka::RdkafkaError` you'll get a `Racecar::MessageDeliveryError` instead. ([#219](https://github.com/zendesk/racecar/pull/219)). If you have set a `Racecar.config.error_handler`, it might need to be updated.
|
|
63
|
+
* [Racecar::Consumer] When message delivery times out, Racecar will reset the producer in an attempt to fix some of the potential causes for this error. ([#219](https://github.com/zendesk/racecar/pull/219))
|
|
64
|
+
* Validate the `process` and `process_batch` method signature on consumer classes when initializing (#236)
|
|
65
|
+
* Add Ruby 3.0 compatibility (#237)
|
|
66
|
+
* Introduce parallel runner, which forks a number of independent consumers, allowing partitions to be processed in parallel. ([#222](https://github.com/zendesk/racecar/pull/222))
|
|
67
|
+
* [Racecar::Runner] Ensure producer is closed, whether it closes or errors. ([#222](https://github.com/zendesk/racecar/pull/222))
|
|
68
|
+
* Configure `statistics_interval` directly in the config. Disable statistics when no callback is defined ([#232](https://github.com/zendesk/racecar/pull/232))
|
|
69
|
+
|
|
70
|
+
## racecar v2.2.0
|
|
71
|
+
|
|
72
|
+
* [Racecar::ConsumerSet] **breaking change** `Racecar::ConsumerSet`'s functions `poll` and `batch_pall` expect the max wait values to be given in milliseconds. The defaults were using `config.max_wait_time`, which is in seconds. If you do not directly use `Racecar::ConsumerSet`, or always call its `poll` and `batch_poll` functions by specfiying the max wait time (the first argument), then this breaking change does not affect you. ([#214](https://github.com/zendesk/racecar/pull/214))
|
|
73
|
+
|
|
74
|
+
## racecar v2.1.1
|
|
75
|
+
|
|
76
|
+
* [Bugfix] Close RdKafka consumer in ConsumerSet#reset_current_consumer to prevent memory leak (#196)
|
|
77
|
+
* [Bugfix] `poll`/`batch_poll` would not retry in edge cases and raise immediately. They still honor the `max_wait_time` setting, but might return no messages instead and only retry on their next call. ([#177](https://github.com/zendesk/racecar/pull/177))
|
|
78
|
+
|
|
79
|
+
## racecar v2.1.0
|
|
80
|
+
|
|
81
|
+
* Bump rdkafka to 0.8.0 (#191)
|
|
82
|
+
|
|
3
83
|
## racecar v2.0.0
|
|
4
84
|
|
|
5
|
-
* Replace `ruby-kafka` with `rdkafka-ruby` as the low-level library underneath Racecar.
|
|
85
|
+
* Replace `ruby-kafka` with `rdkafka-ruby` as the low-level library underneath Racecar (#91).
|
|
6
86
|
* Fix `max_wait_time` usage (#179).
|
|
7
87
|
* Removed config option `sasl_over_ssl`.
|
|
8
88
|
* [Racecar::Consumer] Do not pause consuming partitions on exception.
|
|
@@ -26,6 +106,8 @@
|
|
|
26
106
|
* [Instrumentation] if processors define a `statistics_callback`, it will be called once every second for every subscription or producer connection. The first argument will be a Hash, for contents see [librdkafka STATISTICS.md](https://github.com/edenhill/librdkafka/blob/master/STATISTICS.md).
|
|
27
107
|
* Add current directory to `$LOAD_PATH` only when `--require` option is used (#117).
|
|
28
108
|
* Remove manual heartbeat support, see [Long-running message processing section in README](README.md#long-running-message-processing).
|
|
109
|
+
* Rescue exceptions--then log and pass to `on_error`--at the outermost level of `exe/racecar`, so that exceptions raised outside `Cli.run` are not silently discarded (#186).
|
|
110
|
+
* When exceptions with a `cause` are logged, recursively log the `cause` detail, separated by `--- Caused by: ---\n`.
|
|
29
111
|
|
|
30
112
|
## racecar v1.0.0
|
|
31
113
|
|
data/Dockerfile
ADDED
data/Gemfile
CHANGED
|
@@ -1,4 +1,10 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
source 'https://rubygems.org'
|
|
2
4
|
|
|
3
5
|
# Specify your gem's dependencies in racecar.gemspec
|
|
4
6
|
gemspec
|
|
7
|
+
|
|
8
|
+
# We actually support version 7.x (see gemspec); this extra restriction is added just for running the test suite also
|
|
9
|
+
# on Ruby 2.6, which activesupport 7.0 does not support.
|
|
10
|
+
gem 'activesupport', '~> 6.1.0'
|
data/Gemfile.lock
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
PATH
|
|
2
|
+
remote: .
|
|
3
|
+
specs:
|
|
4
|
+
racecar (2.10.0.beta2)
|
|
5
|
+
king_konf (~> 1.0.0)
|
|
6
|
+
rdkafka (~> 0.13.0)
|
|
7
|
+
|
|
8
|
+
GEM
|
|
9
|
+
remote: https://rubygems.org/
|
|
10
|
+
specs:
|
|
11
|
+
activesupport (6.1.7.3)
|
|
12
|
+
concurrent-ruby (~> 1.0, >= 1.0.2)
|
|
13
|
+
i18n (>= 1.6, < 2)
|
|
14
|
+
minitest (>= 5.1)
|
|
15
|
+
tzinfo (~> 2.0)
|
|
16
|
+
zeitwerk (~> 2.3)
|
|
17
|
+
byebug (11.1.3)
|
|
18
|
+
coderay (1.1.3)
|
|
19
|
+
concurrent-ruby (1.2.2)
|
|
20
|
+
diff-lcs (1.5.0)
|
|
21
|
+
dogstatsd-ruby (5.5.0)
|
|
22
|
+
ffi (1.15.5)
|
|
23
|
+
i18n (1.12.0)
|
|
24
|
+
concurrent-ruby (~> 1.0)
|
|
25
|
+
king_konf (1.0.1)
|
|
26
|
+
method_source (1.0.0)
|
|
27
|
+
mini_portile2 (2.8.1)
|
|
28
|
+
minitest (5.18.0)
|
|
29
|
+
pry (0.14.2)
|
|
30
|
+
coderay (~> 1.1)
|
|
31
|
+
method_source (~> 1.0)
|
|
32
|
+
pry-byebug (3.10.1)
|
|
33
|
+
byebug (~> 11.0)
|
|
34
|
+
pry (>= 0.13, < 0.15)
|
|
35
|
+
rake (13.0.6)
|
|
36
|
+
rdkafka (0.13.0)
|
|
37
|
+
ffi (~> 1.15)
|
|
38
|
+
mini_portile2 (~> 2.6)
|
|
39
|
+
rake (> 12)
|
|
40
|
+
rspec (3.12.0)
|
|
41
|
+
rspec-core (~> 3.12.0)
|
|
42
|
+
rspec-expectations (~> 3.12.0)
|
|
43
|
+
rspec-mocks (~> 3.12.0)
|
|
44
|
+
rspec-core (3.12.1)
|
|
45
|
+
rspec-support (~> 3.12.0)
|
|
46
|
+
rspec-expectations (3.12.2)
|
|
47
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
|
48
|
+
rspec-support (~> 3.12.0)
|
|
49
|
+
rspec-mocks (3.12.4)
|
|
50
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
|
51
|
+
rspec-support (~> 3.12.0)
|
|
52
|
+
rspec-support (3.12.0)
|
|
53
|
+
timecop (0.9.6)
|
|
54
|
+
tzinfo (2.0.6)
|
|
55
|
+
concurrent-ruby (~> 1.0)
|
|
56
|
+
zeitwerk (2.6.7)
|
|
57
|
+
|
|
58
|
+
PLATFORMS
|
|
59
|
+
ruby
|
|
60
|
+
|
|
61
|
+
DEPENDENCIES
|
|
62
|
+
activesupport (~> 6.1.0)
|
|
63
|
+
bundler (>= 1.13, < 3)
|
|
64
|
+
dogstatsd-ruby (>= 4.0.0, < 6.0.0)
|
|
65
|
+
pry-byebug
|
|
66
|
+
racecar!
|
|
67
|
+
rake (> 10.0)
|
|
68
|
+
rspec (~> 3.0)
|
|
69
|
+
timecop
|
|
70
|
+
|
|
71
|
+
BUNDLED WITH
|
|
72
|
+
2.4.9
|