racecar 2.3.0 → 2.6.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +1 -1
- data/CHANGELOG.md +17 -0
- data/Gemfile +4 -0
- data/Gemfile.lock +14 -16
- data/README.md +4 -0
- data/lib/racecar/config.rb +1 -0
- data/lib/racecar/consumer.rb +25 -21
- data/lib/racecar/consumer_set.rb +1 -1
- data/lib/racecar/datadog.rb +5 -1
- data/lib/racecar/runner.rb +1 -0
- data/lib/racecar/version.rb +1 -1
- data/racecar.gemspec +4 -2
- metadata +12 -12
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d0bbfee00d05236ed92c00a93176b230086eb91d1eba33a567a6afef91998851
|
4
|
+
data.tar.gz: f787c5a8ed72301351c8dc6428bd02303f4489cbfe98133f60c546f39f6bf2cf
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7fd5d70a55050add5a8862d32717ecdadb3d75d9cab93b2d89d8be9bd543bd31ded571ce8685d6db8f7ee5b0997520fe21986046c72a656e4323c0e923de4c5e
|
7
|
+
data.tar.gz: 5e6f929613dda6118cade9b5d486883f935ec8b18538860b4edd4439f460c10a0dda9eb296195b9cef5b87750c9e9ca268f6b8fe2166deb9b64e597d9adddd85
|
data/.github/workflows/ci.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -2,6 +2,23 @@
|
|
2
2
|
|
3
3
|
## Unreleased
|
4
4
|
|
5
|
+
## v2.6.0
|
6
|
+
|
7
|
+
* Add capability to specify partition number when producing messages
|
8
|
+
|
9
|
+
## v2.5.0
|
10
|
+
|
11
|
+
* `fetch_messages` can be configured per consumer, just as the maximum timeout to wait for a full batch.
|
12
|
+
|
13
|
+
## v2.4.0
|
14
|
+
|
15
|
+
* Update librdkafka version from 1.4.0 to 1.5.0 by upgrading from rdkafka 0.8.0 to 0.10.0. ([#263](https://github.com/zendesk/racecar/pull/263))
|
16
|
+
* Restore support for Ruby 2.4 (#258)
|
17
|
+
|
18
|
+
## racecar v2.3.1
|
19
|
+
|
20
|
+
* Handle `ERR_NOT_COORDINATOR` (#209)
|
21
|
+
|
5
22
|
## racecar v2.3.0
|
6
23
|
|
7
24
|
* Add native support for Heroku (#248)
|
data/Gemfile
CHANGED
@@ -4,3 +4,7 @@ source 'https://rubygems.org'
|
|
4
4
|
|
5
5
|
# Specify your gem's dependencies in racecar.gemspec
|
6
6
|
gemspec
|
7
|
+
|
8
|
+
# We actually support version 6.0 (see gemspec); this extra restriction is added just for running the test suite also
|
9
|
+
# on Ruby 2.4, which activesupport 6.0 no longer supports
|
10
|
+
gem 'activesupport', '< 6.0'
|
data/Gemfile.lock
CHANGED
@@ -1,35 +1,34 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
racecar (2.
|
4
|
+
racecar (2.6.0)
|
5
5
|
king_konf (~> 1.0.0)
|
6
|
-
rdkafka (~> 0.
|
6
|
+
rdkafka (~> 0.10.0)
|
7
7
|
|
8
8
|
GEM
|
9
9
|
remote: https://rubygems.org/
|
10
10
|
specs:
|
11
|
-
activesupport (
|
11
|
+
activesupport (5.2.6)
|
12
12
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
13
13
|
i18n (>= 0.7, < 2)
|
14
14
|
minitest (~> 5.1)
|
15
15
|
tzinfo (~> 1.1)
|
16
|
-
zeitwerk (~> 2.2, >= 2.2.2)
|
17
16
|
coderay (1.1.3)
|
18
|
-
concurrent-ruby (1.1.
|
17
|
+
concurrent-ruby (1.1.9)
|
19
18
|
diff-lcs (1.4.4)
|
20
|
-
dogstatsd-ruby (
|
21
|
-
ffi (1.15.
|
22
|
-
i18n (1.8.
|
19
|
+
dogstatsd-ruby (5.2.0)
|
20
|
+
ffi (1.15.5)
|
21
|
+
i18n (1.8.10)
|
23
22
|
concurrent-ruby (~> 1.0)
|
24
23
|
king_konf (1.0.0)
|
25
24
|
method_source (1.0.0)
|
26
|
-
mini_portile2 (2.
|
27
|
-
minitest (5.14.
|
25
|
+
mini_portile2 (2.7.1)
|
26
|
+
minitest (5.14.4)
|
28
27
|
pry (0.13.1)
|
29
28
|
coderay (~> 1.1)
|
30
29
|
method_source (~> 1.0)
|
31
30
|
rake (13.0.1)
|
32
|
-
rdkafka (0.
|
31
|
+
rdkafka (0.10.0)
|
33
32
|
ffi (~> 1.9)
|
34
33
|
mini_portile2 (~> 2.1)
|
35
34
|
rake (>= 12.3)
|
@@ -48,17 +47,16 @@ GEM
|
|
48
47
|
rspec-support (3.10.2)
|
49
48
|
thread_safe (0.3.6)
|
50
49
|
timecop (0.9.2)
|
51
|
-
tzinfo (1.2.
|
50
|
+
tzinfo (1.2.9)
|
52
51
|
thread_safe (~> 0.1)
|
53
|
-
zeitwerk (2.4.2)
|
54
52
|
|
55
53
|
PLATFORMS
|
56
54
|
ruby
|
57
55
|
|
58
56
|
DEPENDENCIES
|
59
|
-
activesupport (
|
57
|
+
activesupport (< 6.0)
|
60
58
|
bundler (>= 1.13, < 3)
|
61
|
-
dogstatsd-ruby (>= 4.0.0, <
|
59
|
+
dogstatsd-ruby (>= 4.0.0, < 6.0.0)
|
62
60
|
pry
|
63
61
|
racecar!
|
64
62
|
rake (> 10.0)
|
@@ -66,4 +64,4 @@ DEPENDENCIES
|
|
66
64
|
timecop
|
67
65
|
|
68
66
|
BUNDLED WITH
|
69
|
-
2.
|
67
|
+
2.3.0
|
data/README.md
CHANGED
@@ -269,6 +269,10 @@ end
|
|
269
269
|
- `group_id` – The group id to use for a given group of consumers. Note that this _must_ be different for each consumer class. If left blank a group id is generated based on the consumer class name such that (for example) a consumer with the class name `BaconConsumer` would default to a group id of `bacon-consumer`.
|
270
270
|
- `group_id_prefix` – A prefix used when generating consumer group names. For instance, if you set the prefix to be `kevin.` and your consumer class is named `BaconConsumer`, the resulting consumer group will be named `kevin.bacon-consumer`.
|
271
271
|
|
272
|
+
#### Batches
|
273
|
+
|
274
|
+
- `fetch_messages` - The number of messages to fetch in a single batch. This can be set on a per consumer basis.
|
275
|
+
|
272
276
|
#### Logging
|
273
277
|
|
274
278
|
- `logfile` – A filename that log messages should be written to. Default is `nil`, which means logs will be written to standard output.
|
data/lib/racecar/config.rb
CHANGED
@@ -223,6 +223,7 @@ module Racecar
|
|
223
223
|
self.parallel_workers = consumer_class.parallel_workers
|
224
224
|
self.subscriptions = consumer_class.subscriptions
|
225
225
|
self.max_wait_time = consumer_class.max_wait_time || self.max_wait_time
|
226
|
+
self.fetch_messages = consumer_class.fetch_messages || self.fetch_messages
|
226
227
|
self.pidfile ||= "#{group_id}.pid"
|
227
228
|
end
|
228
229
|
|
data/lib/racecar/consumer.rb
CHANGED
@@ -9,7 +9,7 @@ module Racecar
|
|
9
9
|
class << self
|
10
10
|
attr_accessor :max_wait_time
|
11
11
|
attr_accessor :group_id
|
12
|
-
attr_accessor :producer, :consumer, :parallel_workers
|
12
|
+
attr_accessor :producer, :consumer, :parallel_workers, :fetch_messages
|
13
13
|
|
14
14
|
def subscriptions
|
15
15
|
@subscriptions ||= []
|
@@ -63,25 +63,27 @@ module Racecar
|
|
63
63
|
|
64
64
|
@instrumenter.instrument('deliver_messages', instrumentation_payload) do
|
65
65
|
@delivery_handles.each do |handle|
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
66
|
+
begin
|
67
|
+
# rdkafka-ruby checks every wait_timeout seconds if the message was
|
68
|
+
# successfully delivered, up to max_wait_timeout seconds before raising
|
69
|
+
# Rdkafka::AbstractHandle::WaitTimeoutError. librdkafka will (re)try to
|
70
|
+
# deliver all messages in the background, until "config.message_timeout"
|
71
|
+
# (message.timeout.ms) is exceeded. Phrased differently, rdkafka-ruby's
|
72
|
+
# WaitTimeoutError is just informative.
|
73
|
+
# The raising can be avoided if max_wait_timeout below is greater than
|
74
|
+
# config.message_timeout, but config is not available here (without
|
75
|
+
# changing the interface).
|
76
|
+
handle.wait(max_wait_timeout: 60, wait_timeout: 0.1)
|
77
|
+
rescue Rdkafka::AbstractHandle::WaitTimeoutError => e
|
78
|
+
partition = MessageDeliveryError.partition_from_delivery_handle(handle)
|
79
|
+
# ideally we could use the logger passed to the Runner, but it is not
|
80
|
+
# available here. The runner sets it for Rdkafka, though, so we can use
|
81
|
+
# that instead.
|
82
|
+
@config.logger.debug "Still trying to deliver message to (partition #{partition})... (will try up to Racecar.config.message_timeout)"
|
83
|
+
retry
|
84
|
+
rescue Rdkafka::RdkafkaError => e
|
85
|
+
raise MessageDeliveryError.new(e, handle)
|
86
|
+
end
|
85
87
|
end
|
86
88
|
end
|
87
89
|
end
|
@@ -91,13 +93,14 @@ module Racecar
|
|
91
93
|
protected
|
92
94
|
|
93
95
|
# https://github.com/appsignal/rdkafka-ruby#producing-messages
|
94
|
-
def produce(payload, topic:, key: nil, partition_key: nil, headers: nil, create_time: nil)
|
96
|
+
def produce(payload, topic:, key: nil, partition: nil, partition_key: nil, headers: nil, create_time: nil)
|
95
97
|
@delivery_handles ||= []
|
96
98
|
message_size = payload.respond_to?(:bytesize) ? payload.bytesize : 0
|
97
99
|
instrumentation_payload = {
|
98
100
|
value: payload,
|
99
101
|
headers: headers,
|
100
102
|
key: key,
|
103
|
+
partition: partition,
|
101
104
|
partition_key: partition_key,
|
102
105
|
topic: topic,
|
103
106
|
message_size: message_size,
|
@@ -110,6 +113,7 @@ module Racecar
|
|
110
113
|
topic: topic,
|
111
114
|
payload: payload,
|
112
115
|
key: key,
|
116
|
+
partition: partition,
|
113
117
|
partition_key: partition_key,
|
114
118
|
timestamp: create_time,
|
115
119
|
headers: headers,
|
data/lib/racecar/consumer_set.rb
CHANGED
@@ -156,7 +156,7 @@ module Racecar
|
|
156
156
|
msg = current.poll(max_wait_time_ms)
|
157
157
|
rescue Rdkafka::RdkafkaError => e
|
158
158
|
case e.code
|
159
|
-
when :max_poll_exceeded, :transport # -147, -195
|
159
|
+
when :max_poll_exceeded, :transport, :not_coordinator # -147, -195, 16
|
160
160
|
reset_current_consumer
|
161
161
|
end
|
162
162
|
raise
|
data/lib/racecar/datadog.rb
CHANGED
data/lib/racecar/runner.rb
CHANGED
data/lib/racecar/version.rb
CHANGED
data/racecar.gemspec
CHANGED
@@ -20,14 +20,16 @@ Gem::Specification.new do |spec|
|
|
20
20
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
21
21
|
spec.require_paths = ["lib"]
|
22
22
|
|
23
|
+
spec.required_ruby_version = '>= 2.4'
|
24
|
+
|
23
25
|
spec.add_runtime_dependency "king_konf", "~> 1.0.0"
|
24
|
-
spec.add_runtime_dependency "rdkafka", "~> 0.
|
26
|
+
spec.add_runtime_dependency "rdkafka", "~> 0.10.0"
|
25
27
|
|
26
28
|
spec.add_development_dependency "bundler", [">= 1.13", "< 3"]
|
27
29
|
spec.add_development_dependency "pry"
|
28
30
|
spec.add_development_dependency "rake", "> 10.0"
|
29
31
|
spec.add_development_dependency "rspec", "~> 3.0"
|
30
32
|
spec.add_development_dependency "timecop"
|
31
|
-
spec.add_development_dependency "dogstatsd-ruby", ">= 4.0.0", "<
|
33
|
+
spec.add_development_dependency "dogstatsd-ruby", ">= 4.0.0", "< 6.0.0"
|
32
34
|
spec.add_development_dependency "activesupport", ">= 4.0", "< 6.1"
|
33
35
|
end
|
metadata
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: racecar
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Schierbeck
|
8
8
|
- Benjamin Quorning
|
9
|
-
autorequire:
|
9
|
+
autorequire:
|
10
10
|
bindir: exe
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2022-02-01 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: king_konf
|
@@ -31,14 +31,14 @@ dependencies:
|
|
31
31
|
requirements:
|
32
32
|
- - "~>"
|
33
33
|
- !ruby/object:Gem::Version
|
34
|
-
version: 0.
|
34
|
+
version: 0.10.0
|
35
35
|
type: :runtime
|
36
36
|
prerelease: false
|
37
37
|
version_requirements: !ruby/object:Gem::Requirement
|
38
38
|
requirements:
|
39
39
|
- - "~>"
|
40
40
|
- !ruby/object:Gem::Version
|
41
|
-
version: 0.
|
41
|
+
version: 0.10.0
|
42
42
|
- !ruby/object:Gem::Dependency
|
43
43
|
name: bundler
|
44
44
|
requirement: !ruby/object:Gem::Requirement
|
@@ -124,7 +124,7 @@ dependencies:
|
|
124
124
|
version: 4.0.0
|
125
125
|
- - "<"
|
126
126
|
- !ruby/object:Gem::Version
|
127
|
-
version:
|
127
|
+
version: 6.0.0
|
128
128
|
type: :development
|
129
129
|
prerelease: false
|
130
130
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -134,7 +134,7 @@ dependencies:
|
|
134
134
|
version: 4.0.0
|
135
135
|
- - "<"
|
136
136
|
- !ruby/object:Gem::Version
|
137
|
-
version:
|
137
|
+
version: 6.0.0
|
138
138
|
- !ruby/object:Gem::Dependency
|
139
139
|
name: activesupport
|
140
140
|
requirement: !ruby/object:Gem::Requirement
|
@@ -155,7 +155,7 @@ dependencies:
|
|
155
155
|
- - "<"
|
156
156
|
- !ruby/object:Gem::Version
|
157
157
|
version: '6.1'
|
158
|
-
description:
|
158
|
+
description:
|
159
159
|
email:
|
160
160
|
- dschierbeck@zendesk.com
|
161
161
|
- bquorning@zendesk.com
|
@@ -214,7 +214,7 @@ homepage: https://github.com/zendesk/racecar
|
|
214
214
|
licenses:
|
215
215
|
- Apache License Version 2.0
|
216
216
|
metadata: {}
|
217
|
-
post_install_message:
|
217
|
+
post_install_message:
|
218
218
|
rdoc_options: []
|
219
219
|
require_paths:
|
220
220
|
- lib
|
@@ -222,15 +222,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
222
222
|
requirements:
|
223
223
|
- - ">="
|
224
224
|
- !ruby/object:Gem::Version
|
225
|
-
version: '
|
225
|
+
version: '2.4'
|
226
226
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
227
227
|
requirements:
|
228
228
|
- - ">="
|
229
229
|
- !ruby/object:Gem::Version
|
230
230
|
version: '0'
|
231
231
|
requirements: []
|
232
|
-
rubygems_version: 3.
|
233
|
-
signing_key:
|
232
|
+
rubygems_version: 3.0.3
|
233
|
+
signing_key:
|
234
234
|
specification_version: 4
|
235
235
|
summary: A framework for running Kafka consumers
|
236
236
|
test_files: []
|