karafka 2.2.4 → 2.2.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.github/workflows/ci.yml +66 -11
- data/CHANGELOG.md +8 -0
- data/Gemfile.lock +15 -15
- data/bin/benchmarks +14 -0
- data/bin/integrations +15 -2
- data/bin/wait_for_kafka +24 -0
- data/config/locales/errors.yml +1 -1
- data/docker-compose.yml +17 -21
- data/lib/karafka/admin.rb +5 -1
- data/lib/karafka/base_consumer.rb +1 -0
- data/lib/karafka/connection/client.rb +15 -1
- data/lib/karafka/contracts/topic.rb +1 -1
- data/lib/karafka/errors.rb +5 -0
- data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +1 -2
- data/lib/karafka/pro/routing/features/patterns/detector.rb +3 -0
- data/lib/karafka/railtie.rb +10 -1
- data/lib/karafka/routing/consumer_group.rb +5 -5
- data/lib/karafka/routing/subscription_group.rb +8 -2
- data/lib/karafka/routing/subscription_groups_builder.rb +8 -1
- data/lib/karafka/routing/topic.rb +6 -1
- data/lib/karafka/version.rb +1 -1
- data.tar.gz.sig +2 -2
- metadata +3 -2
- metadata.gz.sig +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 729320b972ba55fe500db0729aa651c6308e1975db33d3ebd4db8061561e38bc
|
4
|
+
data.tar.gz: f26fa836af6b7777692c7fd36fad1c89ce22c7ab60ab243662df40993acd7bd4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 86cecfe51fbdb21943f75a98a1ee8e755496f9777bf5c4445ad44ecff45fa5be34fc79c0ad6358ded3886a59ebde4f0cd47598fdcdda41f75c5152489cf5ca35
|
7
|
+
data.tar.gz: fec15466a63fc6e49089f339d3f0d9bb3d4d308c7855adf17f23e93ef10e3e09daf6e8453296a53bf74a57d1004883fc2b144c4564342daec8b94b9a7288bd82
|
checksums.yaml.gz.sig
CHANGED
Binary file
|
data/.github/workflows/ci.yml
CHANGED
@@ -61,8 +61,10 @@ jobs:
|
|
61
61
|
- name: Run Coditsu
|
62
62
|
run: \curl -sSL https://api.coditsu.io/run/ci | bash
|
63
63
|
|
64
|
+
# We do not split RSpec specs to OSS and Pro like integrations because they do not overload
|
65
|
+
# Kafka heavily, compute total coverage for specs and are fast enough
|
64
66
|
specs:
|
65
|
-
timeout-minutes:
|
67
|
+
timeout-minutes: 10
|
66
68
|
runs-on: ubuntu-latest
|
67
69
|
needs: diffend
|
68
70
|
strategy:
|
@@ -88,7 +90,7 @@ jobs:
|
|
88
90
|
|
89
91
|
- name: Start Kafka with docker-compose
|
90
92
|
run: |
|
91
|
-
docker-compose up -d
|
93
|
+
docker-compose up -d || (sleep 5 && docker-compose up -d)
|
92
94
|
|
93
95
|
- name: Set up Ruby
|
94
96
|
uses: ruby/setup-ruby@v1
|
@@ -96,13 +98,17 @@ jobs:
|
|
96
98
|
ruby-version: ${{matrix.ruby}}
|
97
99
|
bundler-cache: true
|
98
100
|
|
101
|
+
- name: Wait for Kafka
|
102
|
+
run: |
|
103
|
+
bundle exec bin/wait_for_kafka
|
104
|
+
|
99
105
|
- name: Run all specs
|
100
106
|
env:
|
101
107
|
GITHUB_COVERAGE: ${{matrix.coverage}}
|
102
108
|
run: bin/rspecs
|
103
109
|
|
104
|
-
|
105
|
-
timeout-minutes:
|
110
|
+
integrations_oss:
|
111
|
+
timeout-minutes: 20
|
106
112
|
runs-on: ubuntu-latest
|
107
113
|
needs: diffend
|
108
114
|
strategy:
|
@@ -113,9 +119,6 @@ jobs:
|
|
113
119
|
- '3.1'
|
114
120
|
- '3.0'
|
115
121
|
- '2.7'
|
116
|
-
include:
|
117
|
-
- ruby: '3.2'
|
118
|
-
coverage: 'true'
|
119
122
|
steps:
|
120
123
|
- uses: actions/checkout@v4
|
121
124
|
- name: Install package dependencies
|
@@ -126,7 +129,55 @@ jobs:
|
|
126
129
|
|
127
130
|
- name: Start Kafka with docker-compose
|
128
131
|
run: |
|
129
|
-
docker-compose up -d
|
132
|
+
docker-compose up -d || (sleep 5 && docker-compose up -d)
|
133
|
+
|
134
|
+
- name: Set up Ruby
|
135
|
+
uses: ruby/setup-ruby@v1
|
136
|
+
with:
|
137
|
+
# Do not use cache here as we run bundle install also later in some of the integration
|
138
|
+
# tests and we need to be able to run it without cache
|
139
|
+
#
|
140
|
+
# We also want to check that librdkafka is compiling as expected on all versions of Ruby
|
141
|
+
ruby-version: ${{matrix.ruby}}
|
142
|
+
|
143
|
+
- name: Install latest Bundler
|
144
|
+
run: |
|
145
|
+
gem install bundler --no-document
|
146
|
+
gem update --system --no-document
|
147
|
+
bundle config set without 'tools benchmarks docs'
|
148
|
+
|
149
|
+
- name: Bundle install
|
150
|
+
run: |
|
151
|
+
bundle config set without development
|
152
|
+
bundle install
|
153
|
+
|
154
|
+
- name: Wait for Kafka
|
155
|
+
run: |
|
156
|
+
bundle exec bin/wait_for_kafka
|
157
|
+
|
158
|
+
- name: Run OSS integration tests
|
159
|
+
run: bin/integrations --exclude '/pro'
|
160
|
+
|
161
|
+
integrations_pro:
|
162
|
+
timeout-minutes: 40
|
163
|
+
runs-on: ubuntu-latest
|
164
|
+
needs: diffend
|
165
|
+
strategy:
|
166
|
+
fail-fast: false
|
167
|
+
matrix:
|
168
|
+
ruby:
|
169
|
+
- '3.2'
|
170
|
+
- '3.1'
|
171
|
+
- '3.0'
|
172
|
+
- '2.7'
|
173
|
+
steps:
|
174
|
+
- uses: actions/checkout@v4
|
175
|
+
- name: Install package dependencies
|
176
|
+
run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
|
177
|
+
|
178
|
+
- name: Start Kafka with docker-compose
|
179
|
+
run: |
|
180
|
+
docker-compose up -d || (sleep 5 && docker-compose up -d)
|
130
181
|
|
131
182
|
- name: Set up Ruby
|
132
183
|
uses: ruby/setup-ruby@v1
|
@@ -144,12 +195,16 @@ jobs:
|
|
144
195
|
bundle config set without development
|
145
196
|
bundle install
|
146
197
|
|
147
|
-
- name:
|
198
|
+
- name: Wait for Kafka
|
199
|
+
run: |
|
200
|
+
bundle exec bin/wait_for_kafka
|
201
|
+
|
202
|
+
- name: Run Pro integration tests
|
148
203
|
env:
|
149
204
|
KARAFKA_PRO_LICENSE_TOKEN: ${{ secrets.KARAFKA_PRO_LICENSE_TOKEN }}
|
150
205
|
KARAFKA_PRO_USERNAME: ${{ secrets.KARAFKA_PRO_USERNAME }}
|
151
206
|
KARAFKA_PRO_PASSWORD: ${{ secrets.KARAFKA_PRO_PASSWORD }}
|
152
207
|
KARAFKA_PRO_VERSION: ${{ secrets.KARAFKA_PRO_VERSION }}
|
153
208
|
KARAFKA_PRO_LICENSE_CHECKSUM: ${{ secrets.KARAFKA_PRO_LICENSE_CHECKSUM }}
|
154
|
-
|
155
|
-
run: bin/integrations
|
209
|
+
|
210
|
+
run: bin/integrations '/pro'
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,13 @@
|
|
1
1
|
# Karafka framework changelog
|
2
2
|
|
3
|
+
## 2.2.5 (2023-09-25)
|
4
|
+
- [Enhancement] Ensure, that when topic related operations end, the result is usable. There were few cases where admin operations on topics would finish successfully but internal Kafka caches would not report changes for a short period of time.
|
5
|
+
- [Enhancement] Stabilize cooperative-sticky early shutdown procedure.
|
6
|
+
- [Fix] use `#nil?` instead of `#present?` on `DataDog::Tracing::SpanOperation` (vitellochris)
|
7
|
+
- [Maintenance] Align connection clearing API with Rails 7.1 deprecation warning.
|
8
|
+
- [Maintenance] Make `#subscription_group` reference consistent in the Routing and Instrumentation.
|
9
|
+
- [Maintenance] Align the consumer pause instrumentation with client pause instrumentation by adding `subscription_group` visibility to the consumer.
|
10
|
+
|
3
11
|
## 2.2.4 (2023-09-13)
|
4
12
|
- [Enhancement] Compensate for potential Kafka cluster drifts vs consumer drift in batch metadata (#1611).
|
5
13
|
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
karafka (2.2.
|
4
|
+
karafka (2.2.5)
|
5
5
|
karafka-core (>= 2.2.2, < 2.3.0)
|
6
6
|
thor (>= 0.20)
|
7
7
|
waterdrop (>= 2.6.6, < 3.0.0)
|
@@ -10,10 +10,10 @@ PATH
|
|
10
10
|
GEM
|
11
11
|
remote: https://rubygems.org/
|
12
12
|
specs:
|
13
|
-
activejob (7.0.
|
14
|
-
activesupport (= 7.0.
|
13
|
+
activejob (7.0.8)
|
14
|
+
activesupport (= 7.0.8)
|
15
15
|
globalid (>= 0.3.6)
|
16
|
-
activesupport (7.0.
|
16
|
+
activesupport (7.0.8)
|
17
17
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
18
18
|
i18n (>= 1.6, < 2)
|
19
19
|
minitest (>= 5.1)
|
@@ -26,28 +26,28 @@ GEM
|
|
26
26
|
factory_bot (6.3.0)
|
27
27
|
activesupport (>= 5.0.0)
|
28
28
|
ffi (1.15.5)
|
29
|
-
globalid (1.1
|
30
|
-
activesupport (>=
|
29
|
+
globalid (1.2.1)
|
30
|
+
activesupport (>= 6.1)
|
31
31
|
i18n (1.14.1)
|
32
32
|
concurrent-ruby (~> 1.0)
|
33
33
|
karafka-core (2.2.2)
|
34
34
|
concurrent-ruby (>= 1.1)
|
35
35
|
karafka-rdkafka (>= 0.13.1, < 0.14.0)
|
36
|
-
karafka-rdkafka (0.13.
|
36
|
+
karafka-rdkafka (0.13.5)
|
37
37
|
ffi (~> 1.15)
|
38
38
|
mini_portile2 (~> 2.6)
|
39
39
|
rake (> 12)
|
40
|
-
karafka-web (0.
|
40
|
+
karafka-web (0.7.4)
|
41
41
|
erubi (~> 1.4)
|
42
|
-
karafka (>= 2.
|
43
|
-
karafka-core (>= 2.
|
44
|
-
roda (~> 3.68, >= 3.
|
42
|
+
karafka (>= 2.2.3, < 3.0.0)
|
43
|
+
karafka-core (>= 2.2.2, < 3.0.0)
|
44
|
+
roda (~> 3.68, >= 3.69)
|
45
45
|
tilt (~> 2.0)
|
46
46
|
mini_portile2 (2.8.4)
|
47
|
-
minitest (5.
|
47
|
+
minitest (5.20.0)
|
48
48
|
rack (3.0.8)
|
49
49
|
rake (13.0.6)
|
50
|
-
roda (3.
|
50
|
+
roda (3.72.0)
|
51
51
|
rack
|
52
52
|
rspec (3.12.0)
|
53
53
|
rspec-core (~> 3.12.0)
|
@@ -72,7 +72,7 @@ GEM
|
|
72
72
|
tilt (2.2.0)
|
73
73
|
tzinfo (2.0.6)
|
74
74
|
concurrent-ruby (~> 1.0)
|
75
|
-
waterdrop (2.6.
|
75
|
+
waterdrop (2.6.7)
|
76
76
|
karafka-core (>= 2.1.1, < 3.0.0)
|
77
77
|
zeitwerk (~> 2.3)
|
78
78
|
zeitwerk (2.6.11)
|
@@ -90,4 +90,4 @@ DEPENDENCIES
|
|
90
90
|
simplecov
|
91
91
|
|
92
92
|
BUNDLED WITH
|
93
|
-
2.4.
|
93
|
+
2.4.19
|
data/bin/benchmarks
CHANGED
@@ -13,6 +13,13 @@ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..'))
|
|
13
13
|
|
14
14
|
ROOT_PATH = Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../')))
|
15
15
|
|
16
|
+
BENCHMARK_TOPICS = {
|
17
|
+
'benchmarks_00_01' => 1,
|
18
|
+
'benchmarks_00_05' => 5,
|
19
|
+
'benchmarks_01_05' => 5,
|
20
|
+
'benchmarks_00_10' => 10
|
21
|
+
}
|
22
|
+
|
16
23
|
# Load all the benchmarks
|
17
24
|
benchmarks = Dir[ROOT_PATH.join('spec/benchmarks/**/*.rb')]
|
18
25
|
|
@@ -37,6 +44,13 @@ if ENV['SEED']
|
|
37
44
|
# We make our data json compatible so we can also benchmark serialization
|
38
45
|
elements = Array.new(100_000) { { a: :b }.to_json }
|
39
46
|
|
47
|
+
topics = Karafka::Admin.cluster_info.topics.map { |details| details.fetch(:topic_name) }
|
48
|
+
|
49
|
+
BENCHMARK_TOPICS.each do |topic_name, partitions_count|
|
50
|
+
::Karafka::Admin.delete_topic(topic_name) if topics.include?(topic_name)
|
51
|
+
::Karafka::Admin.create_topic(topic_name, partitions_count, 1)
|
52
|
+
end
|
53
|
+
|
40
54
|
# We do not populate data of benchmarks_0_10 as we use it with life-stream data only
|
41
55
|
%w[
|
42
56
|
benchmarks_00_01
|
data/bin/integrations
CHANGED
@@ -218,13 +218,26 @@ end
|
|
218
218
|
# Load all the specs
|
219
219
|
specs = Dir[ROOT_PATH.join('spec/integrations/**/*_spec.rb')]
|
220
220
|
|
221
|
+
FILTER_TYPE = ARGV[0] == '--exclude' ? 'exclude' : 'include'
|
222
|
+
|
223
|
+
# Remove the exclude flag
|
224
|
+
ARGV.shift if FILTER_TYPE == '--exclude'
|
225
|
+
|
221
226
|
# If filters is provided, apply
|
222
227
|
# Allows to provide several filters one after another and applies all of them
|
223
228
|
ARGV.each do |filter|
|
224
|
-
specs.delete_if
|
229
|
+
specs.delete_if do |name|
|
230
|
+
case FILTER_TYPE
|
231
|
+
when 'include'
|
232
|
+
!name.include?(filter)
|
233
|
+
when 'exclude'
|
234
|
+
name.include?(filter)
|
235
|
+
else
|
236
|
+
raise 'Invalid filter type'
|
237
|
+
end
|
238
|
+
end
|
225
239
|
end
|
226
240
|
|
227
|
-
|
228
241
|
raise ArgumentError, "No integration specs with filters: #{ARGV.join(', ')}" if specs.empty?
|
229
242
|
|
230
243
|
# Randomize order
|
data/bin/wait_for_kafka
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
# Waits for Kafka to be ready
|
4
|
+
# Useful in CI where Kafka needs to be fully started before we run any tests
|
5
|
+
|
6
|
+
require 'karafka'
|
7
|
+
|
8
|
+
Karafka::App.setup do |config|
|
9
|
+
config.kafka[:'bootstrap.servers'] = '127.0.0.1:9092'
|
10
|
+
end
|
11
|
+
|
12
|
+
60.times do
|
13
|
+
begin
|
14
|
+
# Stop if we can connect to the cluster and get info
|
15
|
+
exit if Karafka::Admin.cluster_info
|
16
|
+
rescue Rdkafka::RdkafkaError
|
17
|
+
puts "Kafka not available, retrying..."
|
18
|
+
sleep(1)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
puts 'Kafka not available!'
|
23
|
+
|
24
|
+
exit 1
|
data/config/locales/errors.yml
CHANGED
@@ -57,7 +57,7 @@ en:
|
|
57
57
|
consumer_format: needs to be present
|
58
58
|
id_format: 'needs to be a string with a Kafka accepted format'
|
59
59
|
initial_offset_format: needs to be either earliest or latest
|
60
|
-
|
60
|
+
subscription_group_name_format: must be a non-empty string
|
61
61
|
manual_offset_management.active_format: needs to be either true or false
|
62
62
|
consumer_active_job_missing: ActiveJob needs to be available
|
63
63
|
manual_offset_management_must_be_enabled: cannot be disabled for ActiveJob topics
|
data/docker-compose.yml
CHANGED
@@ -1,27 +1,23 @@
|
|
1
1
|
version: '2'
|
2
|
-
services:
|
3
|
-
zookeeper:
|
4
|
-
container_name: karafka_22_zookeeper
|
5
|
-
image: wurstmeister/zookeeper
|
6
|
-
restart: on-failure
|
7
|
-
ports:
|
8
|
-
- '2181:2181'
|
9
2
|
|
3
|
+
services:
|
10
4
|
kafka:
|
11
|
-
container_name:
|
12
|
-
image:
|
5
|
+
container_name: kafka
|
6
|
+
image: confluentinc/cp-kafka:7.5.0
|
7
|
+
|
13
8
|
ports:
|
14
|
-
-
|
9
|
+
- 9092:9092
|
10
|
+
|
15
11
|
environment:
|
16
|
-
|
17
|
-
|
18
|
-
|
12
|
+
CLUSTER_ID: kafka-docker-cluster-1
|
13
|
+
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
|
14
|
+
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
|
15
|
+
KAFKA_PROCESS_ROLES: broker,controller
|
16
|
+
KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
|
17
|
+
KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093
|
18
|
+
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
|
19
|
+
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:9092
|
20
|
+
KAFKA_BROKER_ID: 1
|
21
|
+
KAFKA_CONTROLLER_QUORUM_VOTERS: 1@127.0.0.1:9093
|
22
|
+
ALLOW_PLAINTEXT_LISTENER: 'yes'
|
19
23
|
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
|
20
|
-
KAFKA_CREATE_TOPICS:
|
21
|
-
"benchmarks_00_01:1:1,\
|
22
|
-
benchmarks_00_05:5:1,\
|
23
|
-
benchmarks_01_05:5:1,\
|
24
|
-
benchmarks_00_10:10:1"
|
25
|
-
volumes:
|
26
|
-
- /var/run/docker.sock:/var/run/docker.sock
|
27
|
-
restart: on-failure
|
data/lib/karafka/admin.rb
CHANGED
@@ -216,7 +216,11 @@ module Karafka
|
|
216
216
|
attempt += 1
|
217
217
|
|
218
218
|
handler.call
|
219
|
-
|
219
|
+
|
220
|
+
# If breaker does not operate, it means that the requested change was applied but is still
|
221
|
+
# not visible and we need to wait
|
222
|
+
raise(Errors::ResultNotVisibleError) unless breaker.call
|
223
|
+
rescue Rdkafka::AbstractHandle::WaitTimeoutError, Errors::ResultNotVisibleError
|
220
224
|
return if breaker.call
|
221
225
|
|
222
226
|
retry if attempt <= app_config.admin.max_attempts
|
@@ -197,6 +197,7 @@ module Karafka
|
|
197
197
|
manual: manual_pause,
|
198
198
|
topic: topic.name,
|
199
199
|
partition: partition,
|
200
|
+
subscription_group: topic.subscription_group,
|
200
201
|
offset: offset,
|
201
202
|
timeout: coordinator.pause_tracker.current_timeout,
|
202
203
|
attempt: coordinator.pause_tracker.attempt
|
@@ -236,9 +236,23 @@ module Karafka
|
|
236
236
|
#
|
237
237
|
# @see https://github.com/confluentinc/librdkafka/issues/4312
|
238
238
|
if @subscription_group.kafka[:'partition.assignment.strategy'] == 'cooperative-sticky'
|
239
|
+
active_wait = false
|
240
|
+
|
239
241
|
(COOPERATIVE_STICKY_MAX_WAIT / 100).times do
|
240
242
|
# If we're past the first rebalance, no need to wait
|
241
|
-
|
243
|
+
if @rebalance_manager.active?
|
244
|
+
# We give it a a bit of time because librdkafka has a tendency to do some-post
|
245
|
+
# callback work that from its perspective is still under rebalance
|
246
|
+
sleep(5) if active_wait
|
247
|
+
|
248
|
+
break
|
249
|
+
end
|
250
|
+
|
251
|
+
active_wait = true
|
252
|
+
|
253
|
+
# poll to trigger potential rebalances that could occur during stopping and to trigger
|
254
|
+
# potential callbacks
|
255
|
+
poll(100)
|
242
256
|
|
243
257
|
sleep(0.1)
|
244
258
|
end
|
@@ -20,7 +20,7 @@ module Karafka
|
|
20
20
|
required(:max_wait_time) { |val| val.is_a?(Integer) && val >= 10 }
|
21
21
|
required(:name) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
|
22
22
|
required(:active) { |val| [true, false].include?(val) }
|
23
|
-
required(:
|
23
|
+
required(:subscription_group_name) { |val| val.is_a?(String) && !val.empty? }
|
24
24
|
|
25
25
|
# Consumer needs to be present only if topic is active
|
26
26
|
# We allow not to define consumer for non-active because they may be only used via admin
|
data/lib/karafka/errors.rb
CHANGED
@@ -55,5 +55,10 @@ module Karafka
|
|
55
55
|
|
56
56
|
# This should never happen. Please open an issue if it does.
|
57
57
|
InvalidTimeBasedOffsetError = Class.new(BaseError)
|
58
|
+
|
59
|
+
# For internal usage only
|
60
|
+
# Raised when we run operations that require certain result but despite successfully finishing
|
61
|
+
# it is not yet available due to some synchronization mechanisms and caches
|
62
|
+
ResultNotVisibleError = Class.new(BaseError)
|
58
63
|
end
|
59
64
|
end
|
@@ -56,6 +56,9 @@ module Karafka
|
|
56
56
|
topic = consumer_group.public_send(:topic=, discovered_topic, &pattern.config)
|
57
57
|
topic.patterns(active: true, type: :discovered)
|
58
58
|
|
59
|
+
# Assign the appropriate subscription group to this topic
|
60
|
+
topic.subscription_group = pattern.topic.subscription_group
|
61
|
+
|
59
62
|
# Inject into subscription group topics array always, so everything is reflected
|
60
63
|
# there but since it is not active, will not be picked
|
61
64
|
sg_topics << topic
|
data/lib/karafka/railtie.rb
CHANGED
@@ -81,12 +81,21 @@ if Karafka.rails?
|
|
81
81
|
end
|
82
82
|
|
83
83
|
initializer 'karafka.release_active_record_connections' do
|
84
|
+
rails7plus = Rails.gem_version >= Gem::Version.new('7.0.0')
|
85
|
+
|
84
86
|
ActiveSupport.on_load(:active_record) do
|
85
87
|
::Karafka::App.monitor.subscribe('worker.completed') do
|
86
88
|
# Always release the connection after processing is done. Otherwise thread may hang
|
87
89
|
# blocking the reload and further processing
|
88
90
|
# @see https://github.com/rails/rails/issues/44183
|
89
|
-
|
91
|
+
#
|
92
|
+
# The change technically happens in 7.1 but 7.0 already supports this so we can make
|
93
|
+
# a proper change for 7.0+
|
94
|
+
if rails7plus
|
95
|
+
ActiveRecord::Base.connection_handler.clear_active_connections!
|
96
|
+
else
|
97
|
+
ActiveRecord::Base.clear_active_connections!
|
98
|
+
end
|
90
99
|
end
|
91
100
|
end
|
92
101
|
end
|
@@ -14,7 +14,7 @@ module Karafka
|
|
14
14
|
# It allows us to store the "current" subscription group defined in the routing
|
15
15
|
# This subscription group id is then injected into topics, so we can compute the subscription
|
16
16
|
# groups
|
17
|
-
attr_accessor :
|
17
|
+
attr_accessor :current_subscription_group_name
|
18
18
|
|
19
19
|
# @param name [String, Symbol] raw name of this consumer group. Raw means, that it does not
|
20
20
|
# yet have an application client_id namespace, this will be added here by default.
|
@@ -26,7 +26,7 @@ module Karafka
|
|
26
26
|
@topics = Topics.new([])
|
27
27
|
# Initialize the subscription group so there's always a value for it, since even if not
|
28
28
|
# defined directly, a subscription group will be created
|
29
|
-
@
|
29
|
+
@current_subscription_group_name = SubscriptionGroup.id
|
30
30
|
end
|
31
31
|
|
32
32
|
# @return [Boolean] true if this consumer group should be active in our current process
|
@@ -48,7 +48,7 @@ module Karafka
|
|
48
48
|
built_topic = @topics.last
|
49
49
|
# We overwrite it conditionally in case it was not set by the user inline in the topic
|
50
50
|
# block definition
|
51
|
-
built_topic.
|
51
|
+
built_topic.subscription_group_name ||= current_subscription_group_name
|
52
52
|
built_topic
|
53
53
|
end
|
54
54
|
|
@@ -59,13 +59,13 @@ module Karafka
|
|
59
59
|
def subscription_group=(name = SubscriptionGroup.id, &block)
|
60
60
|
# We cast it here, so the routing supports symbol based but that's anyhow later on
|
61
61
|
# validated as a string
|
62
|
-
@
|
62
|
+
@current_subscription_group_name = name.to_s
|
63
63
|
|
64
64
|
Proxy.new(self, &block)
|
65
65
|
|
66
66
|
# We need to reset the current subscription group after it is used, so it won't leak
|
67
67
|
# outside to other topics that would be defined without a defined subscription group
|
68
|
-
@
|
68
|
+
@current_subscription_group_name = SubscriptionGroup.id
|
69
69
|
end
|
70
70
|
|
71
71
|
# @return [Array<Routing::SubscriptionGroup>] all the subscription groups build based on
|
@@ -32,7 +32,7 @@ module Karafka
|
|
32
32
|
# @param topics [Karafka::Routing::Topics] all the topics that share the same key settings
|
33
33
|
# @return [SubscriptionGroup] built subscription group
|
34
34
|
def initialize(position, topics)
|
35
|
-
@name = topics.first.
|
35
|
+
@name = topics.first.subscription_group_name
|
36
36
|
@consumer_group = topics.first.consumer_group
|
37
37
|
@id = "#{@name}_#{position}"
|
38
38
|
@position = position
|
@@ -70,6 +70,12 @@ module Karafka
|
|
70
70
|
topics.select(&:active?).map(&:subscription_name)
|
71
71
|
end
|
72
72
|
|
73
|
+
# @return [String] id of the subscription group
|
74
|
+
# @note This is an alias for displaying in places where we print the stringified version.
|
75
|
+
def to_s
|
76
|
+
id
|
77
|
+
end
|
78
|
+
|
73
79
|
private
|
74
80
|
|
75
81
|
# @return [Hash] kafka settings are a bit special. They are exactly the same for all of the
|
@@ -87,7 +93,7 @@ module Karafka
|
|
87
93
|
|
88
94
|
kafka[:'group.instance.id'] = "#{group_instance_id}_#{@position}" if group_instance_id
|
89
95
|
kafka[:'client.id'] ||= Karafka::App.config.client_id
|
90
|
-
kafka[:'group.id'] ||= @
|
96
|
+
kafka[:'group.id'] ||= @consumer_group.id
|
91
97
|
kafka[:'auto.offset.reset'] ||= @topics.first.initial_offset
|
92
98
|
# Karafka manages the offsets based on the processing state, thus we do not rely on the
|
93
99
|
# rdkafka offset auto-storing
|
@@ -19,7 +19,7 @@ module Karafka
|
|
19
19
|
max_messages
|
20
20
|
max_wait_time
|
21
21
|
initial_offset
|
22
|
-
|
22
|
+
subscription_group_name
|
23
23
|
].freeze
|
24
24
|
|
25
25
|
private_constant :DISTRIBUTION_KEYS
|
@@ -39,6 +39,13 @@ module Karafka
|
|
39
39
|
.map { |value| value.map(&:last) }
|
40
40
|
.map { |topics_array| Routing::Topics.new(topics_array) }
|
41
41
|
.map { |grouped_topics| SubscriptionGroup.new(@position += 1, grouped_topics) }
|
42
|
+
.tap do |subscription_groups|
|
43
|
+
subscription_groups.each do |subscription_group|
|
44
|
+
subscription_group.topics.each do |topic|
|
45
|
+
topic.subscription_group = subscription_group
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
42
49
|
end
|
43
50
|
|
44
51
|
private
|
@@ -8,6 +8,11 @@ module Karafka
|
|
8
8
|
class Topic
|
9
9
|
attr_reader :id, :name, :consumer_group
|
10
10
|
attr_writer :consumer
|
11
|
+
|
12
|
+
attr_accessor :subscription_group_name
|
13
|
+
|
14
|
+
# Full subscription group reference can be built only when we have knowledge about the
|
15
|
+
# whole routing tree, this is why it is going to be set later on
|
11
16
|
attr_accessor :subscription_group
|
12
17
|
|
13
18
|
# Attributes we can inherit from the root unless they were defined on this level
|
@@ -112,7 +117,7 @@ module Karafka
|
|
112
117
|
active: active?,
|
113
118
|
consumer: consumer,
|
114
119
|
consumer_group_id: consumer_group.id,
|
115
|
-
|
120
|
+
subscription_group_name: subscription_group_name
|
116
121
|
).freeze
|
117
122
|
end
|
118
123
|
end
|
data/lib/karafka/version.rb
CHANGED
data.tar.gz.sig
CHANGED
@@ -1,2 +1,2 @@
|
|
1
|
-
|
2
|
-
���
|
1
|
+
��{J��4����cT�8o8�jՉ9R!w���>%�?kJ`�Ɨ��%@�h �jJ��pq9�*�ٺd�-dE� �5�d��n��KR�+F���%�@�rDA#Κ`�!��H\�=y ob��#xq�Gi�@�Ji�3ӸΉF�8uػ� l��/��mN���AX�
|
2
|
+
�J����cNa�\xD�h���Kp��KA��5T2ګ�t�0I�%��v BD$`�ͻ�3��n���$@T�(���#<��lܱ�K 5L�3�)ܐ���L�L���g�w�(����E�Pb���~�"��+�Y���[��-�+v�)�0#�nO�ŝ�5��ц:�!�^!ƽ�{nل���E2�<M�C�O'���\{�?e��!��Wl����H��\Y
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.2.
|
4
|
+
version: 2.2.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Maciej Mensfeld
|
@@ -35,7 +35,7 @@ cert_chain:
|
|
35
35
|
AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
|
36
36
|
msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
|
37
37
|
-----END CERTIFICATE-----
|
38
|
-
date: 2023-09-
|
38
|
+
date: 2023-09-25 00:00:00.000000000 Z
|
39
39
|
dependencies:
|
40
40
|
- !ruby/object:Gem::Dependency
|
41
41
|
name: karafka-core
|
@@ -147,6 +147,7 @@ files:
|
|
147
147
|
- bin/stress_many
|
148
148
|
- bin/stress_one
|
149
149
|
- bin/verify_license_integrity
|
150
|
+
- bin/wait_for_kafka
|
150
151
|
- certs/cert_chain.pem
|
151
152
|
- certs/karafka-pro.pem
|
152
153
|
- config/locales/errors.yml
|
metadata.gz.sig
CHANGED
Binary file
|