karafka 2.2.4 → 2.2.5

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 7e6cb6a9b75c85409da5835992ea35b8db57e7f38906d31e2dd4e6345027a357
4
- data.tar.gz: 40ceebe714cf3db25dcc33a8b5ff8ea1426101aaf18862be907298334beddc16
3
+ metadata.gz: 729320b972ba55fe500db0729aa651c6308e1975db33d3ebd4db8061561e38bc
4
+ data.tar.gz: f26fa836af6b7777692c7fd36fad1c89ce22c7ab60ab243662df40993acd7bd4
5
5
  SHA512:
6
- metadata.gz: 5e0f780dc0d46e5917df86afba04cb9620547abfc188b550b7667035ed624ab5a36a2dd35e9437fe43d3be66868f409f9d9c3b71d61825b6830261de2a973e27
7
- data.tar.gz: bc2182903ab8bc48ed431054f2bf23226277608b017b070d8a7cf697b700dc612bea38de3ff1659b481a788e507e1dba0a3972f15e9434bbba8ad0cb48ecb5f5
6
+ metadata.gz: 86cecfe51fbdb21943f75a98a1ee8e755496f9777bf5c4445ad44ecff45fa5be34fc79c0ad6358ded3886a59ebde4f0cd47598fdcdda41f75c5152489cf5ca35
7
+ data.tar.gz: fec15466a63fc6e49089f339d3f0d9bb3d4d308c7855adf17f23e93ef10e3e09daf6e8453296a53bf74a57d1004883fc2b144c4564342daec8b94b9a7288bd82
checksums.yaml.gz.sig CHANGED
Binary file
@@ -61,8 +61,10 @@ jobs:
61
61
  - name: Run Coditsu
62
62
  run: \curl -sSL https://api.coditsu.io/run/ci | bash
63
63
 
64
+ # We do not split RSpec specs to OSS and Pro like integrations because they do not overload
65
+ # Kafka heavily, compute total coverage for specs and are fast enough
64
66
  specs:
65
- timeout-minutes: 30
67
+ timeout-minutes: 10
66
68
  runs-on: ubuntu-latest
67
69
  needs: diffend
68
70
  strategy:
@@ -88,7 +90,7 @@ jobs:
88
90
 
89
91
  - name: Start Kafka with docker-compose
90
92
  run: |
91
- docker-compose up -d
93
+ docker-compose up -d || (sleep 5 && docker-compose up -d)
92
94
 
93
95
  - name: Set up Ruby
94
96
  uses: ruby/setup-ruby@v1
@@ -96,13 +98,17 @@ jobs:
96
98
  ruby-version: ${{matrix.ruby}}
97
99
  bundler-cache: true
98
100
 
101
+ - name: Wait for Kafka
102
+ run: |
103
+ bundle exec bin/wait_for_kafka
104
+
99
105
  - name: Run all specs
100
106
  env:
101
107
  GITHUB_COVERAGE: ${{matrix.coverage}}
102
108
  run: bin/rspecs
103
109
 
104
- integrations:
105
- timeout-minutes: 45
110
+ integrations_oss:
111
+ timeout-minutes: 20
106
112
  runs-on: ubuntu-latest
107
113
  needs: diffend
108
114
  strategy:
@@ -113,9 +119,6 @@ jobs:
113
119
  - '3.1'
114
120
  - '3.0'
115
121
  - '2.7'
116
- include:
117
- - ruby: '3.2'
118
- coverage: 'true'
119
122
  steps:
120
123
  - uses: actions/checkout@v4
121
124
  - name: Install package dependencies
@@ -126,7 +129,55 @@ jobs:
126
129
 
127
130
  - name: Start Kafka with docker-compose
128
131
  run: |
129
- docker-compose up -d
132
+ docker-compose up -d || (sleep 5 && docker-compose up -d)
133
+
134
+ - name: Set up Ruby
135
+ uses: ruby/setup-ruby@v1
136
+ with:
137
+ # Do not use cache here as we run bundle install also later in some of the integration
138
+ # tests and we need to be able to run it without cache
139
+ #
140
+ # We also want to check that librdkafka is compiling as expected on all versions of Ruby
141
+ ruby-version: ${{matrix.ruby}}
142
+
143
+ - name: Install latest Bundler
144
+ run: |
145
+ gem install bundler --no-document
146
+ gem update --system --no-document
147
+ bundle config set without 'tools benchmarks docs'
148
+
149
+ - name: Bundle install
150
+ run: |
151
+ bundle config set without development
152
+ bundle install
153
+
154
+ - name: Wait for Kafka
155
+ run: |
156
+ bundle exec bin/wait_for_kafka
157
+
158
+ - name: Run OSS integration tests
159
+ run: bin/integrations --exclude '/pro'
160
+
161
+ integrations_pro:
162
+ timeout-minutes: 40
163
+ runs-on: ubuntu-latest
164
+ needs: diffend
165
+ strategy:
166
+ fail-fast: false
167
+ matrix:
168
+ ruby:
169
+ - '3.2'
170
+ - '3.1'
171
+ - '3.0'
172
+ - '2.7'
173
+ steps:
174
+ - uses: actions/checkout@v4
175
+ - name: Install package dependencies
176
+ run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
177
+
178
+ - name: Start Kafka with docker-compose
179
+ run: |
180
+ docker-compose up -d || (sleep 5 && docker-compose up -d)
130
181
 
131
182
  - name: Set up Ruby
132
183
  uses: ruby/setup-ruby@v1
@@ -144,12 +195,16 @@ jobs:
144
195
  bundle config set without development
145
196
  bundle install
146
197
 
147
- - name: Run integration tests
198
+ - name: Wait for Kafka
199
+ run: |
200
+ bundle exec bin/wait_for_kafka
201
+
202
+ - name: Run Pro integration tests
148
203
  env:
149
204
  KARAFKA_PRO_LICENSE_TOKEN: ${{ secrets.KARAFKA_PRO_LICENSE_TOKEN }}
150
205
  KARAFKA_PRO_USERNAME: ${{ secrets.KARAFKA_PRO_USERNAME }}
151
206
  KARAFKA_PRO_PASSWORD: ${{ secrets.KARAFKA_PRO_PASSWORD }}
152
207
  KARAFKA_PRO_VERSION: ${{ secrets.KARAFKA_PRO_VERSION }}
153
208
  KARAFKA_PRO_LICENSE_CHECKSUM: ${{ secrets.KARAFKA_PRO_LICENSE_CHECKSUM }}
154
- GITHUB_COVERAGE: ${{matrix.coverage}}
155
- run: bin/integrations
209
+
210
+ run: bin/integrations '/pro'
data/CHANGELOG.md CHANGED
@@ -1,5 +1,13 @@
1
1
  # Karafka framework changelog
2
2
 
3
+ ## 2.2.5 (2023-09-25)
4
+ - [Enhancement] Ensure, that when topic related operations end, the result is usable. There were few cases where admin operations on topics would finish successfully but internal Kafka caches would not report changes for a short period of time.
5
+ - [Enhancement] Stabilize cooperative-sticky early shutdown procedure.
6
+ - [Fix] use `#nil?` instead of `#present?` on `DataDog::Tracing::SpanOperation` (vitellochris)
7
+ - [Maintenance] Align connection clearing API with Rails 7.1 deprecation warning.
8
+ - [Maintenance] Make `#subscription_group` reference consistent in the Routing and Instrumentation.
9
+ - [Maintenance] Align the consumer pause instrumentation with client pause instrumentation by adding `subscription_group` visibility to the consumer.
10
+
3
11
  ## 2.2.4 (2023-09-13)
4
12
  - [Enhancement] Compensate for potential Kafka cluster drifts vs consumer drift in batch metadata (#1611).
5
13
 
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- karafka (2.2.4)
4
+ karafka (2.2.5)
5
5
  karafka-core (>= 2.2.2, < 2.3.0)
6
6
  thor (>= 0.20)
7
7
  waterdrop (>= 2.6.6, < 3.0.0)
@@ -10,10 +10,10 @@ PATH
10
10
  GEM
11
11
  remote: https://rubygems.org/
12
12
  specs:
13
- activejob (7.0.7)
14
- activesupport (= 7.0.7)
13
+ activejob (7.0.8)
14
+ activesupport (= 7.0.8)
15
15
  globalid (>= 0.3.6)
16
- activesupport (7.0.7)
16
+ activesupport (7.0.8)
17
17
  concurrent-ruby (~> 1.0, >= 1.0.2)
18
18
  i18n (>= 1.6, < 2)
19
19
  minitest (>= 5.1)
@@ -26,28 +26,28 @@ GEM
26
26
  factory_bot (6.3.0)
27
27
  activesupport (>= 5.0.0)
28
28
  ffi (1.15.5)
29
- globalid (1.1.0)
30
- activesupport (>= 5.0)
29
+ globalid (1.2.1)
30
+ activesupport (>= 6.1)
31
31
  i18n (1.14.1)
32
32
  concurrent-ruby (~> 1.0)
33
33
  karafka-core (2.2.2)
34
34
  concurrent-ruby (>= 1.1)
35
35
  karafka-rdkafka (>= 0.13.1, < 0.14.0)
36
- karafka-rdkafka (0.13.4)
36
+ karafka-rdkafka (0.13.5)
37
37
  ffi (~> 1.15)
38
38
  mini_portile2 (~> 2.6)
39
39
  rake (> 12)
40
- karafka-web (0.6.3)
40
+ karafka-web (0.7.4)
41
41
  erubi (~> 1.4)
42
- karafka (>= 2.1.4, < 3.0.0)
43
- karafka-core (>= 2.0.13, < 3.0.0)
44
- roda (~> 3.68, >= 3.68)
42
+ karafka (>= 2.2.3, < 3.0.0)
43
+ karafka-core (>= 2.2.2, < 3.0.0)
44
+ roda (~> 3.68, >= 3.69)
45
45
  tilt (~> 2.0)
46
46
  mini_portile2 (2.8.4)
47
- minitest (5.19.0)
47
+ minitest (5.20.0)
48
48
  rack (3.0.8)
49
49
  rake (13.0.6)
50
- roda (3.71.0)
50
+ roda (3.72.0)
51
51
  rack
52
52
  rspec (3.12.0)
53
53
  rspec-core (~> 3.12.0)
@@ -72,7 +72,7 @@ GEM
72
72
  tilt (2.2.0)
73
73
  tzinfo (2.0.6)
74
74
  concurrent-ruby (~> 1.0)
75
- waterdrop (2.6.6)
75
+ waterdrop (2.6.7)
76
76
  karafka-core (>= 2.1.1, < 3.0.0)
77
77
  zeitwerk (~> 2.3)
78
78
  zeitwerk (2.6.11)
@@ -90,4 +90,4 @@ DEPENDENCIES
90
90
  simplecov
91
91
 
92
92
  BUNDLED WITH
93
- 2.4.12
93
+ 2.4.19
data/bin/benchmarks CHANGED
@@ -13,6 +13,13 @@ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..'))
13
13
 
14
14
  ROOT_PATH = Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../')))
15
15
 
16
+ BENCHMARK_TOPICS = {
17
+ 'benchmarks_00_01' => 1,
18
+ 'benchmarks_00_05' => 5,
19
+ 'benchmarks_01_05' => 5,
20
+ 'benchmarks_00_10' => 10
21
+ }
22
+
16
23
  # Load all the benchmarks
17
24
  benchmarks = Dir[ROOT_PATH.join('spec/benchmarks/**/*.rb')]
18
25
 
@@ -37,6 +44,13 @@ if ENV['SEED']
37
44
  # We make our data json compatible so we can also benchmark serialization
38
45
  elements = Array.new(100_000) { { a: :b }.to_json }
39
46
 
47
+ topics = Karafka::Admin.cluster_info.topics.map { |details| details.fetch(:topic_name) }
48
+
49
+ BENCHMARK_TOPICS.each do |topic_name, partitions_count|
50
+ ::Karafka::Admin.delete_topic(topic_name) if topics.include?(topic_name)
51
+ ::Karafka::Admin.create_topic(topic_name, partitions_count, 1)
52
+ end
53
+
40
54
  # We do not populate data of benchmarks_0_10 as we use it with life-stream data only
41
55
  %w[
42
56
  benchmarks_00_01
data/bin/integrations CHANGED
@@ -218,13 +218,26 @@ end
218
218
  # Load all the specs
219
219
  specs = Dir[ROOT_PATH.join('spec/integrations/**/*_spec.rb')]
220
220
 
221
+ FILTER_TYPE = ARGV[0] == '--exclude' ? 'exclude' : 'include'
222
+
223
+ # Remove the exclude flag
224
+ ARGV.shift if FILTER_TYPE == '--exclude'
225
+
221
226
  # If filters is provided, apply
222
227
  # Allows to provide several filters one after another and applies all of them
223
228
  ARGV.each do |filter|
224
- specs.delete_if { |name| !name.include?(filter) }
229
+ specs.delete_if do |name|
230
+ case FILTER_TYPE
231
+ when 'include'
232
+ !name.include?(filter)
233
+ when 'exclude'
234
+ name.include?(filter)
235
+ else
236
+ raise 'Invalid filter type'
237
+ end
238
+ end
225
239
  end
226
240
 
227
-
228
241
  raise ArgumentError, "No integration specs with filters: #{ARGV.join(', ')}" if specs.empty?
229
242
 
230
243
  # Randomize order
@@ -0,0 +1,24 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ # Waits for Kafka to be ready
4
+ # Useful in CI where Kafka needs to be fully started before we run any tests
5
+
6
+ require 'karafka'
7
+
8
+ Karafka::App.setup do |config|
9
+ config.kafka[:'bootstrap.servers'] = '127.0.0.1:9092'
10
+ end
11
+
12
+ 60.times do
13
+ begin
14
+ # Stop if we can connect to the cluster and get info
15
+ exit if Karafka::Admin.cluster_info
16
+ rescue Rdkafka::RdkafkaError
17
+ puts "Kafka not available, retrying..."
18
+ sleep(1)
19
+ end
20
+ end
21
+
22
+ puts 'Kafka not available!'
23
+
24
+ exit 1
@@ -57,7 +57,7 @@ en:
57
57
  consumer_format: needs to be present
58
58
  id_format: 'needs to be a string with a Kafka accepted format'
59
59
  initial_offset_format: needs to be either earliest or latest
60
- subscription_group_format: must be a non-empty string
60
+ subscription_group_name_format: must be a non-empty string
61
61
  manual_offset_management.active_format: needs to be either true or false
62
62
  consumer_active_job_missing: ActiveJob needs to be available
63
63
  manual_offset_management_must_be_enabled: cannot be disabled for ActiveJob topics
data/docker-compose.yml CHANGED
@@ -1,27 +1,23 @@
1
1
  version: '2'
2
- services:
3
- zookeeper:
4
- container_name: karafka_22_zookeeper
5
- image: wurstmeister/zookeeper
6
- restart: on-failure
7
- ports:
8
- - '2181:2181'
9
2
 
3
+ services:
10
4
  kafka:
11
- container_name: karafka_22_kafka
12
- image: wurstmeister/kafka
5
+ container_name: kafka
6
+ image: confluentinc/cp-kafka:7.5.0
7
+
13
8
  ports:
14
- - '9092:9092'
9
+ - 9092:9092
10
+
15
11
  environment:
16
- KAFKA_ADVERTISED_HOST_NAME: localhost
17
- KAFKA_ADVERTISED_PORT: 9092
18
- KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
12
+ CLUSTER_ID: kafka-docker-cluster-1
13
+ KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
14
+ KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
15
+ KAFKA_PROCESS_ROLES: broker,controller
16
+ KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
17
+ KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093
18
+ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
19
+ KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:9092
20
+ KAFKA_BROKER_ID: 1
21
+ KAFKA_CONTROLLER_QUORUM_VOTERS: 1@127.0.0.1:9093
22
+ ALLOW_PLAINTEXT_LISTENER: 'yes'
19
23
  KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
20
- KAFKA_CREATE_TOPICS:
21
- "benchmarks_00_01:1:1,\
22
- benchmarks_00_05:5:1,\
23
- benchmarks_01_05:5:1,\
24
- benchmarks_00_10:10:1"
25
- volumes:
26
- - /var/run/docker.sock:/var/run/docker.sock
27
- restart: on-failure
data/lib/karafka/admin.rb CHANGED
@@ -216,7 +216,11 @@ module Karafka
216
216
  attempt += 1
217
217
 
218
218
  handler.call
219
- rescue Rdkafka::AbstractHandle::WaitTimeoutError
219
+
220
+ # If breaker does not operate, it means that the requested change was applied but is still
221
+ # not visible and we need to wait
222
+ raise(Errors::ResultNotVisibleError) unless breaker.call
223
+ rescue Rdkafka::AbstractHandle::WaitTimeoutError, Errors::ResultNotVisibleError
220
224
  return if breaker.call
221
225
 
222
226
  retry if attempt <= app_config.admin.max_attempts
@@ -197,6 +197,7 @@ module Karafka
197
197
  manual: manual_pause,
198
198
  topic: topic.name,
199
199
  partition: partition,
200
+ subscription_group: topic.subscription_group,
200
201
  offset: offset,
201
202
  timeout: coordinator.pause_tracker.current_timeout,
202
203
  attempt: coordinator.pause_tracker.attempt
@@ -236,9 +236,23 @@ module Karafka
236
236
  #
237
237
  # @see https://github.com/confluentinc/librdkafka/issues/4312
238
238
  if @subscription_group.kafka[:'partition.assignment.strategy'] == 'cooperative-sticky'
239
+ active_wait = false
240
+
239
241
  (COOPERATIVE_STICKY_MAX_WAIT / 100).times do
240
242
  # If we're past the first rebalance, no need to wait
241
- break if @rebalance_manager.active?
243
+ if @rebalance_manager.active?
244
+ # We give it a a bit of time because librdkafka has a tendency to do some-post
245
+ # callback work that from its perspective is still under rebalance
246
+ sleep(5) if active_wait
247
+
248
+ break
249
+ end
250
+
251
+ active_wait = true
252
+
253
+ # poll to trigger potential rebalances that could occur during stopping and to trigger
254
+ # potential callbacks
255
+ poll(100)
242
256
 
243
257
  sleep(0.1)
244
258
  end
@@ -20,7 +20,7 @@ module Karafka
20
20
  required(:max_wait_time) { |val| val.is_a?(Integer) && val >= 10 }
21
21
  required(:name) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
22
22
  required(:active) { |val| [true, false].include?(val) }
23
- required(:subscription_group) { |val| val.is_a?(String) && !val.empty? }
23
+ required(:subscription_group_name) { |val| val.is_a?(String) && !val.empty? }
24
24
 
25
25
  # Consumer needs to be present only if topic is active
26
26
  # We allow not to define consumer for non-active because they may be only used via admin
@@ -55,5 +55,10 @@ module Karafka
55
55
 
56
56
  # This should never happen. Please open an issue if it does.
57
57
  InvalidTimeBasedOffsetError = Class.new(BaseError)
58
+
59
+ # For internal usage only
60
+ # Raised when we run operations that require certain result but despite successfully finishing
61
+ # it is not yet available due to some synchronization mechanisms and caches
62
+ ResultNotVisibleError = Class.new(BaseError)
58
63
  end
59
64
  end
@@ -75,8 +75,7 @@ module Karafka
75
75
 
76
76
  info "[#{job.id}] #{job_type} job for #{consumer} on #{topic} finished in #{time}ms"
77
77
 
78
- current_span = client.active_span
79
- current_span.finish if current_span.present?
78
+ client.active_span&.finish
80
79
 
81
80
  pop_tags
82
81
  end
@@ -56,6 +56,9 @@ module Karafka
56
56
  topic = consumer_group.public_send(:topic=, discovered_topic, &pattern.config)
57
57
  topic.patterns(active: true, type: :discovered)
58
58
 
59
+ # Assign the appropriate subscription group to this topic
60
+ topic.subscription_group = pattern.topic.subscription_group
61
+
59
62
  # Inject into subscription group topics array always, so everything is reflected
60
63
  # there but since it is not active, will not be picked
61
64
  sg_topics << topic
@@ -81,12 +81,21 @@ if Karafka.rails?
81
81
  end
82
82
 
83
83
  initializer 'karafka.release_active_record_connections' do
84
+ rails7plus = Rails.gem_version >= Gem::Version.new('7.0.0')
85
+
84
86
  ActiveSupport.on_load(:active_record) do
85
87
  ::Karafka::App.monitor.subscribe('worker.completed') do
86
88
  # Always release the connection after processing is done. Otherwise thread may hang
87
89
  # blocking the reload and further processing
88
90
  # @see https://github.com/rails/rails/issues/44183
89
- ActiveRecord::Base.clear_active_connections!
91
+ #
92
+ # The change technically happens in 7.1 but 7.0 already supports this so we can make
93
+ # a proper change for 7.0+
94
+ if rails7plus
95
+ ActiveRecord::Base.connection_handler.clear_active_connections!
96
+ else
97
+ ActiveRecord::Base.clear_active_connections!
98
+ end
90
99
  end
91
100
  end
92
101
  end
@@ -14,7 +14,7 @@ module Karafka
14
14
  # It allows us to store the "current" subscription group defined in the routing
15
15
  # This subscription group id is then injected into topics, so we can compute the subscription
16
16
  # groups
17
- attr_accessor :current_subscription_group_id
17
+ attr_accessor :current_subscription_group_name
18
18
 
19
19
  # @param name [String, Symbol] raw name of this consumer group. Raw means, that it does not
20
20
  # yet have an application client_id namespace, this will be added here by default.
@@ -26,7 +26,7 @@ module Karafka
26
26
  @topics = Topics.new([])
27
27
  # Initialize the subscription group so there's always a value for it, since even if not
28
28
  # defined directly, a subscription group will be created
29
- @current_subscription_group_id = SubscriptionGroup.id
29
+ @current_subscription_group_name = SubscriptionGroup.id
30
30
  end
31
31
 
32
32
  # @return [Boolean] true if this consumer group should be active in our current process
@@ -48,7 +48,7 @@ module Karafka
48
48
  built_topic = @topics.last
49
49
  # We overwrite it conditionally in case it was not set by the user inline in the topic
50
50
  # block definition
51
- built_topic.subscription_group ||= current_subscription_group_id
51
+ built_topic.subscription_group_name ||= current_subscription_group_name
52
52
  built_topic
53
53
  end
54
54
 
@@ -59,13 +59,13 @@ module Karafka
59
59
  def subscription_group=(name = SubscriptionGroup.id, &block)
60
60
  # We cast it here, so the routing supports symbol based but that's anyhow later on
61
61
  # validated as a string
62
- @current_subscription_group_id = name.to_s
62
+ @current_subscription_group_name = name.to_s
63
63
 
64
64
  Proxy.new(self, &block)
65
65
 
66
66
  # We need to reset the current subscription group after it is used, so it won't leak
67
67
  # outside to other topics that would be defined without a defined subscription group
68
- @current_subscription_group_id = SubscriptionGroup.id
68
+ @current_subscription_group_name = SubscriptionGroup.id
69
69
  end
70
70
 
71
71
  # @return [Array<Routing::SubscriptionGroup>] all the subscription groups build based on
@@ -32,7 +32,7 @@ module Karafka
32
32
  # @param topics [Karafka::Routing::Topics] all the topics that share the same key settings
33
33
  # @return [SubscriptionGroup] built subscription group
34
34
  def initialize(position, topics)
35
- @name = topics.first.subscription_group
35
+ @name = topics.first.subscription_group_name
36
36
  @consumer_group = topics.first.consumer_group
37
37
  @id = "#{@name}_#{position}"
38
38
  @position = position
@@ -70,6 +70,12 @@ module Karafka
70
70
  topics.select(&:active?).map(&:subscription_name)
71
71
  end
72
72
 
73
+ # @return [String] id of the subscription group
74
+ # @note This is an alias for displaying in places where we print the stringified version.
75
+ def to_s
76
+ id
77
+ end
78
+
73
79
  private
74
80
 
75
81
  # @return [Hash] kafka settings are a bit special. They are exactly the same for all of the
@@ -87,7 +93,7 @@ module Karafka
87
93
 
88
94
  kafka[:'group.instance.id'] = "#{group_instance_id}_#{@position}" if group_instance_id
89
95
  kafka[:'client.id'] ||= Karafka::App.config.client_id
90
- kafka[:'group.id'] ||= @topics.first.consumer_group.id
96
+ kafka[:'group.id'] ||= @consumer_group.id
91
97
  kafka[:'auto.offset.reset'] ||= @topics.first.initial_offset
92
98
  # Karafka manages the offsets based on the processing state, thus we do not rely on the
93
99
  # rdkafka offset auto-storing
@@ -19,7 +19,7 @@ module Karafka
19
19
  max_messages
20
20
  max_wait_time
21
21
  initial_offset
22
- subscription_group
22
+ subscription_group_name
23
23
  ].freeze
24
24
 
25
25
  private_constant :DISTRIBUTION_KEYS
@@ -39,6 +39,13 @@ module Karafka
39
39
  .map { |value| value.map(&:last) }
40
40
  .map { |topics_array| Routing::Topics.new(topics_array) }
41
41
  .map { |grouped_topics| SubscriptionGroup.new(@position += 1, grouped_topics) }
42
+ .tap do |subscription_groups|
43
+ subscription_groups.each do |subscription_group|
44
+ subscription_group.topics.each do |topic|
45
+ topic.subscription_group = subscription_group
46
+ end
47
+ end
48
+ end
42
49
  end
43
50
 
44
51
  private
@@ -8,6 +8,11 @@ module Karafka
8
8
  class Topic
9
9
  attr_reader :id, :name, :consumer_group
10
10
  attr_writer :consumer
11
+
12
+ attr_accessor :subscription_group_name
13
+
14
+ # Full subscription group reference can be built only when we have knowledge about the
15
+ # whole routing tree, this is why it is going to be set later on
11
16
  attr_accessor :subscription_group
12
17
 
13
18
  # Attributes we can inherit from the root unless they were defined on this level
@@ -112,7 +117,7 @@ module Karafka
112
117
  active: active?,
113
118
  consumer: consumer,
114
119
  consumer_group_id: consumer_group.id,
115
- subscription_group: subscription_group
120
+ subscription_group_name: subscription_group_name
116
121
  ).freeze
117
122
  end
118
123
  end
@@ -3,5 +3,5 @@
3
3
  # Main module namespace
4
4
  module Karafka
5
5
  # Current Karafka version
6
- VERSION = '2.2.4'
6
+ VERSION = '2.2.5'
7
7
  end
data.tar.gz.sig CHANGED
@@ -1,2 +1,2 @@
1
- �K|%��ϸ{��s\T��"ɪ}�UC����rs�+7�{�� 3B��%w�av��U�j�� u����֠����<}!6j~�Ki�V0i�����tSx<������V݁���Y�v���m���W��Eϸ^n��v�;-�Z9���_�U�E��bT�������ċM�B�M�C>Q"��>������̨wS±}�����7�+��'WJUG��+=�-Lv��ɻ=��?w���(Me ���W�MANjYl�M{��Eo�=����pp�k�z!�k�<$��6Ϡ��e�y�5<��;&�y;� Q ��=�Y��}
2
- ���D���}�YŻ
1
+ ��{J��4����cT�8o8jՉ9R!w���>%�?kJ`�Ɨ��%@�h jJ��pq9�*�ٺd�-dE�5d��n��KR�+F���%�@�rDA#Κ`�!��H\�=y ob��#xqGi �@�Ji3ӸΉF8uػ� l�� /��mN���AX�
2
+ �J����cNa�\xD�h���Kp��KA��5T2ګ�t�0I�%��v BD$`�ͻ�3��n���$@T�(���#<��lܱ�K 5L�3�)ܐ���L�L���g�w�(����E �Pb���~�"��+�Y���[��-�+v�)�0#�nO�ŝ�5��ц:�!�^!ƽ�{nل���E2�<M�C�O'���\{�?e��!��Wl����H��\Y
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: karafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.2.4
4
+ version: 2.2.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
35
35
  AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
36
36
  msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
37
37
  -----END CERTIFICATE-----
38
- date: 2023-09-13 00:00:00.000000000 Z
38
+ date: 2023-09-25 00:00:00.000000000 Z
39
39
  dependencies:
40
40
  - !ruby/object:Gem::Dependency
41
41
  name: karafka-core
@@ -147,6 +147,7 @@ files:
147
147
  - bin/stress_many
148
148
  - bin/stress_one
149
149
  - bin/verify_license_integrity
150
+ - bin/wait_for_kafka
150
151
  - certs/cert_chain.pem
151
152
  - certs/karafka-pro.pem
152
153
  - config/locales/errors.yml
metadata.gz.sig CHANGED
Binary file