karafka 2.5.0 → 2.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/{ci.yml → ci_linux_ubuntu_x86_64_gnu.yml} +64 -28
  3. data/.github/workflows/ci_macos_arm64.yml +151 -0
  4. data/.github/workflows/push.yml +2 -2
  5. data/.github/workflows/trigger-wiki-refresh.yml +30 -0
  6. data/.github/workflows/verify-action-pins.yml +1 -1
  7. data/.ruby-version +1 -1
  8. data/CHANGELOG.md +35 -1
  9. data/Gemfile +2 -1
  10. data/Gemfile.lock +57 -28
  11. data/LICENSE-COMM +10 -10
  12. data/README.md +3 -3
  13. data/bin/integrations +3 -1
  14. data/bin/verify_kafka_warnings +2 -1
  15. data/config/locales/errors.yml +153 -152
  16. data/config/locales/pro_errors.yml +135 -134
  17. data/docker-compose.yml +1 -1
  18. data/karafka.gemspec +4 -4
  19. data/lib/active_job/queue_adapters/karafka_adapter.rb +30 -1
  20. data/lib/karafka/active_job/dispatcher.rb +19 -9
  21. data/lib/karafka/admin/acl.rb +7 -8
  22. data/lib/karafka/admin/configs/config.rb +2 -2
  23. data/lib/karafka/admin/configs/resource.rb +2 -2
  24. data/lib/karafka/admin/configs.rb +3 -7
  25. data/lib/karafka/admin/consumer_groups.rb +351 -0
  26. data/lib/karafka/admin/topics.rb +206 -0
  27. data/lib/karafka/admin.rb +42 -451
  28. data/lib/karafka/base_consumer.rb +22 -0
  29. data/lib/karafka/{pro/contracts/server_cli_options.rb → cli/contracts/server.rb} +4 -12
  30. data/lib/karafka/cli/info.rb +1 -1
  31. data/lib/karafka/cli/install.rb +0 -2
  32. data/lib/karafka/cli/server.rb +1 -1
  33. data/lib/karafka/cli/swarm.rb +1 -1
  34. data/lib/karafka/connection/client.rb +12 -3
  35. data/lib/karafka/connection/conductor.rb +1 -1
  36. data/lib/karafka/connection/listener.rb +5 -1
  37. data/lib/karafka/connection/messages_buffer.rb +2 -4
  38. data/lib/karafka/connection/mode.rb +75 -0
  39. data/lib/karafka/connection/status.rb +12 -9
  40. data/lib/karafka/embedded.rb +1 -1
  41. data/lib/karafka/errors.rb +0 -8
  42. data/lib/karafka/execution_mode.rb +103 -0
  43. data/lib/karafka/instrumentation/assignments_tracker.rb +16 -0
  44. data/lib/karafka/instrumentation/logger_listener.rb +109 -50
  45. data/lib/karafka/pro/active_job/dispatcher.rb +5 -0
  46. data/lib/karafka/pro/cleaner/messages/messages.rb +18 -8
  47. data/lib/karafka/pro/cli/contracts/server.rb +106 -0
  48. data/lib/karafka/pro/encryption/contracts/config.rb +1 -1
  49. data/lib/karafka/pro/loader.rb +1 -1
  50. data/lib/karafka/pro/recurring_tasks/contracts/config.rb +1 -1
  51. data/lib/karafka/pro/routing/features/adaptive_iterator/contracts/topic.rb +1 -1
  52. data/lib/karafka/pro/routing/features/adaptive_iterator/topic.rb +9 -0
  53. data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +1 -1
  54. data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb +9 -0
  55. data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb +1 -1
  56. data/lib/karafka/pro/routing/features/delaying/topic.rb +9 -0
  57. data/lib/karafka/pro/routing/features/direct_assignments/contracts/consumer_group.rb +1 -1
  58. data/lib/karafka/pro/routing/features/direct_assignments/contracts/topic.rb +1 -1
  59. data/lib/karafka/pro/routing/features/direct_assignments/topic.rb +9 -0
  60. data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb +1 -1
  61. data/lib/karafka/pro/routing/features/expiring/topic.rb +9 -0
  62. data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb +1 -1
  63. data/lib/karafka/pro/routing/features/filtering/topic.rb +9 -0
  64. data/lib/karafka/pro/routing/features/inline_insights/contracts/topic.rb +1 -1
  65. data/lib/karafka/pro/routing/features/inline_insights/topic.rb +9 -0
  66. data/lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb +1 -1
  67. data/lib/karafka/pro/routing/features/long_running_job/topic.rb +9 -0
  68. data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb +1 -1
  69. data/lib/karafka/pro/routing/features/multiplexing.rb +1 -1
  70. data/lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb +1 -1
  71. data/lib/karafka/pro/routing/features/offset_metadata/topic.rb +9 -0
  72. data/lib/karafka/pro/routing/features/parallel_segments/contracts/consumer_group.rb +1 -1
  73. data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb +1 -1
  74. data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb +1 -1
  75. data/lib/karafka/pro/routing/features/patterns/topic.rb +9 -0
  76. data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb +1 -1
  77. data/lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb +1 -1
  78. data/lib/karafka/pro/routing/features/periodic_job/topic.rb +9 -0
  79. data/lib/karafka/pro/routing/features/recurring_tasks/contracts/topic.rb +1 -1
  80. data/lib/karafka/pro/routing/features/recurring_tasks/topic.rb +9 -0
  81. data/lib/karafka/pro/routing/features/scheduled_messages/contracts/topic.rb +1 -1
  82. data/lib/karafka/pro/routing/features/scheduled_messages/topic.rb +9 -0
  83. data/lib/karafka/pro/routing/features/swarm/contracts/topic.rb +1 -1
  84. data/lib/karafka/pro/routing/features/swarm/topic.rb +9 -0
  85. data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb +1 -1
  86. data/lib/karafka/pro/routing/features/throttling/topic.rb +9 -0
  87. data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +1 -1
  88. data/lib/karafka/pro/routing/features/virtual_partitions/topic.rb +9 -0
  89. data/lib/karafka/pro/scheduled_messages/contracts/config.rb +1 -1
  90. data/lib/karafka/pro/scheduled_messages/daily_buffer.rb +9 -3
  91. data/lib/karafka/pro/swarm/liveness_listener.rb +17 -2
  92. data/lib/karafka/processing/executor.rb +1 -1
  93. data/lib/karafka/processing/jobs_queue.rb +1 -1
  94. data/lib/karafka/routing/builder.rb +0 -3
  95. data/lib/karafka/routing/consumer_group.rb +1 -4
  96. data/lib/karafka/routing/contracts/consumer_group.rb +84 -0
  97. data/lib/karafka/routing/contracts/routing.rb +61 -0
  98. data/lib/karafka/routing/contracts/topic.rb +83 -0
  99. data/lib/karafka/routing/features/active_job/contracts/topic.rb +1 -1
  100. data/lib/karafka/routing/features/active_job/topic.rb +9 -0
  101. data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +1 -1
  102. data/lib/karafka/routing/features/dead_letter_queue/topic.rb +9 -0
  103. data/lib/karafka/routing/features/declaratives/contracts/topic.rb +1 -1
  104. data/lib/karafka/routing/features/declaratives/topic.rb +9 -0
  105. data/lib/karafka/routing/features/deserializers/contracts/topic.rb +1 -1
  106. data/lib/karafka/routing/features/deserializers/topic.rb +9 -0
  107. data/lib/karafka/routing/features/eofed/contracts/topic.rb +1 -1
  108. data/lib/karafka/routing/features/eofed/topic.rb +9 -0
  109. data/lib/karafka/routing/features/inline_insights/contracts/topic.rb +1 -1
  110. data/lib/karafka/routing/features/inline_insights/topic.rb +9 -0
  111. data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb +1 -1
  112. data/lib/karafka/routing/features/manual_offset_management/topic.rb +9 -0
  113. data/lib/karafka/routing/subscription_group.rb +1 -10
  114. data/lib/karafka/routing/topic.rb +9 -1
  115. data/lib/karafka/server.rb +19 -12
  116. data/lib/karafka/setup/attributes_map.rb +36 -0
  117. data/lib/karafka/setup/config.rb +6 -7
  118. data/lib/karafka/setup/contracts/config.rb +217 -0
  119. data/lib/karafka/setup/defaults_injector.rb +3 -1
  120. data/lib/karafka/swarm/node.rb +67 -7
  121. data/lib/karafka/swarm/supervisor.rb +1 -1
  122. data/lib/karafka/swarm.rb +2 -2
  123. data/lib/karafka/templates/karafka.rb.erb +2 -7
  124. data/lib/karafka/version.rb +1 -1
  125. data/lib/karafka.rb +17 -18
  126. metadata +22 -18
  127. data/lib/karafka/contracts/config.rb +0 -210
  128. data/lib/karafka/contracts/consumer_group.rb +0 -81
  129. data/lib/karafka/contracts/routing.rb +0 -59
  130. data/lib/karafka/contracts/server_cli_options.rb +0 -92
  131. data/lib/karafka/contracts/topic.rb +0 -81
  132. data/lib/karafka/processing/timed_queue.rb +0 -62
  133. data/lib/karafka/swarm/pidfd.rb +0 -147
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 03a08ef42e32f92069ef95b4380b744f7188dd2248f296abc752e5cee9d12c7f
4
- data.tar.gz: e23896dcf66e16cddf193ee1b412fb0560dd82e50dcf01b31f3bb93d451afcc3
3
+ metadata.gz: fb41b6fee168ce7585a82c29f8053391ebca63ae3416f4280456f6caeab316a2
4
+ data.tar.gz: 0dbb438af65380581bd0501793d645195359232c9ada649b63e17c9269c95002
5
5
  SHA512:
6
- metadata.gz: 52356bcb5a97f121a6e383bccc7530ef3e5252442ff0a13122b3cfebc8579a00a396576e4965f74473f950b28c820d898d259cffd90b26ff80c40701527cf97f
7
- data.tar.gz: 8fe1550960de8de921e21a45b170a0b1282fdd6d9c5f81f2285cfc5dc958c8d54dbfcb953a997d87891f2ab29f010d8b418987a55825f7d25bb124c717876fab
6
+ metadata.gz: 4862117c97a174e43708fb7e163ab60222a0199b109ca194a172995928968a2799ea5bc39151cee55b0c67a6138f1cdf89e5566a5136f584540ffa0bad4248f3
7
+ data.tar.gz: d0ea2ddbe22b2a85afa77629d68c2af70685fd390d9b3bf87a461d04929cfc3855d40e4c25e57e02d4b78c32107c435e3d039edd9d64feb85f8057e674e77f02
@@ -1,4 +1,4 @@
1
- name: CI
1
+ name: CI Linux x86_64 GNU
2
2
 
3
3
  concurrency:
4
4
  group: ${{ github.workflow }}-${{ github.ref }}
@@ -6,9 +6,7 @@ concurrency:
6
6
 
7
7
  on:
8
8
  pull_request:
9
- branches: [ main, master ]
10
- push:
11
- branches: [ main, master ]
9
+ branches: [ master ]
12
10
  schedule:
13
11
  - cron: '0 1 * * *'
14
12
 
@@ -26,12 +24,12 @@ jobs:
26
24
  strategy:
27
25
  fail-fast: false
28
26
  steps:
29
- - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
27
+ - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
30
28
  with:
31
29
  fetch-depth: 0
32
30
 
33
31
  - name: Set up Ruby
34
- uses: ruby/setup-ruby@13e7a03dc3ac6c3798f4570bfead2aed4d96abfb # v1.244.0
32
+ uses: ruby/setup-ruby@cf7216d52fba1017929b4d7162fabe2b30af5b49 # v1.262.0
35
33
  with:
36
34
  ruby-version: 3.4
37
35
  bundler-cache: true
@@ -48,7 +46,7 @@ jobs:
48
46
  strategy:
49
47
  fail-fast: false
50
48
  steps:
51
- - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
49
+ - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
52
50
  with:
53
51
  fetch-depth: 0
54
52
  - name: Run Karafka license checksum verification
@@ -65,7 +63,7 @@ jobs:
65
63
  strategy:
66
64
  fail-fast: false
67
65
  steps:
68
- - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
66
+ - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
69
67
  with:
70
68
  fetch-depth: 0
71
69
  - name: Download Coditsu script
@@ -90,19 +88,24 @@ jobs:
90
88
  timeout-minutes: 15
91
89
  runs-on: ubuntu-latest
92
90
  needs: diffend
91
+ env:
92
+ BUNDLE_FORCE_RUBY_PLATFORM: ${{ matrix.force_ruby_platform }}
93
93
  strategy:
94
94
  fail-fast: false
95
95
  matrix:
96
96
  ruby:
97
+ - '3.5.0-preview1'
97
98
  - '3.4'
98
99
  - '3.3'
99
100
  - '3.2'
100
- - '3.1'
101
+ force_ruby_platform:
102
+ - true
103
+ - false
101
104
  include:
102
105
  - ruby: '3.4'
103
106
  coverage: 'true'
104
107
  steps:
105
- - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
108
+ - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
106
109
  - name: Install package dependencies
107
110
  run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
108
111
 
@@ -110,15 +113,12 @@ jobs:
110
113
  run: |
111
114
  docker compose up -d || (sleep 5 && docker compose up -d)
112
115
 
113
- # Newer versions of ActiveSupport and Rails do not work with Ruby 3.1 anymore.
114
- # While we use newer by default we do want to resolve older and test, thus we remove
115
- # Gemfile.lock and let it resolve to the most compatible version possible
116
- - name: Remove Gemfile.lock if Ruby 3.1
117
- if: matrix.ruby == '3.1'
116
+ - name: Remove Gemfile.lock for Ruby dev/preview versions
117
+ if: contains(matrix.ruby, 'dev') || contains(matrix.ruby, 'preview') || contains(matrix.ruby, 'rc')
118
118
  run: rm -f Gemfile.lock
119
119
 
120
120
  - name: Set up Ruby
121
- uses: ruby/setup-ruby@13e7a03dc3ac6c3798f4570bfead2aed4d96abfb # v1.244.0
121
+ uses: ruby/setup-ruby@cf7216d52fba1017929b4d7162fabe2b30af5b49 # v1.262.0
122
122
  with:
123
123
  ruby-version: ${{matrix.ruby}}
124
124
  bundler-cache: true
@@ -137,22 +137,27 @@ jobs:
137
137
  run: bin/verify_kafka_warnings
138
138
 
139
139
  - name: Check test topics naming convention
140
- run: bin/verify_topics_naming
140
+ run: bundle exec bin/verify_topics_naming
141
141
 
142
142
  integrations_oss:
143
143
  timeout-minutes: 30
144
144
  runs-on: ubuntu-latest
145
145
  needs: diffend
146
+ env:
147
+ BUNDLE_FORCE_RUBY_PLATFORM: ${{ matrix.force_ruby_platform }}
146
148
  strategy:
147
149
  fail-fast: false
148
150
  matrix:
149
151
  ruby:
152
+ - '3.5.0-preview1'
150
153
  - '3.4'
151
154
  - '3.3'
152
155
  - '3.2'
153
- - '3.1'
156
+ force_ruby_platform:
157
+ - true
158
+ - false
154
159
  steps:
155
- - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
160
+ - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
156
161
  - name: Install package dependencies
157
162
  run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
158
163
 
@@ -164,7 +169,7 @@ jobs:
164
169
  docker compose up -d || (sleep 5 && docker compose up -d)
165
170
 
166
171
  - name: Set up Ruby
167
- uses: ruby/setup-ruby@13e7a03dc3ac6c3798f4570bfead2aed4d96abfb # v1.244.0
172
+ uses: ruby/setup-ruby@cf7216d52fba1017929b4d7162fabe2b30af5b49 # v1.262.0
168
173
  with:
169
174
  # Do not use cache here as we run bundle install also later in some of the integration
170
175
  # tests and we need to be able to run it without cache
@@ -176,8 +181,7 @@ jobs:
176
181
  - name: Install latest Bundler
177
182
  run: |
178
183
  gem install bundler --no-document
179
- gem update --system --no-document
180
-
184
+ gem update --system
181
185
  bundle config set without 'tools benchmarks docs'
182
186
 
183
187
  - name: Fix directory permissions for Bundler
@@ -185,6 +189,10 @@ jobs:
185
189
  chmod -R o-w /opt/hostedtoolcache/Ruby/3*/x64/lib/ruby/gems/3*/gems
186
190
  chmod +t /opt/hostedtoolcache/Ruby/3*/x64/lib/ruby/gems/3*/gems
187
191
 
192
+ - name: Force Ruby platform for ffi gem on dev/preview Ruby
193
+ if: contains(matrix.ruby, 'dev') || contains(matrix.ruby, 'preview') || contains(matrix.ruby, 'rc')
194
+ run: bundle config set force_ruby_platform ffi
195
+
188
196
  - name: Bundle install
189
197
  run: |
190
198
  bundle config set without development
@@ -201,25 +209,30 @@ jobs:
201
209
  run: bin/verify_kafka_warnings
202
210
 
203
211
  - name: Check test topics naming convention
204
- run: bin/verify_topics_naming
212
+ run: bundle exec bin/verify_topics_naming
205
213
 
206
214
  integrations_pro:
207
215
  timeout-minutes: 45
208
216
  runs-on: ubuntu-latest
209
217
  needs: diffend
218
+ env:
219
+ BUNDLE_FORCE_RUBY_PLATFORM: ${{ matrix.force_ruby_platform }}
210
220
  strategy:
211
221
  fail-fast: false
212
222
  matrix:
213
223
  ruby:
224
+ - '3.5.0-preview1'
214
225
  - '3.4'
215
226
  - '3.3'
216
227
  - '3.2'
217
- - '3.1'
228
+ force_ruby_platform:
229
+ - true
230
+ - false
218
231
  parallel_group:
219
232
  - '0'
220
233
  - '1'
221
234
  steps:
222
- - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
235
+ - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
223
236
  - name: Install package dependencies
224
237
  run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
225
238
 
@@ -228,7 +241,7 @@ jobs:
228
241
  docker compose up -d || (sleep 5 && docker compose up -d)
229
242
 
230
243
  - name: Set up Ruby
231
- uses: ruby/setup-ruby@13e7a03dc3ac6c3798f4570bfead2aed4d96abfb # v1.244.0
244
+ uses: ruby/setup-ruby@cf7216d52fba1017929b4d7162fabe2b30af5b49 # v1.262.0
232
245
  with:
233
246
  ruby-version: ${{matrix.ruby}}
234
247
  bundler: 'latest'
@@ -236,10 +249,13 @@ jobs:
236
249
  - name: Install latest Bundler
237
250
  run: |
238
251
  gem install bundler --no-document
239
- gem update --system --no-document
240
-
252
+ gem update --system
241
253
  bundle config set without 'tools benchmarks docs'
242
254
 
255
+ - name: Force Ruby platform for ffi gem on dev/preview Ruby
256
+ if: contains(matrix.ruby, 'dev') || contains(matrix.ruby, 'preview') || contains(matrix.ruby, 'rc')
257
+ run: bundle config set force_ruby_platform ffi
258
+
243
259
  - name: Bundle install
244
260
  run: |
245
261
  bundle config set without development
@@ -266,3 +282,23 @@ jobs:
266
282
 
267
283
  - name: Check test topics naming convention
268
284
  run: bin/verify_topics_naming
285
+
286
+ ci-success:
287
+ name: CI Linux Ubuntu x86_64 GNU Success
288
+ runs-on: ubuntu-latest
289
+ if: always()
290
+ needs:
291
+ - diffend
292
+ - karafka-checksum
293
+ - coditsu
294
+ - specs
295
+ - integrations_oss
296
+ - integrations_pro
297
+ steps:
298
+ - name: Check all jobs passed
299
+ if: |
300
+ contains(needs.*.result, 'failure') ||
301
+ contains(needs.*.result, 'cancelled') ||
302
+ contains(needs.*.result, 'skipped')
303
+ run: exit 1
304
+ - run: echo "All CI checks passed!"
@@ -0,0 +1,151 @@
1
+ name: CI macOS ARM64 Swarm
2
+
3
+ concurrency:
4
+ group: ${{ github.workflow }}-${{ github.ref }}
5
+ cancel-in-progress: true
6
+
7
+ on:
8
+ pull_request:
9
+ branches: [ master ]
10
+ schedule:
11
+ - cron: '0 2 * * *'
12
+
13
+ permissions:
14
+ contents: read
15
+
16
+ env:
17
+ BUNDLE_RETRY: 6
18
+ BUNDLE_JOBS: 4
19
+ CONFLUENT_VERSION: "8.0.0"
20
+
21
+ jobs:
22
+ swarm-macos:
23
+ timeout-minutes: 30
24
+ strategy:
25
+ fail-fast: false
26
+ matrix:
27
+ ruby:
28
+ - '3.5.0-preview1'
29
+ - '3.4'
30
+ - '3.3'
31
+ - '3.2'
32
+ macos-version:
33
+ - 'macos-14'
34
+ - 'macos-15'
35
+ exclude:
36
+ - ruby: '3.5.0-preview1'
37
+ macos-version: 'macos-14'
38
+ runs-on: ${{ matrix.macos-version }}
39
+ steps:
40
+ - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
41
+
42
+ - name: Remove Gemfile.lock for Ruby preview versions
43
+ if: contains(matrix.ruby, 'preview')
44
+ run: rm -f Gemfile.lock
45
+
46
+ - name: Install Bash 4+ and Kerberos
47
+ run: |
48
+ brew install bash
49
+ brew list krb5 &>/dev/null || brew install krb5
50
+ echo "/opt/homebrew/bin" >> $GITHUB_PATH
51
+ - name: Install and Start Confluent Community Kafka (KRaft)
52
+ run: |
53
+ brew install openjdk@17
54
+ export PATH="/opt/homebrew/opt/openjdk@17/bin:$PATH"
55
+ export JAVA_HOME="/opt/homebrew/opt/openjdk@17"
56
+
57
+ curl -O "https://packages.confluent.io/archive/8.0/confluent-community-${CONFLUENT_VERSION}.tar.gz"
58
+ tar -xzf "confluent-community-${CONFLUENT_VERSION}.tar.gz"
59
+
60
+ export CONFLUENT_HOME="$(pwd)/confluent-${CONFLUENT_VERSION}"
61
+ export PATH="$CONFLUENT_HOME/bin:$PATH"
62
+ cd "$CONFLUENT_HOME"
63
+
64
+ # Find the correct server config
65
+ KRAFT_CONFIG=""
66
+ for config in "etc/kafka/kraft/server.properties" "config/kraft/server.properties" "etc/kafka/server.properties"; do
67
+ if [ -f "$config" ]; then
68
+ KRAFT_CONFIG="$config"
69
+ echo "Found config: $KRAFT_CONFIG"
70
+ break
71
+ fi
72
+ done
73
+
74
+ if [ -z "$KRAFT_CONFIG" ]; then
75
+ echo "❌ No server config found"
76
+ exit 1
77
+ fi
78
+
79
+ # Configure KRaft for single-node setup
80
+ cat >> "$KRAFT_CONFIG" << 'EOF'
81
+
82
+ # KRaft mode configuration for single-node setup
83
+ process.roles=broker,controller
84
+ node.id=1
85
+ controller.quorum.voters=1@127.0.0.1:9093
86
+ listeners=PLAINTEXT://0.0.0.0:9092,CONTROLLER://127.0.0.1:9093
87
+ advertised.listeners=PLAINTEXT://127.0.0.1:9092
88
+ controller.listener.names=CONTROLLER
89
+ inter.broker.listener.name=PLAINTEXT
90
+ log.dirs=/tmp/kraft-combined-logs
91
+
92
+ # Enable simple ACL authorization for testing
93
+ authorizer.class.name=org.apache.kafka.metadata.authorizer.StandardAuthorizer
94
+ super.users=User:ANONYMOUS
95
+ allow.everyone.if.no.acl.found=true
96
+ EOF
97
+
98
+ echo "Updated KRaft configuration"
99
+
100
+ CLUSTER_ID=$(bin/kafka-storage random-uuid)
101
+ bin/kafka-storage format -t "$CLUSTER_ID" -c "$KRAFT_CONFIG"
102
+ bin/kafka-server-start "$KRAFT_CONFIG" &
103
+
104
+ sleep 20
105
+
106
+ for i in {1..30}; do
107
+ if bin/kafka-topics --bootstrap-server 127.0.0.1:9092 --list >/dev/null 2>&1; then
108
+ echo "✅ Confluent Community ${CONFLUENT_VERSION} (KRaft) is ready!"
109
+ break
110
+ fi
111
+ [ $i -eq 30 ] && { echo "❌ Kafka failed to start"; exit 1; }
112
+ sleep 2
113
+ done
114
+
115
+ - name: Remove Gemfile.lock for Ruby dev/preview versions
116
+ if: contains(matrix.ruby, 'dev') || contains(matrix.ruby, 'preview') || contains(matrix.ruby, 'rc')
117
+ run: rm -f Gemfile.lock
118
+
119
+ - name: Set up Ruby
120
+ uses: ruby/setup-ruby@cf7216d52fba1017929b4d7162fabe2b30af5b49 # v1.262.0
121
+ with:
122
+ ruby-version: ${{ matrix.ruby }}
123
+ bundler-cache: true
124
+ bundler: 'latest'
125
+
126
+ - name: Run swarm unit specs
127
+ run: bundle exec rspec spec/lib/karafka/swarm/ --tag mode:fork
128
+
129
+ - name: Run all swarm integration specs
130
+ env:
131
+ KARAFKA_PRO_LICENSE_TOKEN: ${{ secrets.KARAFKA_PRO_LICENSE_TOKEN }}
132
+ KARAFKA_PRO_USERNAME: ${{ secrets.KARAFKA_PRO_USERNAME }}
133
+ KARAFKA_PRO_PASSWORD: ${{ secrets.KARAFKA_PRO_PASSWORD }}
134
+ KARAFKA_PRO_VERSION: ${{ secrets.KARAFKA_PRO_VERSION }}
135
+ KARAFKA_PRO_LICENSE_CHECKSUM: ${{ secrets.KARAFKA_PRO_LICENSE_CHECKSUM }}
136
+ run: bin/integrations swarm
137
+
138
+ ci-success-macos:
139
+ name: CI macOS ARM64 Swarm Success
140
+ runs-on: ubuntu-latest
141
+ if: always()
142
+ needs:
143
+ - swarm-macos
144
+ steps:
145
+ - name: Check all jobs passed
146
+ if: |
147
+ contains(needs.*.result, 'failure') ||
148
+ contains(needs.*.result, 'cancelled') ||
149
+ contains(needs.*.result, 'skipped')
150
+ run: exit 1
151
+ - run: echo "All macOS ARM64 Swarm CI checks passed!"
@@ -19,12 +19,12 @@ jobs:
19
19
  id-token: write
20
20
 
21
21
  steps:
22
- - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
22
+ - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
23
23
  with:
24
24
  fetch-depth: 0
25
25
 
26
26
  - name: Set up Ruby
27
- uses: ruby/setup-ruby@13e7a03dc3ac6c3798f4570bfead2aed4d96abfb # v1.244.0
27
+ uses: ruby/setup-ruby@cf7216d52fba1017929b4d7162fabe2b30af5b49 # v1.262.0
28
28
  with:
29
29
  bundler-cache: false
30
30
 
@@ -0,0 +1,30 @@
1
+ name: Trigger Wiki Refresh
2
+
3
+ on:
4
+ release:
5
+ types: [published]
6
+ push:
7
+ branches: [master]
8
+
9
+ jobs:
10
+ trigger-wiki-refresh:
11
+ runs-on: ubuntu-latest
12
+ environment: wiki-trigger
13
+ if: github.repository_owner == 'karafka'
14
+ steps:
15
+ - name: Trigger wiki refresh
16
+ uses: peter-evans/repository-dispatch@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # v3.0.0
17
+ with:
18
+ token: ${{ secrets.WIKI_REPO_TOKEN }}
19
+ repository: karafka/wiki
20
+ event-type: sync-trigger
21
+ client-payload: |
22
+ {
23
+ "repository": "${{ github.repository }}",
24
+ "event_name": "${{ github.event_name }}",
25
+ "release_tag": "${{ github.event.release.tag_name || '' }}",
26
+ "release_name": "${{ github.event.release.name || '' }}",
27
+ "commit_sha": "${{ github.sha }}",
28
+ "commit_message": "Trigger Wiki Refresh",
29
+ "triggered_by": "${{ github.actor }}"
30
+ }
@@ -7,7 +7,7 @@ jobs:
7
7
  verify_action_pins:
8
8
  runs-on: ubuntu-latest
9
9
  steps:
10
- - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
10
+ - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
11
11
  - name: Check SHA pins
12
12
  run: |
13
13
  if grep -E -r "uses: .*/.*@(v[0-9]+|main|master)($|[[:space:]]|$)" --include="*.yml" --include="*.yaml" .github/workflows/ | grep -v "#"; then
data/.ruby-version CHANGED
@@ -1 +1 @@
1
- 3.4.4
1
+ 3.4.6
data/CHANGELOG.md CHANGED
@@ -1,12 +1,46 @@
1
1
  # Karafka Framework Changelog
2
2
 
3
+ ## 2.5.1 (2025-09-29)
4
+ - **[Breaking]** Remove Ruby 3.1 support according to EOL.
5
+ - **[Feature]** Support Swarm mode on MacOS.
6
+ - [Enhancement] Support past `dispatch_at` times with `jitter: 0` in the OSS Karafka to support ActiveJob continuation.
7
+ - [Enhancement] Use direct topic dispatches when `dispatch_at` is used for past times to bypass Scheduled Messages flow.
8
+ - [Enhancement] Support immediate error raising with `auto.offset.reset` set to `error`.
9
+ - [Enhancement] Don't create not needed dirs in the non-Rails setup template.
10
+ - [Enhancement] Improve printing of TTIN to separate threads.
11
+ - [Enhancement] Support transactional ID resource in the Admin ACL management.
12
+ - [Enhancement] Move post-forceful shutdown termination wait value to a config.
13
+ - [Enhancement] Include consumer group, subscription group and other details in error logs for key error locations.
14
+ - [Enhancement] Inherit from `ActiveJob::QueueAdapters::AbstractAdapter` when possible for ActiveJob base class.
15
+ - [Enhancement] Disable Nagle algorithm by default for improved network performance.
16
+ - [Enhancement] Optimize the messages buffer array memory allocation pattern.
17
+ - [Maintenance] Add basic direct DD integration spec via DD gem karafka monitoring feature.
18
+ - [Maintenance] Add integration specs for WaterDrop connection pool usage from within consumers.
19
+ - [Refactoring] Comprehensive Admin module refactoring: Extract topic operations into Admin::Topics class and consumer group operations into Admin::ConsumerGroups class with proper inheritance hierarchy, cross-class method usage optimization, and constants moved to appropriate locations where they are actually used.
20
+ - [Refactoring] Move routing-related contracts from `Karafka::Contracts::` to `Karafka::Routing::Contracts::` namespace and reorganize error message structure in YAML files under `routing:` scope for better code organization and logical grouping.
21
+ - [Refactoring] Move config-related contracts from `Karafka::Contracts::Config` to `Karafka::Setup::Contracts::Config` namespace and reorganize error message structure in YAML files under `setup:` scope for better code organization and logical grouping.
22
+ - [Refactoring] Move CLI server contracts from `Karafka::Contracts::ServerCliOptions` to `Karafka::Cli::Contracts::Server` namespace and reorganize error message structure in YAML files under `cli:` scope for improved naming consistency and logical grouping.
23
+ - [Refactoring] Replace execution mode symbol-based checks with dedicated `ExecutionMode` class providing cleaner API with query methods (`#swarm?`, `#embedded?`) and state change methods (`#swarm!`, `#embedded!`) for improved type safety and code clarity.
24
+ - [Refactoring] Replace connection client mode symbol-based checks with dedicated `Connection::Mode` class providing cleaner API with query methods (`#subscribe?`, `#assign?`) and state change methods (`#subscribe!`, `#assign!`) for improved code clarity.
25
+ - [Fix] Improve same timestamp dispatch in scheduled messages on Ruby 3.2.
26
+ - [Fix] Fix incorrect (6 seconds vs 60 seconds) reset of connections on non-recoverable errors.
27
+ - [Fix] Introduce mutex-safe and thread-safe `#inspect` where needed.
28
+ - [Fix] Fix too loose requirement of Ruby `3.0` when it was `3.1` via transitive dependencies.
29
+ - [Fix] Fix Pro Cleaner Messages compatibility with external libraries that prepend modules to `#each` method (e.g., DataDog tracing).
30
+ - [Fix] SG exclusion in swarm triggers a contract validation error.
31
+ - [Change] Require `waterdrop` `>=` `2.8.10` to support new features.
32
+ - [Change] Require `karafka-rdkafka` `>=` `0.22.0` to support new features and require SSL-bug free version.
33
+ - [Change] Remove no longer needed `cooperative.sticky` rebalance patch.
34
+ - [Change] Normalize how libs and deps are required (no functional change for the end user)
35
+ - [Change] Remove Ruby `3.1` specs according to the EOL schedule.
36
+
3
37
  ## 2.5.0 (2025-06-15)
4
38
  - **[Breaking]** Change how consistency of DLQ dispatches works in Pro (`partition_key` vs. direct partition id mapping).
5
39
  - **[Breaking]** Remove the headers `source_key` from the Pro DLQ dispatched messages as the original key is now fully preserved.
6
40
  - **[Breaking]** Use DLQ and Piping prefix `source_` instead of `original_` to align with naming convention of Kafka Streams and Apache Flink for future usage.
7
41
  - **[Breaking]** Rename scheduled jobs topics names in their config (Pro).
8
42
  - **[Breaking]** Change K8s listener response from `204` to `200` and include JSON body with reasons.
9
- - **[Breaking]** Replace admin config `max_attempts` with `max_retries_duration` and
43
+ - **[Breaking]** Replace admin config `max_attempts` with `max_retries_duration` and `retry_backoff`.
10
44
  - **[Feature]** Parallel Segments for concurrent processing of the same partition with more than partition count of processes (Pro).
11
45
  - [Enhancement] Normalize topic + partition logs format.
12
46
  - [Enhancement] Support KIP-82 (header values of arrays).
data/Gemfile CHANGED
@@ -13,12 +13,13 @@ group :integrations, :test do
13
13
  gem 'fugit', require: false
14
14
  gem 'rspec', require: false
15
15
  gem 'stringio'
16
+ gem 'warning'
16
17
  end
17
18
 
18
19
  group :integrations do
19
20
  gem 'activejob', require: false
20
21
  gem 'karafka-testing', '>= 2.5.0', require: false
21
- gem 'karafka-web', '>= 0.11.0.rc2', require: false
22
+ gem 'karafka-web', '>= 0.11.1', require: false
22
23
  end
23
24
 
24
25
  group :test do