karafka 2.5.1 → 2.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (151) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ci_linux_ubuntu_x86_64_gnu.yml +3 -29
  3. data/.github/workflows/ci_macos_arm64.yml +1 -1
  4. data/.github/workflows/push.yml +2 -2
  5. data/.github/workflows/trigger-wiki-refresh.yml +1 -1
  6. data/.ruby-version +1 -1
  7. data/CHANGELOG.md +14 -4
  8. data/Gemfile +0 -2
  9. data/Gemfile.lock +30 -31
  10. data/bin/integrations +2 -1
  11. data/bin/rspecs +4 -0
  12. data/config/locales/errors.yml +6 -4
  13. data/config/locales/pro_errors.yml +5 -4
  14. data/docker-compose.yml +1 -1
  15. data/examples/payloads/json/sample_set_02/download.json +191 -0
  16. data/examples/payloads/json/sample_set_03/event_type_1.json +18 -0
  17. data/examples/payloads/json/sample_set_03/event_type_2.json +263 -0
  18. data/examples/payloads/json/sample_set_03/event_type_3.json +41 -0
  19. data/karafka.gemspec +1 -1
  20. data/lib/active_job/queue_adapters/karafka_adapter.rb +1 -1
  21. data/lib/karafka/active_job/consumer.rb +5 -1
  22. data/lib/karafka/active_job/current_attributes/job_wrapper.rb +45 -0
  23. data/lib/karafka/active_job/current_attributes/loading.rb +1 -1
  24. data/lib/karafka/active_job/current_attributes/persistence.rb +19 -7
  25. data/lib/karafka/active_job/current_attributes.rb +1 -0
  26. data/lib/karafka/active_job/deserializer.rb +61 -0
  27. data/lib/karafka/active_job/dispatcher.rb +32 -12
  28. data/lib/karafka/active_job/job_options_contract.rb +2 -4
  29. data/lib/karafka/admin/acl.rb +8 -4
  30. data/lib/karafka/admin/configs/config.rb +6 -4
  31. data/lib/karafka/admin/consumer_groups.rb +74 -4
  32. data/lib/karafka/admin/topics.rb +40 -7
  33. data/lib/karafka/admin.rb +13 -4
  34. data/lib/karafka/base_consumer.rb +5 -5
  35. data/lib/karafka/cli/base.rb +1 -1
  36. data/lib/karafka/cli/contracts/server.rb +2 -4
  37. data/lib/karafka/cli/install.rb +1 -1
  38. data/lib/karafka/cli/topics/align.rb +1 -1
  39. data/lib/karafka/cli/topics/repartition.rb +2 -2
  40. data/lib/karafka/connection/client.rb +12 -2
  41. data/lib/karafka/connection/listeners_batch.rb +2 -3
  42. data/lib/karafka/connection/proxy.rb +11 -7
  43. data/lib/karafka/env.rb +1 -2
  44. data/lib/karafka/helpers/interval_runner.rb +4 -1
  45. data/lib/karafka/instrumentation/assignments_tracker.rb +17 -0
  46. data/lib/karafka/instrumentation/monitor.rb +1 -1
  47. data/lib/karafka/instrumentation/notifications.rb +1 -0
  48. data/lib/karafka/instrumentation/vendors/appsignal/base.rb +2 -3
  49. data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +2 -3
  50. data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +8 -9
  51. data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +2 -3
  52. data/lib/karafka/messages/builders/batch_metadata.rb +1 -1
  53. data/lib/karafka/messages/builders/message.rb +1 -1
  54. data/lib/karafka/messages/messages.rb +2 -3
  55. data/lib/karafka/patches/rdkafka/bindings.rb +6 -6
  56. data/lib/karafka/patches/rdkafka/opaque.rb +1 -1
  57. data/lib/karafka/pro/active_job/dispatcher.rb +7 -3
  58. data/lib/karafka/pro/active_job/job_options_contract.rb +2 -4
  59. data/lib/karafka/pro/cleaner/messages/messages.rb +2 -3
  60. data/lib/karafka/pro/cli/contracts/server.rb +2 -4
  61. data/lib/karafka/pro/cli/parallel_segments/base.rb +1 -2
  62. data/lib/karafka/pro/cli/parallel_segments/collapse.rb +2 -2
  63. data/lib/karafka/pro/cli/parallel_segments/distribute.rb +2 -2
  64. data/lib/karafka/pro/connection/manager.rb +2 -2
  65. data/lib/karafka/pro/encryption/contracts/config.rb +4 -6
  66. data/lib/karafka/pro/encryption/messages/parser.rb +3 -3
  67. data/lib/karafka/pro/instrumentation/performance_tracker.rb +3 -3
  68. data/lib/karafka/pro/iterator/expander.rb +1 -1
  69. data/lib/karafka/pro/iterator/tpl_builder.rb +1 -1
  70. data/lib/karafka/pro/iterator.rb +2 -2
  71. data/lib/karafka/pro/processing/coordinators/errors_tracker.rb +2 -3
  72. data/lib/karafka/pro/processing/coordinators/filters_applier.rb +3 -3
  73. data/lib/karafka/pro/processing/filters/delayer.rb +1 -1
  74. data/lib/karafka/pro/processing/filters/expirer.rb +1 -1
  75. data/lib/karafka/pro/processing/filters/throttler.rb +1 -1
  76. data/lib/karafka/pro/processing/schedulers/default.rb +2 -4
  77. data/lib/karafka/pro/processing/strategies/lrj/default.rb +2 -4
  78. data/lib/karafka/pro/processing/strategies/vp/default.rb +2 -4
  79. data/lib/karafka/pro/processing/subscription_groups_coordinator.rb +2 -3
  80. data/lib/karafka/pro/recurring_tasks/contracts/config.rb +2 -4
  81. data/lib/karafka/pro/recurring_tasks/contracts/task.rb +2 -4
  82. data/lib/karafka/pro/recurring_tasks/dispatcher.rb +6 -5
  83. data/lib/karafka/pro/recurring_tasks/schedule.rb +4 -6
  84. data/lib/karafka/pro/recurring_tasks.rb +8 -5
  85. data/lib/karafka/pro/routing/features/adaptive_iterator/contracts/topic.rb +2 -4
  86. data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +2 -4
  87. data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb +2 -4
  88. data/lib/karafka/pro/routing/features/delaying/topic.rb +2 -4
  89. data/lib/karafka/pro/routing/features/direct_assignments/contracts/consumer_group.rb +4 -8
  90. data/lib/karafka/pro/routing/features/direct_assignments/contracts/topic.rb +5 -7
  91. data/lib/karafka/pro/routing/features/direct_assignments/subscription_group.rb +7 -6
  92. data/lib/karafka/pro/routing/features/direct_assignments/topic.rb +2 -2
  93. data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb +2 -4
  94. data/lib/karafka/pro/routing/features/expiring/topic.rb +2 -4
  95. data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb +2 -4
  96. data/lib/karafka/pro/routing/features/filtering/topic.rb +2 -3
  97. data/lib/karafka/pro/routing/features/inline_insights/contracts/topic.rb +2 -4
  98. data/lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb +2 -4
  99. data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb +3 -5
  100. data/lib/karafka/pro/routing/features/non_blocking_job/topic.rb +3 -3
  101. data/lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb +2 -4
  102. data/lib/karafka/pro/routing/features/parallel_segments/contracts/consumer_group.rb +2 -4
  103. data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb +3 -5
  104. data/lib/karafka/pro/routing/features/patterns/contracts/pattern.rb +2 -4
  105. data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb +2 -4
  106. data/lib/karafka/pro/routing/features/pausing/config.rb +26 -0
  107. data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb +17 -11
  108. data/lib/karafka/pro/routing/features/pausing/topic.rb +69 -8
  109. data/lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb +2 -4
  110. data/lib/karafka/pro/routing/features/recurring_tasks/contracts/topic.rb +2 -4
  111. data/lib/karafka/pro/routing/features/scheduled_messages/contracts/topic.rb +2 -4
  112. data/lib/karafka/pro/routing/features/swarm/contracts/routing.rb +2 -4
  113. data/lib/karafka/pro/routing/features/swarm/contracts/topic.rb +6 -8
  114. data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb +2 -4
  115. data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +2 -4
  116. data/lib/karafka/pro/scheduled_messages/contracts/config.rb +2 -4
  117. data/lib/karafka/pro/scheduled_messages/contracts/message.rb +2 -4
  118. data/lib/karafka/pro/scheduled_messages.rb +4 -6
  119. data/lib/karafka/pro/swarm/liveness_listener.rb +2 -2
  120. data/lib/karafka/processing/coordinator.rb +2 -4
  121. data/lib/karafka/processing/coordinators_buffer.rb +2 -3
  122. data/lib/karafka/processing/executor.rb +2 -3
  123. data/lib/karafka/processing/jobs/base.rb +2 -3
  124. data/lib/karafka/processing/workers_batch.rb +2 -3
  125. data/lib/karafka/railtie.rb +1 -0
  126. data/lib/karafka/routing/activity_manager.rb +2 -2
  127. data/lib/karafka/routing/builder.rb +5 -7
  128. data/lib/karafka/routing/consumer_group.rb +4 -6
  129. data/lib/karafka/routing/contracts/consumer_group.rb +3 -5
  130. data/lib/karafka/routing/contracts/routing.rb +2 -4
  131. data/lib/karafka/routing/contracts/topic.rb +2 -4
  132. data/lib/karafka/routing/features/active_job/contracts/topic.rb +2 -4
  133. data/lib/karafka/routing/features/active_job/topic.rb +6 -0
  134. data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +2 -4
  135. data/lib/karafka/routing/features/declaratives/contracts/topic.rb +3 -5
  136. data/lib/karafka/routing/features/deserializers/contracts/topic.rb +2 -4
  137. data/lib/karafka/routing/features/eofed/contracts/topic.rb +2 -4
  138. data/lib/karafka/routing/features/inline_insights/contracts/topic.rb +2 -4
  139. data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb +2 -4
  140. data/lib/karafka/routing/topics.rb +4 -9
  141. data/lib/karafka/server.rb +1 -1
  142. data/lib/karafka/setup/config.rb +66 -9
  143. data/lib/karafka/setup/contracts/config.rb +12 -10
  144. data/lib/karafka/setup/defaults_injector.rb +3 -2
  145. data/lib/karafka/setup/dsl.rb +2 -3
  146. data/lib/karafka/swarm/liveness_listener.rb +2 -3
  147. data/lib/karafka/swarm/supervisor.rb +1 -1
  148. data/lib/karafka/version.rb +1 -1
  149. data/lib/karafka.rb +2 -2
  150. metadata +8 -2
  151. data/.diffend.yml +0 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: fb41b6fee168ce7585a82c29f8053391ebca63ae3416f4280456f6caeab316a2
4
- data.tar.gz: 0dbb438af65380581bd0501793d645195359232c9ada649b63e17c9269c95002
3
+ metadata.gz: bf868e62a336390b27672a8315accbc535b698cef8e0393a93fbae5e36d3ae87
4
+ data.tar.gz: 1bafbd50c3e3c0544fc36e34a3ce01808eda30df67b991aba63200a9453cdc35
5
5
  SHA512:
6
- metadata.gz: 4862117c97a174e43708fb7e163ab60222a0199b109ca194a172995928968a2799ea5bc39151cee55b0c67a6138f1cdf89e5566a5136f584540ffa0bad4248f3
7
- data.tar.gz: d0ea2ddbe22b2a85afa77629d68c2af70685fd390d9b3bf87a461d04929cfc3855d40e4c25e57e02d4b78c32107c435e3d039edd9d64feb85f8057e674e77f02
6
+ metadata.gz: 57714d5fa6d68c7dde6c911a658793d8898531349922109898038e848cc304ee9316b88023095eb0139b74f8c40a155760e998eb36058e2438d2142c566702fa
7
+ data.tar.gz: ac31c2972452d8a17e8797d403fb247f85f9179373329a0519e5f9caece5eab6f1dcd3465b2eae73f3f452a9e46f6c30616cb0f6e25712b2a3ae239fcc5de698
@@ -18,28 +18,6 @@ env:
18
18
  BUNDLE_JOBS: 4
19
19
 
20
20
  jobs:
21
- diffend:
22
- runs-on: ubuntu-latest
23
- timeout-minutes: 5
24
- strategy:
25
- fail-fast: false
26
- steps:
27
- - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
28
- with:
29
- fetch-depth: 0
30
-
31
- - name: Set up Ruby
32
- uses: ruby/setup-ruby@cf7216d52fba1017929b4d7162fabe2b30af5b49 # v1.262.0
33
- with:
34
- ruby-version: 3.4
35
- bundler-cache: true
36
-
37
- - name: Install Diffend plugin
38
- run: bundle plugin install diffend
39
-
40
- - name: Bundle Secure
41
- run: bundle secure
42
-
43
21
  karafka-checksum:
44
22
  runs-on: ubuntu-latest
45
23
  timeout-minutes: 5
@@ -87,7 +65,6 @@ jobs:
87
65
  specs:
88
66
  timeout-minutes: 15
89
67
  runs-on: ubuntu-latest
90
- needs: diffend
91
68
  env:
92
69
  BUNDLE_FORCE_RUBY_PLATFORM: ${{ matrix.force_ruby_platform }}
93
70
  strategy:
@@ -118,7 +95,7 @@ jobs:
118
95
  run: rm -f Gemfile.lock
119
96
 
120
97
  - name: Set up Ruby
121
- uses: ruby/setup-ruby@cf7216d52fba1017929b4d7162fabe2b30af5b49 # v1.262.0
98
+ uses: ruby/setup-ruby@d5126b9b3579e429dd52e51e68624dda2e05be25 # v1.267.0
122
99
  with:
123
100
  ruby-version: ${{matrix.ruby}}
124
101
  bundler-cache: true
@@ -142,7 +119,6 @@ jobs:
142
119
  integrations_oss:
143
120
  timeout-minutes: 30
144
121
  runs-on: ubuntu-latest
145
- needs: diffend
146
122
  env:
147
123
  BUNDLE_FORCE_RUBY_PLATFORM: ${{ matrix.force_ruby_platform }}
148
124
  strategy:
@@ -169,7 +145,7 @@ jobs:
169
145
  docker compose up -d || (sleep 5 && docker compose up -d)
170
146
 
171
147
  - name: Set up Ruby
172
- uses: ruby/setup-ruby@cf7216d52fba1017929b4d7162fabe2b30af5b49 # v1.262.0
148
+ uses: ruby/setup-ruby@d5126b9b3579e429dd52e51e68624dda2e05be25 # v1.267.0
173
149
  with:
174
150
  # Do not use cache here as we run bundle install also later in some of the integration
175
151
  # tests and we need to be able to run it without cache
@@ -214,7 +190,6 @@ jobs:
214
190
  integrations_pro:
215
191
  timeout-minutes: 45
216
192
  runs-on: ubuntu-latest
217
- needs: diffend
218
193
  env:
219
194
  BUNDLE_FORCE_RUBY_PLATFORM: ${{ matrix.force_ruby_platform }}
220
195
  strategy:
@@ -241,7 +216,7 @@ jobs:
241
216
  docker compose up -d || (sleep 5 && docker compose up -d)
242
217
 
243
218
  - name: Set up Ruby
244
- uses: ruby/setup-ruby@cf7216d52fba1017929b4d7162fabe2b30af5b49 # v1.262.0
219
+ uses: ruby/setup-ruby@d5126b9b3579e429dd52e51e68624dda2e05be25 # v1.267.0
245
220
  with:
246
221
  ruby-version: ${{matrix.ruby}}
247
222
  bundler: 'latest'
@@ -288,7 +263,6 @@ jobs:
288
263
  runs-on: ubuntu-latest
289
264
  if: always()
290
265
  needs:
291
- - diffend
292
266
  - karafka-checksum
293
267
  - coditsu
294
268
  - specs
@@ -117,7 +117,7 @@ jobs:
117
117
  run: rm -f Gemfile.lock
118
118
 
119
119
  - name: Set up Ruby
120
- uses: ruby/setup-ruby@cf7216d52fba1017929b4d7162fabe2b30af5b49 # v1.262.0
120
+ uses: ruby/setup-ruby@d5126b9b3579e429dd52e51e68624dda2e05be25 # v1.267.0
121
121
  with:
122
122
  ruby-version: ${{ matrix.ruby }}
123
123
  bundler-cache: true
@@ -24,7 +24,7 @@ jobs:
24
24
  fetch-depth: 0
25
25
 
26
26
  - name: Set up Ruby
27
- uses: ruby/setup-ruby@cf7216d52fba1017929b4d7162fabe2b30af5b49 # v1.262.0
27
+ uses: ruby/setup-ruby@d5126b9b3579e429dd52e51e68624dda2e05be25 # v1.267.0
28
28
  with:
29
29
  bundler-cache: false
30
30
 
@@ -32,4 +32,4 @@ jobs:
32
32
  run: |
33
33
  bundle install --jobs 4 --retry 3
34
34
 
35
- - uses: rubygems/release-gem@a25424ba2ba8b387abc8ef40807c2c85b96cbe32 # v1.1.1
35
+ - uses: rubygems/release-gem@1c162a739e8b4cb21a676e97b087e8268d8fc40b # v1.1.2
@@ -13,7 +13,7 @@ jobs:
13
13
  if: github.repository_owner == 'karafka'
14
14
  steps:
15
15
  - name: Trigger wiki refresh
16
- uses: peter-evans/repository-dispatch@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # v3.0.0
16
+ uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0
17
17
  with:
18
18
  token: ${{ secrets.WIKI_REPO_TOKEN }}
19
19
  repository: karafka/wiki
data/.ruby-version CHANGED
@@ -1 +1 @@
1
- 3.4.6
1
+ 3.4.7
data/CHANGELOG.md CHANGED
@@ -1,7 +1,17 @@
1
1
  # Karafka Framework Changelog
2
2
 
3
+ ## 2.5.2 (2025-10-31)
4
+ - **[EOL]** Remove Rails 7.1 support according to EOL while not blocking Rails 7.1 usage.
5
+ - [Enhancement] Retry on the KIP-848 `stale_member_epoch` error.
6
+ - [Enhancement] Provide `Karafka::Admin.trigger_rebalance` API to programmatically trigger consumer group rebalances for operational purposes.
7
+ - [Enhancement] Nest pause configuration under `config.pause.*` namespace (`config.pause.timeout`, `config.pause.max_timeout`, `config.pause.with_exponential_backoff`) while maintaining backwards compatibility with the old flat API (`config.pause_timeout`, etc.) via delegation methods that will be removed in Karafka 2.6.
8
+ - [Enhancement] Detect and track involuntary assignment loss during long-running processing that exceeds `max.poll.interval.ms` via `client.events_poll` event and automatically update `Karafka::App.assignments` to reflect reality.
9
+ - [Enhancement] Extend `Karafka::Admin.read_watermark_offsets` to accept either a single topic with partition or a hash of multiple topics with partitions, using a single consumer instance for improved efficiency when querying multiple partitions.
10
+ - [Enhancement] Add configurable `Karafka::ActiveJob::Deserializer` to support custom serialization formats (Avro, Protobuf, etc.) for ActiveJob payloads.
11
+ - [Fix] Fix ActiveJob Continuation invalid class reference bug.
12
+
3
13
  ## 2.5.1 (2025-09-29)
4
- - **[Breaking]** Remove Ruby 3.1 support according to EOL.
14
+ - **[EOL]** Remove Ruby 3.1 support according to EOL.
5
15
  - **[Feature]** Support Swarm mode on MacOS.
6
16
  - [Enhancement] Support past `dispatch_at` times with `jitter: 0` in the OSS Karafka to support ActiveJob continuation.
7
17
  - [Enhancement] Use direct topic dispatches when `dispatch_at` is used for past times to bypass Scheduled Messages flow.
@@ -32,7 +42,7 @@
32
42
  - [Change] Require `karafka-rdkafka` `>=` `0.22.0` to support new features and require SSL-bug free version.
33
43
  - [Change] Remove no longer needed `cooperative.sticky` rebalance patch.
34
44
  - [Change] Normalize how libs and deps are required (no functional change for the end user)
35
- - [Change] Remove Ruby `3.1` specs according to the EOL schedule.
45
+ - **[EOL]** Remove Ruby `3.1` specs according to the EOL schedule.
36
46
 
37
47
  ## 2.5.0 (2025-06-15)
38
48
  - **[Breaking]** Change how consistency of DLQ dispatches works in Pro (`partition_key` vs. direct partition id mapping).
@@ -88,7 +98,7 @@
88
98
  - [Maintenance] Lower the `Karafka::Admin` `poll_timeout` to 50 ms to improve responsiveness of admin operations.
89
99
  - [Maintenance] Require `karafka-rdkafka` `>=` `0.19.5` due to usage of `#rd_kafka_global_init`, KIP-82, new producer caching engine and improvements to the `partition_key` assignments.
90
100
  - [Maintenance] Add Deimos routing patch into integration suite not to break it in the future.
91
- - [Maintenance] Remove Rails `7.0` specs due to upcoming EOL.
101
+ - **[EOL]** Remove Rails `7.0` specs due to upcoming EOL.
92
102
  - [Fix] Fix Recurring Tasks and Scheduled Messages not working with Swarm (using closed producer).
93
103
  - [Fix] Fix a case where `unknown_topic_or_part` error could leak out of the consumer on consumer shutdown.
94
104
  - [Fix] Fix missing `virtual_partitions.partitioner.error` custom error logging in the `LoggerListener`.
@@ -254,7 +264,7 @@
254
264
 
255
265
  This release contains **BREAKING** changes. Make sure to read and apply upgrade notes.
256
266
 
257
- - **[Breaking]** Drop Ruby `2.7` support.
267
+ - **[EOL]** Drop Ruby `2.7` support.
258
268
  - **[Breaking]** Drop the concept of consumer group mapping.
259
269
  - **[Breaking]** `karafka topics migrate` will now perform declarative topics configuration alignment.
260
270
  - **[Breaking]** Replace `deserializer` config with `#deserializers` in routing to support key and lazy header deserializers.
data/Gemfile CHANGED
@@ -2,8 +2,6 @@
2
2
 
3
3
  source 'https://rubygems.org'
4
4
 
5
- plugin 'diffend'
6
-
7
5
  gemspec
8
6
 
9
7
  # Karafka gem does not require activejob, karafka-web or fugit to work
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- karafka (2.5.1)
4
+ karafka (2.5.2)
5
5
  base64 (~> 0.2)
6
6
  karafka-core (>= 2.5.6, < 2.6.0)
7
7
  karafka-rdkafka (>= 0.22.0)
@@ -11,25 +11,24 @@ PATH
11
11
  GEM
12
12
  remote: https://rubygems.org/
13
13
  specs:
14
- activejob (8.0.3)
15
- activesupport (= 8.0.3)
14
+ activejob (8.1.1)
15
+ activesupport (= 8.1.1)
16
16
  globalid (>= 0.3.6)
17
- activesupport (8.0.3)
17
+ activesupport (8.1.1)
18
18
  base64
19
- benchmark (>= 0.3)
20
19
  bigdecimal
21
20
  concurrent-ruby (~> 1.0, >= 1.3.1)
22
21
  connection_pool (>= 2.2.5)
23
22
  drb
24
23
  i18n (>= 1.6, < 2)
24
+ json
25
25
  logger (>= 1.4.2)
26
26
  minitest (>= 5.1)
27
27
  securerandom (>= 0.3)
28
28
  tzinfo (~> 2.0, >= 2.0.5)
29
29
  uri (>= 0.13.1)
30
30
  base64 (0.3.0)
31
- benchmark (0.4.1)
32
- bigdecimal (3.2.3)
31
+ bigdecimal (3.3.1)
33
32
  byebug (12.0.0)
34
33
  concurrent-ruby (1.3.5)
35
34
  connection_pool (2.5.4)
@@ -37,9 +36,9 @@ GEM
37
36
  docile (1.4.1)
38
37
  drb (2.2.3)
39
38
  erubi (1.13.1)
40
- et-orbi (1.3.0)
39
+ et-orbi (1.4.0)
41
40
  tzinfo
42
- factory_bot (6.5.5)
41
+ factory_bot (6.5.6)
43
42
  activesupport (>= 6.1.0)
44
43
  ffi (1.17.2)
45
44
  ffi (1.17.2-aarch64-linux-gnu)
@@ -52,78 +51,78 @@ GEM
52
51
  ffi (1.17.2-x86_64-darwin)
53
52
  ffi (1.17.2-x86_64-linux-gnu)
54
53
  ffi (1.17.2-x86_64-linux-musl)
55
- fugit (1.11.2)
56
- et-orbi (~> 1, >= 1.2.11)
54
+ fugit (1.12.1)
55
+ et-orbi (~> 1.4)
57
56
  raabro (~> 1.4)
58
- globalid (1.2.1)
57
+ globalid (1.3.0)
59
58
  activesupport (>= 6.1)
60
59
  i18n (1.14.7)
61
60
  concurrent-ruby (~> 1.0)
62
- json (2.15.0)
61
+ json (2.15.2)
63
62
  karafka-core (2.5.7)
64
63
  karafka-rdkafka (>= 0.20.0)
65
64
  logger (>= 1.6.0)
66
- karafka-rdkafka (0.22.0)
65
+ karafka-rdkafka (0.22.2)
67
66
  ffi (~> 1.15)
68
67
  json (> 2.0)
69
68
  logger
70
69
  mini_portile2 (~> 2.6)
71
70
  rake (> 12)
72
- karafka-rdkafka (0.22.0-aarch64-linux-gnu)
71
+ karafka-rdkafka (0.22.2-aarch64-linux-gnu)
73
72
  ffi (~> 1.15)
74
73
  json (> 2.0)
75
74
  logger
76
75
  mini_portile2 (~> 2.6)
77
76
  rake (> 12)
78
- karafka-rdkafka (0.22.0-arm64-darwin)
77
+ karafka-rdkafka (0.22.2-arm64-darwin)
79
78
  ffi (~> 1.15)
80
79
  json (> 2.0)
81
80
  logger
82
81
  mini_portile2 (~> 2.6)
83
82
  rake (> 12)
84
- karafka-rdkafka (0.22.0-x86_64-linux-gnu)
83
+ karafka-rdkafka (0.22.2-x86_64-linux-gnu)
85
84
  ffi (~> 1.15)
86
85
  json (> 2.0)
87
86
  logger
88
87
  mini_portile2 (~> 2.6)
89
88
  rake (> 12)
90
- karafka-rdkafka (0.22.0-x86_64-linux-musl)
89
+ karafka-rdkafka (0.22.2-x86_64-linux-musl)
91
90
  ffi (~> 1.15)
92
91
  json (> 2.0)
93
92
  logger
94
93
  mini_portile2 (~> 2.6)
95
94
  rake (> 12)
96
- karafka-testing (2.5.3)
97
- karafka (>= 2.5.0.beta1, < 2.6.0)
95
+ karafka-testing (2.5.4)
96
+ karafka (>= 2.5.0, < 2.6.0)
98
97
  waterdrop (>= 2.8.0)
99
- karafka-web (0.11.2)
98
+ karafka-web (0.11.3)
100
99
  erubi (~> 1.4)
101
- karafka (>= 2.5.0.rc2, < 2.6.0)
100
+ karafka (>= 2.5.0, < 2.6.0)
102
101
  karafka-core (>= 2.5.0, < 2.6.0)
103
102
  roda (~> 3.68, >= 3.69)
104
103
  tilt (~> 2.0)
105
104
  logger (1.7.0)
106
105
  mini_portile2 (2.8.9)
107
- minitest (5.25.5)
106
+ minitest (5.26.0)
108
107
  ostruct (0.6.3)
109
108
  raabro (1.4.0)
110
- rack (3.2.1)
109
+ rack (3.2.3)
111
110
  rake (13.3.0)
112
- roda (3.95.0)
111
+ roda (3.97.0)
113
112
  rack
114
- rspec (3.13.1)
113
+ rspec (3.13.2)
115
114
  rspec-core (~> 3.13.0)
116
115
  rspec-expectations (~> 3.13.0)
117
116
  rspec-mocks (~> 3.13.0)
118
- rspec-core (3.13.5)
117
+ rspec-core (3.13.6)
119
118
  rspec-support (~> 3.13.0)
120
119
  rspec-expectations (3.13.5)
121
120
  diff-lcs (>= 1.2.0, < 2.0)
122
121
  rspec-support (~> 3.13.0)
123
- rspec-mocks (3.13.5)
122
+ rspec-mocks (3.13.6)
124
123
  diff-lcs (>= 1.2.0, < 2.0)
125
124
  rspec-support (~> 3.13.0)
126
- rspec-support (3.13.5)
125
+ rspec-support (3.13.6)
127
126
  securerandom (0.4.1)
128
127
  simplecov (0.22.0)
129
128
  docile (~> 1.1)
@@ -135,9 +134,9 @@ GEM
135
134
  tilt (2.6.1)
136
135
  tzinfo (2.0.6)
137
136
  concurrent-ruby (~> 1.0)
138
- uri (1.0.3)
137
+ uri (1.0.4)
139
138
  warning (1.5.0)
140
- waterdrop (2.8.10)
139
+ waterdrop (2.8.12)
141
140
  karafka-core (>= 2.4.9, < 3.0.0)
142
141
  karafka-rdkafka (>= 0.20.0)
143
142
  zeitwerk (~> 2.3)
data/bin/integrations CHANGED
@@ -52,7 +52,8 @@ class Scenario
52
52
  'cli/declaratives/delete/existing_with_exit_code_spec.rb' => [2].freeze,
53
53
  'cli/declaratives/create/new_with_exit_code_spec.rb' => [2].freeze,
54
54
  'cli/declaratives/plan/when_changes_with_detailed_exit_code_spec.rb' => [2].freeze,
55
- 'cli/declaratives/align/incorrectly_spec.rb' => [1].freeze
55
+ 'cli/declaratives/align/incorrectly_spec.rb' => [1].freeze,
56
+ 'setup/with_kip_848_protocol_incorrect_config_spec.rb' => [1].freeze
56
57
  }.freeze
57
58
 
58
59
  private_constant :MAX_RUN_TIME, :EXIT_CODES
data/bin/rspecs CHANGED
@@ -2,6 +2,10 @@
2
2
 
3
3
  set -e
4
4
 
5
+ # We remove old coverage because under heavy development and parallel spec execution, the
6
+ # rspec results formatter tends to crash
7
+ rm -rf ./coverage
8
+
5
9
  # Run only regular non-forking specs first
6
10
  SPECS_TYPE=regular bundle exec rspec \
7
11
  --tag ~type:pro \
@@ -11,9 +11,11 @@ en:
11
11
  group_id_format: 'needs to be a string with a Kafka accepted format'
12
12
  concurrency_format: needs to be an integer bigger than 0
13
13
  consumer_persistence_format: needs to be either true or false
14
- pause_timeout_format: needs to be an integer bigger than 0
15
- pause_max_timeout_format: needs to be an integer bigger than 0
16
- pause_with_exponential_backoff_format: needs to be either true or false
14
+
15
+ pause.timeout_format: needs to be an integer bigger than 0
16
+ pause.max_timeout_format: needs to be an integer bigger than 0
17
+ pause.with_exponential_backoff_format: needs to be either true or false
18
+
17
19
  strict_topics_namespacing_format: needs to be either true or false
18
20
  strict_declarative_topics_format: needs to be either true or false
19
21
  shutdown_timeout_format: needs to be an integer bigger than 0
@@ -23,7 +25,7 @@ en:
23
25
  Decrease max_wait_time or increase node_report_timeout
24
26
  kafka_format: needs to be a filled hash
25
27
  key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
26
- max_timeout_vs_pause_max_timeout: pause_timeout must be less or equal to pause_max_timeout
28
+ pause.timeout_max_timeout_vs_pause_max_timeout: pause.timeout must be less or equal to pause.max_timeout
27
29
  shutdown_timeout_vs_max_wait_time: shutdown_timeout must be more than max_wait_time
28
30
  worker_thread_priority_format: must be between -3 and 3
29
31
 
@@ -31,10 +31,11 @@ en:
31
31
  delaying.delay_format: 'needs to be equal or more than 0 and an integer'
32
32
  delaying.active_format: 'needs to be boolean'
33
33
 
34
- pause_timeout_format: needs to be an integer bigger than 0
35
- pause_max_timeout_format: needs to be an integer bigger than 0
36
- pause_with_exponential_backoff_format: needs to be either true or false
37
- pause_timeout_max_timeout_vs_pause_max_timeout: pause_timeout must be less or equal to pause_max_timeout
34
+ pausing.active_format: 'needs to be boolean'
35
+ pausing.timeout_format: needs to be an integer bigger than 0
36
+ pausing.max_timeout_format: needs to be an integer bigger than 0
37
+ pausing.with_exponential_backoff_format: needs to be either true or false
38
+ pausing.timeout_max_timeout_vs_pause_max_timeout: timeout must be less or equal to max_timeout
38
39
 
39
40
  patterns.active_format: 'needs to be boolean'
40
41
  patterns.type_format: 'needs to be :matcher, :discovered or :regular'
data/docker-compose.yml CHANGED
@@ -1,7 +1,7 @@
1
1
  services:
2
2
  kafka:
3
3
  container_name: kafka
4
- image: confluentinc/cp-kafka:8.0.1
4
+ image: confluentinc/cp-kafka:8.1.0
5
5
 
6
6
  ports:
7
7
  - 9092:9092
@@ -0,0 +1,191 @@
1
+ {
2
+ "audience_data_versions": {
3
+ "s1_build_epoch": 1634594595,
4
+ "s2_build_epoch": 1634093288,
5
+ "s3_build_epoch": 1634095356
6
+ },
7
+ "combination_cache_hits": null,
8
+ "content_insertion_info": {
9
+ "ad_rep_account_revenue": {
10
+ "code": "USD",
11
+ "scale": "Nano",
12
+ "value": 0
13
+ },
14
+ "ad_rep_account": {
15
+ "id": "f8a3c7d2-9b41-4e56-a2f3-8d7c1b9e4a6f",
16
+ "name": "AdOps_Beta"
17
+ },
18
+ "advertisement": {
19
+ "duration": {
20
+ "scale": "Micro",
21
+ "value": 29022040
22
+ },
23
+ "external_identifier": null,
24
+ "id": "a9e2f4b7-3c81-4d92-b5e6-7a4f8c2d1e9b",
25
+ "title": "soda",
26
+ "type": "TraditionalAd"
27
+ },
28
+ "platform_revenue": {
29
+ "code": "USD",
30
+ "scale": "Nano",
31
+ "value": 0
32
+ },
33
+ "audience": null,
34
+ "brand": {
35
+ "id": "d4c8e1a9-7f62-4b3d-9e85-c2a7f4b9d1e6",
36
+ "name": "Brand_Beta"
37
+ },
38
+ "campaign": {
39
+ "id": "b7d3a5e8-4c91-4f2b-a6d9-1e8c7f3a5b4d",
40
+ "title": "Pepsi Max",
41
+ "frequency_counting_targets": null
42
+ },
43
+ "campaign_value": {
44
+ "code": "USD",
45
+ "scale": "Nano",
46
+ "value": 500000
47
+ },
48
+ "normalized_campaign_value": {
49
+ "code": "USD",
50
+ "scale": "Nano",
51
+ "value": 500000
52
+ },
53
+ "external_payload_id": "c5e9a2f7-8d41-4b36-9e72-a1f4c8b3d7e5",
54
+ "marker_point": {
55
+ "id": "e2f7c9a4-1d58-4b63-8e49-f7a3c2d8b1e6",
56
+ "episode_version_id": "a8c4e7f2-9b31-4d65-b8a9-e3f7c1d4a2b6",
57
+ "maximum_content_count": 1,
58
+ "maximum_content_duration": {
59
+ "scale": "Micro",
60
+ "value": 120000000
61
+ },
62
+ "position_type": "Preroll",
63
+ "position_sequence": 1
64
+ },
65
+ "position_sequence": 1,
66
+ "series_revenue": {
67
+ "code": "USD",
68
+ "scale": "Nano",
69
+ "value": 0
70
+ }
71
+ },
72
+ "content_insertion_system_type": "ServerSide",
73
+ "count": 1,
74
+ "deduplication_keys": null,
75
+ "distribution_characteristic_id": "c9e3f2a7-4d81-4b56-9e73-a8f2c4d1b7e6",
76
+ "distribution_identification_method_id": "f7a2c8e4-9b31-4d65-8e49-b3f7a1c2d9e8",
77
+ "episode": {
78
+ "created_at": 1698698168000,
79
+ "capacities": [
80
+ {
81
+ "position_type": "Preroll",
82
+ "count": 1,
83
+ "duration": 120
84
+ },
85
+ {
86
+ "position_type": "Midroll",
87
+ "count": 0,
88
+ "duration": 0
89
+ },
90
+ {
91
+ "position_type": "Postroll",
92
+ "count": 0,
93
+ "duration": 0
94
+ }
95
+ ],
96
+ "duration": {
97
+ "scale": "Micro",
98
+ "value": 4204747760
99
+ },
100
+ "episode_number": null,
101
+ "id": "d8e2a7f4-9c51-4b36-a8e2-f7c4d1b9a3e6",
102
+ "title": "Mountain Breeze",
103
+ "episode_version_id": "a8c4e7f2-9b31-4d65-b8a9-e3f7c1d4a2b6"
104
+ },
105
+ "feed": {
106
+ "id": "b4f9c7e2-8a31-4d56-9e82-a7f3c1d4b8e9",
107
+ "type": null
108
+ },
109
+ "feed_item": {
110
+ "id": "d8e2a7f4-9c51-4b36-a8e2-f7c4d1b9a3e6"
111
+ },
112
+ "id": "e7c2f9a4-8d31-4b65-9e78-a3f4c7d2b1e9",
113
+ "listened_at": 1747928515000000,
114
+ "listened_at_ms": 1747928515000,
115
+ "listener": {
116
+ "connection_type": "Cellular",
117
+ "ip_address": "172.58.243.167",
118
+ "ip_address_encryption_format": "None",
119
+ "isp": {
120
+ "as_organization": "WIRELESS-MOBILE",
121
+ "asn": 8445,
122
+ "name": "Random Telecom",
123
+ "organization": "Random Telecom"
124
+ },
125
+ "location": {
126
+ "continent": {
127
+ "code": "NA",
128
+ "geoname_id": 6255149,
129
+ "name": "North America"
130
+ },
131
+ "country": {
132
+ "code": "US",
133
+ "geoname_id": 6252001
134
+ },
135
+ "designated_market_area": {
136
+ "code": "524",
137
+ "name": "Atlanta, GA"
138
+ },
139
+ "is_in_eu": null,
140
+ "locality": {
141
+ "geoname_id": 4180439,
142
+ "name": "Atlanta"
143
+ },
144
+ "region": {
145
+ "geoname_id": 4197000,
146
+ "name": "Georgia"
147
+ },
148
+ "lat": 33.748997,
149
+ "lng": -84.387985,
150
+ "postal_code": "20309"
151
+ },
152
+ "rss_browser": null,
153
+ "time_zone": "America/New_York",
154
+ "user_agent": {
155
+ "browser": "Safari",
156
+ "mobile": true,
157
+ "operating_system": "iOS",
158
+ "platform": null,
159
+ "user_agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 16_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5 Mobile/15E148 Safari/604.1",
160
+ "user_agent_hash": "9k742p/X5m"
161
+ }
162
+ },
163
+ "network": null,
164
+ "playback_group_id": "f9c3e7a2-8d41-4b56-9e73-a8f2c4d1b7e6",
165
+ "request_duration": 0,
166
+ "request_headers": {
167
+ "SERVER_PORT": "80",
168
+ "SERVER_PROTOCOL": "HTTP/1.1",
169
+ "REMOTE_ADDR": "172.58.243.167",
170
+ "ORIGINAL_SCRIPT_NAME": "",
171
+ "HTTP_VERSION": "HTTP/1.1",
172
+ "HTTP_HOST": "foo.bar.com",
173
+ "REQUEST_METHOD": "GET",
174
+ "HTTP_ACCEPT": "*/*",
175
+ "SCRIPT_NAME": "",
176
+ "HTTP_USER_AGENT": "Mozilla/5.0 (iPhone; CPU iPhone OS 16_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5 Mobile/15E148 Safari/604.1",
177
+ "REQUEST_PATH": "/paths/d8e2a7f4-9c51-4b36-a8e2-f7c4d1b9a3e6",
178
+ "HTTP_CONNECTION": "close"
179
+ },
180
+ "rejection_status": "NotRejected",
181
+ "root_network": null,
182
+ "season": null,
183
+ "series": {
184
+ "id": "c8e4a7f2-9b31-4d65-8e49-b3f7a1c2d9e8",
185
+ "title": "Podcast"
186
+ },
187
+ "session_id": "a7f2c9e4-8d31-4b56-9e73-a8f4c2d1b7e6",
188
+ "statistics_system": "TwentyFourHourWindows",
189
+ "tracking_id": "b9e3f7a2-4c81-4d56-9e78-a3f4c7d2b1e9",
190
+ "tracking_id_source": "UniqueClientAttributes"
191
+ }
@@ -0,0 +1,18 @@
1
+ {
2
+ "connect.name": "production.public.automation_actions.Key",
3
+ "fields": [
4
+ {
5
+ "default": "00000000-0000-0000-0000-000000000000",
6
+ "name": "id",
7
+ "type": {
8
+ "connect.default": "00000000-0000-0000-0000-000000000000",
9
+ "connect.name": "io.debezium.data.Uuid",
10
+ "connect.version": 1,
11
+ "type": "string"
12
+ }
13
+ }
14
+ ],
15
+ "name": "Key",
16
+ "namespace": "production.public.automation_actions",
17
+ "type": "record"
18
+ }