karafka 2.5.2 → 2.5.4.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (201) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +16 -0
  3. data/config/locales/errors.yml +14 -0
  4. data/karafka.gemspec +15 -4
  5. data/lib/active_job/queue_adapters/karafka_adapter.rb +2 -2
  6. data/lib/karafka/active_job/consumer.rb +2 -2
  7. data/lib/karafka/active_job/current_attributes.rb +2 -2
  8. data/lib/karafka/active_job/deserializer.rb +1 -1
  9. data/lib/karafka/active_job/dispatcher.rb +2 -2
  10. data/lib/karafka/admin/configs/resource.rb +7 -1
  11. data/lib/karafka/admin/consumer_groups.rb +6 -8
  12. data/lib/karafka/admin/contracts/replication.rb +149 -0
  13. data/lib/karafka/admin/replication.rb +462 -0
  14. data/lib/karafka/admin/topics.rb +5 -4
  15. data/lib/karafka/admin.rb +57 -12
  16. data/lib/karafka/app.rb +3 -3
  17. data/lib/karafka/base_consumer.rb +1 -1
  18. data/lib/karafka/cli/base.rb +1 -1
  19. data/lib/karafka/cli/console.rb +1 -1
  20. data/lib/karafka/cli/contracts/server.rb +1 -1
  21. data/lib/karafka/cli/help.rb +1 -1
  22. data/lib/karafka/cli/install.rb +2 -1
  23. data/lib/karafka/cli/server.rb +1 -1
  24. data/lib/karafka/cli/swarm.rb +1 -1
  25. data/lib/karafka/connection/client.rb +19 -18
  26. data/lib/karafka/connection/manager.rb +1 -0
  27. data/lib/karafka/connection/proxy.rb +1 -1
  28. data/lib/karafka/connection/rebalance_manager.rb +1 -1
  29. data/lib/karafka/connection/status.rb +1 -0
  30. data/lib/karafka/constraints.rb +1 -1
  31. data/lib/karafka/contracts/base.rb +1 -1
  32. data/lib/karafka/deserializers/payload.rb +1 -1
  33. data/lib/karafka/helpers/async.rb +1 -1
  34. data/lib/karafka/helpers/config_importer.rb +3 -3
  35. data/lib/karafka/helpers/multi_delegator.rb +3 -0
  36. data/lib/karafka/instrumentation/assignments_tracker.rb +2 -1
  37. data/lib/karafka/instrumentation/callbacks/error.rb +2 -2
  38. data/lib/karafka/instrumentation/callbacks/statistics.rb +3 -3
  39. data/lib/karafka/instrumentation/logger.rb +6 -6
  40. data/lib/karafka/instrumentation/logger_listener.rb +0 -2
  41. data/lib/karafka/instrumentation/monitor.rb +2 -2
  42. data/lib/karafka/instrumentation/vendors/appsignal/base.rb +1 -1
  43. data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb +4 -0
  44. data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +32 -16
  45. data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +2 -2
  46. data/lib/karafka/instrumentation/vendors/kubernetes/base_listener.rb +1 -1
  47. data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +3 -15
  48. data/lib/karafka/licenser.rb +1 -1
  49. data/lib/karafka/messages/builders/batch_metadata.rb +1 -1
  50. data/lib/karafka/messages/messages.rb +32 -0
  51. data/lib/karafka/pro/active_job/consumer.rb +2 -2
  52. data/lib/karafka/pro/active_job/dispatcher.rb +3 -3
  53. data/lib/karafka/pro/cleaner/messages/messages.rb +1 -1
  54. data/lib/karafka/pro/cleaner.rb +3 -3
  55. data/lib/karafka/pro/cli/contracts/server.rb +1 -1
  56. data/lib/karafka/pro/cli/parallel_segments/base.rb +4 -3
  57. data/lib/karafka/pro/cli/parallel_segments/collapse.rb +1 -1
  58. data/lib/karafka/pro/cli/parallel_segments/distribute.rb +1 -1
  59. data/lib/karafka/pro/cli/parallel_segments.rb +1 -1
  60. data/lib/karafka/pro/connection/manager.rb +1 -2
  61. data/lib/karafka/pro/connection/multiplexing/listener.rb +1 -0
  62. data/lib/karafka/pro/contracts/base.rb +1 -1
  63. data/lib/karafka/pro/encryption/cipher.rb +3 -2
  64. data/lib/karafka/pro/encryption/contracts/config.rb +1 -1
  65. data/lib/karafka/pro/encryption/messages/parser.rb +1 -1
  66. data/lib/karafka/pro/encryption/setup/config.rb +1 -1
  67. data/lib/karafka/pro/iterator/tpl_builder.rb +1 -1
  68. data/lib/karafka/pro/iterator.rb +1 -1
  69. data/lib/karafka/pro/loader.rb +1 -1
  70. data/lib/karafka/pro/processing/coordinator.rb +1 -1
  71. data/lib/karafka/pro/processing/filters/base.rb +1 -0
  72. data/lib/karafka/pro/processing/filters/delayer.rb +1 -1
  73. data/lib/karafka/pro/processing/filters/expirer.rb +1 -1
  74. data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb +1 -1
  75. data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb +1 -1
  76. data/lib/karafka/pro/processing/jobs/eofed_non_blocking.rb +1 -1
  77. data/lib/karafka/pro/processing/jobs/periodic.rb +1 -1
  78. data/lib/karafka/pro/processing/jobs/revoked_non_blocking.rb +1 -1
  79. data/lib/karafka/pro/processing/jobs_builder.rb +1 -1
  80. data/lib/karafka/pro/processing/jobs_queue.rb +0 -2
  81. data/lib/karafka/pro/processing/offset_metadata/fetcher.rb +1 -0
  82. data/lib/karafka/pro/processing/partitioner.rb +1 -1
  83. data/lib/karafka/pro/processing/strategies/base.rb +1 -1
  84. data/lib/karafka/pro/processing/strategies/default.rb +2 -2
  85. data/lib/karafka/pro/processing/strategies/dlq/default.rb +1 -1
  86. data/lib/karafka/pro/processing/strategies/vp/default.rb +1 -1
  87. data/lib/karafka/pro/processing/strategy_selector.rb +1 -0
  88. data/lib/karafka/pro/processing/virtual_partitions/distributors/balanced.rb +4 -2
  89. data/lib/karafka/pro/processing/virtual_partitions/distributors/consistent.rb +4 -2
  90. data/lib/karafka/pro/recurring_tasks/consumer.rb +3 -2
  91. data/lib/karafka/pro/recurring_tasks/contracts/config.rb +2 -2
  92. data/lib/karafka/pro/recurring_tasks/contracts/task.rb +1 -1
  93. data/lib/karafka/pro/recurring_tasks/deserializer.rb +1 -1
  94. data/lib/karafka/pro/recurring_tasks/dispatcher.rb +1 -1
  95. data/lib/karafka/pro/recurring_tasks/executor.rb +2 -1
  96. data/lib/karafka/pro/recurring_tasks/schedule.rb +5 -2
  97. data/lib/karafka/pro/recurring_tasks/serializer.rb +6 -5
  98. data/lib/karafka/pro/recurring_tasks/setup/config.rb +2 -2
  99. data/lib/karafka/pro/recurring_tasks/task.rb +1 -1
  100. data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb +3 -0
  101. data/lib/karafka/pro/routing/features/multiplexing/subscription_groups_builder.rb +1 -1
  102. data/lib/karafka/pro/routing/features/multiplexing.rb +5 -5
  103. data/lib/karafka/pro/routing/features/offset_metadata.rb +4 -4
  104. data/lib/karafka/pro/routing/features/parallel_segments/builder.rb +1 -1
  105. data/lib/karafka/pro/routing/features/patterns/patterns.rb +1 -1
  106. data/lib/karafka/pro/routing/features/periodic_job/topic.rb +1 -1
  107. data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb +1 -1
  108. data/lib/karafka/pro/routing/features/swarm.rb +1 -1
  109. data/lib/karafka/pro/routing/features/throttling/topic.rb +3 -1
  110. data/lib/karafka/pro/scheduled_messages/consumer.rb +1 -1
  111. data/lib/karafka/pro/scheduled_messages/contracts/config.rb +2 -2
  112. data/lib/karafka/pro/scheduled_messages/contracts/message.rb +1 -1
  113. data/lib/karafka/pro/scheduled_messages/daily_buffer.rb +3 -2
  114. data/lib/karafka/pro/scheduled_messages/day.rb +1 -0
  115. data/lib/karafka/pro/scheduled_messages/deserializers/headers.rb +1 -1
  116. data/lib/karafka/pro/scheduled_messages/deserializers/payload.rb +1 -1
  117. data/lib/karafka/pro/scheduled_messages/max_epoch.rb +1 -0
  118. data/lib/karafka/pro/scheduled_messages/proxy.rb +1 -1
  119. data/lib/karafka/pro/scheduled_messages/serializer.rb +3 -3
  120. data/lib/karafka/pro/scheduled_messages/setup/config.rb +2 -2
  121. data/lib/karafka/pro/scheduled_messages/state.rb +1 -0
  122. data/lib/karafka/pro/scheduled_messages/tracker.rb +1 -0
  123. data/lib/karafka/process.rb +4 -4
  124. data/lib/karafka/processing/executor.rb +1 -1
  125. data/lib/karafka/processing/inline_insights/tracker.rb +1 -0
  126. data/lib/karafka/processing/jobs_queue.rb +1 -1
  127. data/lib/karafka/processing/result.rb +1 -0
  128. data/lib/karafka/processing/strategies/dlq.rb +1 -1
  129. data/lib/karafka/processing/strategy_selector.rb +1 -0
  130. data/lib/karafka/routing/activity_manager.rb +1 -0
  131. data/lib/karafka/routing/builder.rb +3 -1
  132. data/lib/karafka/routing/consumer_group.rb +19 -1
  133. data/lib/karafka/routing/contracts/consumer_group.rb +3 -2
  134. data/lib/karafka/routing/contracts/topic.rb +5 -2
  135. data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +1 -1
  136. data/lib/karafka/routing/features/declaratives/topic.rb +5 -2
  137. data/lib/karafka/routing/features/deserializers/topic.rb +3 -3
  138. data/lib/karafka/routing/features/inline_insights.rb +5 -5
  139. data/lib/karafka/routing/router.rb +1 -1
  140. data/lib/karafka/routing/subscription_group.rb +2 -2
  141. data/lib/karafka/routing/subscription_groups_builder.rb +18 -2
  142. data/lib/karafka/routing/topic.rb +3 -3
  143. data/lib/karafka/server.rb +1 -1
  144. data/lib/karafka/setup/attributes_map.rb +4 -2
  145. data/lib/karafka/setup/config.rb +21 -10
  146. data/lib/karafka/setup/config_proxy.rb +209 -0
  147. data/lib/karafka/setup/contracts/config.rb +1 -1
  148. data/lib/karafka/swarm/liveness_listener.rb +1 -0
  149. data/lib/karafka/swarm/manager.rb +7 -6
  150. data/lib/karafka/swarm/node.rb +1 -1
  151. data/lib/karafka/swarm/supervisor.rb +1 -0
  152. data/lib/karafka/time_trackers/base.rb +1 -1
  153. data/lib/karafka/version.rb +1 -1
  154. data/lib/karafka.rb +2 -3
  155. metadata +8 -65
  156. data/.coditsu/ci.yml +0 -3
  157. data/.console_irbrc +0 -11
  158. data/.github/CODEOWNERS +0 -3
  159. data/.github/FUNDING.yml +0 -1
  160. data/.github/ISSUE_TEMPLATE/bug_report.md +0 -43
  161. data/.github/ISSUE_TEMPLATE/feature_request.md +0 -20
  162. data/.github/workflows/ci_linux_ubuntu_x86_64_gnu.yml +0 -278
  163. data/.github/workflows/ci_macos_arm64.yml +0 -151
  164. data/.github/workflows/push.yml +0 -35
  165. data/.github/workflows/trigger-wiki-refresh.yml +0 -30
  166. data/.github/workflows/verify-action-pins.yml +0 -16
  167. data/.gitignore +0 -69
  168. data/.rspec +0 -7
  169. data/.ruby-gemset +0 -1
  170. data/.ruby-version +0 -1
  171. data/CODE_OF_CONDUCT.md +0 -46
  172. data/CONTRIBUTING.md +0 -32
  173. data/Gemfile +0 -28
  174. data/Gemfile.lock +0 -173
  175. data/Rakefile +0 -4
  176. data/SECURITY.md +0 -23
  177. data/bin/benchmarks +0 -99
  178. data/bin/clean_kafka +0 -43
  179. data/bin/create_token +0 -22
  180. data/bin/integrations +0 -341
  181. data/bin/record_rss +0 -50
  182. data/bin/rspecs +0 -26
  183. data/bin/scenario +0 -29
  184. data/bin/stress_many +0 -13
  185. data/bin/stress_one +0 -13
  186. data/bin/verify_kafka_warnings +0 -36
  187. data/bin/verify_license_integrity +0 -37
  188. data/bin/verify_topics_naming +0 -27
  189. data/bin/wait_for_kafka +0 -24
  190. data/docker-compose.yml +0 -25
  191. data/examples/payloads/avro/.gitkeep +0 -0
  192. data/examples/payloads/json/sample_set_01/enrollment_event.json +0 -579
  193. data/examples/payloads/json/sample_set_01/ingestion_event.json +0 -30
  194. data/examples/payloads/json/sample_set_01/transaction_event.json +0 -17
  195. data/examples/payloads/json/sample_set_01/user_event.json +0 -11
  196. data/examples/payloads/json/sample_set_02/download.json +0 -191
  197. data/examples/payloads/json/sample_set_03/event_type_1.json +0 -18
  198. data/examples/payloads/json/sample_set_03/event_type_2.json +0 -263
  199. data/examples/payloads/json/sample_set_03/event_type_3.json +0 -41
  200. data/log/.gitkeep +0 -0
  201. data/renovate.json +0 -21
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: karafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.5.2
4
+ version: 2.5.4.rc1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Maciej Mensfeld
@@ -9,20 +9,6 @@ bindir: bin
9
9
  cert_chain: []
10
10
  date: 1980-01-02 00:00:00.000000000 Z
11
11
  dependencies:
12
- - !ruby/object:Gem::Dependency
13
- name: base64
14
- requirement: !ruby/object:Gem::Requirement
15
- requirements:
16
- - - "~>"
17
- - !ruby/object:Gem::Version
18
- version: '0.2'
19
- type: :runtime
20
- prerelease: false
21
- version_requirements: !ruby/object:Gem::Requirement
22
- requirements:
23
- - - "~>"
24
- - !ruby/object:Gem::Version
25
- version: '0.2'
26
12
  - !ruby/object:Gem::Dependency
27
13
  name: karafka-core
28
14
  requirement: !ruby/object:Gem::Requirement
@@ -49,21 +35,21 @@ dependencies:
49
35
  requirements:
50
36
  - - ">="
51
37
  - !ruby/object:Gem::Version
52
- version: 0.22.0
38
+ version: 0.23.1
53
39
  type: :runtime
54
40
  prerelease: false
55
41
  version_requirements: !ruby/object:Gem::Requirement
56
42
  requirements:
57
43
  - - ">="
58
44
  - !ruby/object:Gem::Version
59
- version: 0.22.0
45
+ version: 0.23.1
60
46
  - !ruby/object:Gem::Dependency
61
47
  name: waterdrop
62
48
  requirement: !ruby/object:Gem::Requirement
63
49
  requirements:
64
50
  - - ">="
65
51
  - !ruby/object:Gem::Version
66
- version: 2.8.9
52
+ version: 2.8.14
67
53
  - - "<"
68
54
  - !ruby/object:Gem::Version
69
55
  version: 3.0.0
@@ -73,7 +59,7 @@ dependencies:
73
59
  requirements:
74
60
  - - ">="
75
61
  - !ruby/object:Gem::Version
76
- version: 2.8.9
62
+ version: 2.8.14
77
63
  - - "<"
78
64
  - !ruby/object:Gem::Version
79
65
  version: 3.0.0
@@ -103,59 +89,15 @@ executables:
103
89
  extensions: []
104
90
  extra_rdoc_files: []
105
91
  files:
106
- - ".coditsu/ci.yml"
107
- - ".console_irbrc"
108
- - ".github/CODEOWNERS"
109
- - ".github/FUNDING.yml"
110
- - ".github/ISSUE_TEMPLATE/bug_report.md"
111
- - ".github/ISSUE_TEMPLATE/feature_request.md"
112
- - ".github/workflows/ci_linux_ubuntu_x86_64_gnu.yml"
113
- - ".github/workflows/ci_macos_arm64.yml"
114
- - ".github/workflows/push.yml"
115
- - ".github/workflows/trigger-wiki-refresh.yml"
116
- - ".github/workflows/verify-action-pins.yml"
117
- - ".gitignore"
118
- - ".rspec"
119
- - ".ruby-gemset"
120
- - ".ruby-version"
121
92
  - CHANGELOG.md
122
- - CODE_OF_CONDUCT.md
123
- - CONTRIBUTING.md
124
- - Gemfile
125
- - Gemfile.lock
126
93
  - LICENSE
127
94
  - LICENSE-COMM
128
95
  - LICENSE-LGPL
129
96
  - README.md
130
- - Rakefile
131
- - SECURITY.md
132
- - bin/benchmarks
133
- - bin/clean_kafka
134
- - bin/create_token
135
- - bin/integrations
136
97
  - bin/karafka
137
- - bin/record_rss
138
- - bin/rspecs
139
- - bin/scenario
140
- - bin/stress_many
141
- - bin/stress_one
142
- - bin/verify_kafka_warnings
143
- - bin/verify_license_integrity
144
- - bin/verify_topics_naming
145
- - bin/wait_for_kafka
146
98
  - certs/karafka-pro.pem
147
99
  - config/locales/errors.yml
148
100
  - config/locales/pro_errors.yml
149
- - docker-compose.yml
150
- - examples/payloads/avro/.gitkeep
151
- - examples/payloads/json/sample_set_01/enrollment_event.json
152
- - examples/payloads/json/sample_set_01/ingestion_event.json
153
- - examples/payloads/json/sample_set_01/transaction_event.json
154
- - examples/payloads/json/sample_set_01/user_event.json
155
- - examples/payloads/json/sample_set_02/download.json
156
- - examples/payloads/json/sample_set_03/event_type_1.json
157
- - examples/payloads/json/sample_set_03/event_type_2.json
158
- - examples/payloads/json/sample_set_03/event_type_3.json
159
101
  - karafka.gemspec
160
102
  - lib/active_job/karafka.rb
161
103
  - lib/active_job/queue_adapters/karafka_adapter.rb
@@ -175,6 +117,8 @@ files:
175
117
  - lib/karafka/admin/configs/config.rb
176
118
  - lib/karafka/admin/configs/resource.rb
177
119
  - lib/karafka/admin/consumer_groups.rb
120
+ - lib/karafka/admin/contracts/replication.rb
121
+ - lib/karafka/admin/replication.rb
178
122
  - lib/karafka/admin/topics.rb
179
123
  - lib/karafka/app.rb
180
124
  - lib/karafka/base_consumer.rb
@@ -586,6 +530,7 @@ files:
586
530
  - lib/karafka/server.rb
587
531
  - lib/karafka/setup/attributes_map.rb
588
532
  - lib/karafka/setup/config.rb
533
+ - lib/karafka/setup/config_proxy.rb
589
534
  - lib/karafka/setup/contracts/config.rb
590
535
  - lib/karafka/setup/defaults_injector.rb
591
536
  - lib/karafka/setup/dsl.rb
@@ -602,8 +547,6 @@ files:
602
547
  - lib/karafka/time_trackers/pause.rb
603
548
  - lib/karafka/time_trackers/poll.rb
604
549
  - lib/karafka/version.rb
605
- - log/.gitkeep
606
- - renovate.json
607
550
  homepage: https://karafka.io
608
551
  licenses:
609
552
  - LGPL-3.0-only
data/.coditsu/ci.yml DELETED
@@ -1,3 +0,0 @@
1
- repository_id: 'd4482d42-f6b5-44ba-a5e4-00989ac519ee'
2
- api_key: <%= ENV['CODITSU_API_KEY'] %>
3
- api_secret: <%= ENV['CODITSU_API_SECRET'] %>
data/.console_irbrc DELETED
@@ -1,11 +0,0 @@
1
- # irbrc for Karafka console
2
-
3
- IRB.conf[:AUTO_INDENT] = true
4
- IRB.conf[:SAVE_HISTORY] = 1000
5
- IRB.conf[:USE_READLINE] = true
6
- IRB.conf[:HISTORY_FILE] = ".irb-history"
7
- IRB.conf[:LOAD_MODULES] = [] unless IRB.conf.key?(:LOAD_MODULES)
8
-
9
- unless IRB.conf[:LOAD_MODULES].include?('irb/completion')
10
- IRB.conf[:LOAD_MODULES] << 'irb/completion'
11
- end
data/.github/CODEOWNERS DELETED
@@ -1,3 +0,0 @@
1
- /.github @mensfeld
2
- /.github/workflows/ @mensfeld
3
- /.github/actions/ @mensfeld
data/.github/FUNDING.yml DELETED
@@ -1 +0,0 @@
1
- custom: ['https://karafka.io/#become-pro']
@@ -1,43 +0,0 @@
1
- ---
2
- name: Bug Report
3
- about: Report an issue within the Karafka ecosystem you've discovered.
4
- ---
5
-
6
- To make this process smoother for everyone involved, please read the following information before filling out the template.
7
-
8
- Scope of the OSS Support
9
- ===========
10
-
11
- We do not provide OSS support for outdated versions of Karafka and its components.
12
-
13
- Please ensure that you are using a version that is still actively supported. We cannot assist with any no longer maintained versions unless you support us with our Pro offering (https://karafka.io/docs/Pro-Support/).
14
-
15
- We acknowledge that understanding the specifics of your application and its configuration can be essential for resolving certain issues. However, due to the extensive time and resources such analysis can require, this may fall beyond our Open Source Support scope.
16
-
17
- If Karafka or its components are critical to your infrastructure, we encourage you to consider our Pro Offering.
18
-
19
- By backing us up, you can gain direct assistance and ensure your use case receives the dedicated attention it deserves.
20
-
21
-
22
- Important Links to Read
23
- ===========
24
-
25
- Please take a moment to review the following resources before submitting your report:
26
-
27
- - Issue Reporting Guide: https://karafka.io/docs/Support/#issue-reporting-guide
28
- - Support Policy: https://karafka.io/docs/Support/
29
- - Versions, Lifecycle, and EOL: https://karafka.io/docs/Versions-Lifecycle-and-EOL/
30
-
31
-
32
- Bug Report Details
33
- ===========
34
-
35
- Please provide all the details per our Issue Reporting Guide: https://karafka.io/docs/Support/#issue-reporting-guide
36
-
37
- Failing to provide the required details may result in the issue being closed. Please include all necessary information to help us understand and resolve your issue effectively.
38
-
39
-
40
- Additional Context
41
- ===========
42
-
43
- Add any other context about the problem here.
@@ -1,20 +0,0 @@
1
- ---
2
- name: Feature Request
3
- about: Suggest new Karafka features or improvements to existing features.
4
- ---
5
-
6
- ## Is your feature request related to a problem? Please describe.
7
-
8
- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
9
-
10
- ## Describe the solution you'd like
11
-
12
- A clear and concise description of what you want to happen.
13
-
14
- ## Describe alternatives you've considered
15
-
16
- A clear and concise description of any alternative solutions or features you've considered.
17
-
18
- ## Additional context
19
-
20
- Add any other context or screenshots about the feature request here.
@@ -1,278 +0,0 @@
1
- name: CI Linux x86_64 GNU
2
-
3
- concurrency:
4
- group: ${{ github.workflow }}-${{ github.ref }}
5
- cancel-in-progress: true
6
-
7
- on:
8
- pull_request:
9
- branches: [ master ]
10
- schedule:
11
- - cron: '0 1 * * *'
12
-
13
- permissions:
14
- contents: read
15
-
16
- env:
17
- BUNDLE_RETRY: 6
18
- BUNDLE_JOBS: 4
19
-
20
- jobs:
21
- karafka-checksum:
22
- runs-on: ubuntu-latest
23
- timeout-minutes: 5
24
- strategy:
25
- fail-fast: false
26
- steps:
27
- - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
28
- with:
29
- fetch-depth: 0
30
- - name: Run Karafka license checksum verification
31
- env:
32
- KARAFKA_PRO_USERNAME: ${{ secrets.KARAFKA_PRO_USERNAME }}
33
- KARAFKA_PRO_PASSWORD: ${{ secrets.KARAFKA_PRO_PASSWORD }}
34
- KARAFKA_PRO_VERSION: ${{ secrets.KARAFKA_PRO_VERSION }}
35
- KARAFKA_PRO_LICENSE_CHECKSUM: ${{ secrets.KARAFKA_PRO_LICENSE_CHECKSUM }}
36
- run: bin/verify_license_integrity
37
-
38
- coditsu:
39
- runs-on: ubuntu-latest
40
- timeout-minutes: 5
41
- strategy:
42
- fail-fast: false
43
- steps:
44
- - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
45
- with:
46
- fetch-depth: 0
47
- - name: Download Coditsu script
48
- run: |
49
- curl -sSL https://api.coditsu.io/run/ci -o coditsu_script.sh
50
- chmod +x coditsu_script.sh
51
- - name: Verify Coditsu script checksum
52
- run: |
53
- EXPECTED_SHA256="0aecc5aa010f53fca264548a41467a2b0a1208d750ce1da3e98a217304cacbbc"
54
-
55
- ACTUAL_SHA256=$(sha256sum coditsu_script.sh | awk '{ print $1 }')
56
- if [ "$ACTUAL_SHA256" != "$EXPECTED_SHA256" ]; then
57
- echo "::error::Checksum verification failed. Expected $EXPECTED_SHA256 but got $ACTUAL_SHA256."
58
- exit 1
59
- fi
60
- - name: Run Coditsu
61
- run: ./coditsu_script.sh
62
-
63
- # We do not split RSpec specs to OSS and Pro like integrations because they do not overload
64
- # Kafka heavily, compute total coverage for specs and are fast enough
65
- specs:
66
- timeout-minutes: 15
67
- runs-on: ubuntu-latest
68
- env:
69
- BUNDLE_FORCE_RUBY_PLATFORM: ${{ matrix.force_ruby_platform }}
70
- strategy:
71
- fail-fast: false
72
- matrix:
73
- ruby:
74
- - '3.5.0-preview1'
75
- - '3.4'
76
- - '3.3'
77
- - '3.2'
78
- force_ruby_platform:
79
- - true
80
- - false
81
- include:
82
- - ruby: '3.4'
83
- coverage: 'true'
84
- steps:
85
- - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
86
- - name: Install package dependencies
87
- run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
88
-
89
- - name: Start Kafka with docker compose
90
- run: |
91
- docker compose up -d || (sleep 5 && docker compose up -d)
92
-
93
- - name: Remove Gemfile.lock for Ruby dev/preview versions
94
- if: contains(matrix.ruby, 'dev') || contains(matrix.ruby, 'preview') || contains(matrix.ruby, 'rc')
95
- run: rm -f Gemfile.lock
96
-
97
- - name: Set up Ruby
98
- uses: ruby/setup-ruby@d5126b9b3579e429dd52e51e68624dda2e05be25 # v1.267.0
99
- with:
100
- ruby-version: ${{matrix.ruby}}
101
- bundler-cache: true
102
- bundler: 'latest'
103
-
104
- - name: Wait for Kafka
105
- run: |
106
- bundle exec bin/wait_for_kafka
107
-
108
- - name: Run all specs
109
- env:
110
- GITHUB_COVERAGE: ${{matrix.coverage}}
111
- run: bin/rspecs
112
-
113
- - name: Check Kafka logs for unexpected warnings
114
- run: bin/verify_kafka_warnings
115
-
116
- - name: Check test topics naming convention
117
- run: bundle exec bin/verify_topics_naming
118
-
119
- integrations_oss:
120
- timeout-minutes: 30
121
- runs-on: ubuntu-latest
122
- env:
123
- BUNDLE_FORCE_RUBY_PLATFORM: ${{ matrix.force_ruby_platform }}
124
- strategy:
125
- fail-fast: false
126
- matrix:
127
- ruby:
128
- - '3.5.0-preview1'
129
- - '3.4'
130
- - '3.3'
131
- - '3.2'
132
- force_ruby_platform:
133
- - true
134
- - false
135
- steps:
136
- - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
137
- - name: Install package dependencies
138
- run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
139
-
140
- - name: Remove libzstd-dev to check no supported compressions
141
- run: sudo apt-get -y remove libzstd-dev
142
-
143
- - name: Start Kafka with docker compose
144
- run: |
145
- docker compose up -d || (sleep 5 && docker compose up -d)
146
-
147
- - name: Set up Ruby
148
- uses: ruby/setup-ruby@d5126b9b3579e429dd52e51e68624dda2e05be25 # v1.267.0
149
- with:
150
- # Do not use cache here as we run bundle install also later in some of the integration
151
- # tests and we need to be able to run it without cache
152
- #
153
- # We also want to check that librdkafka is compiling as expected on all versions of Ruby
154
- ruby-version: ${{matrix.ruby}}
155
- bundler: 'latest'
156
-
157
- - name: Install latest Bundler
158
- run: |
159
- gem install bundler --no-document
160
- gem update --system
161
- bundle config set without 'tools benchmarks docs'
162
-
163
- - name: Fix directory permissions for Bundler
164
- run: |
165
- chmod -R o-w /opt/hostedtoolcache/Ruby/3*/x64/lib/ruby/gems/3*/gems
166
- chmod +t /opt/hostedtoolcache/Ruby/3*/x64/lib/ruby/gems/3*/gems
167
-
168
- - name: Force Ruby platform for ffi gem on dev/preview Ruby
169
- if: contains(matrix.ruby, 'dev') || contains(matrix.ruby, 'preview') || contains(matrix.ruby, 'rc')
170
- run: bundle config set force_ruby_platform ffi
171
-
172
- - name: Bundle install
173
- run: |
174
- bundle config set without development
175
- bundle install --jobs 4 --retry 3
176
-
177
- - name: Wait for Kafka
178
- run: |
179
- bundle exec bin/wait_for_kafka
180
-
181
- - name: Run OSS integration tests
182
- run: bin/integrations --exclude '/pro'
183
-
184
- - name: Check Kafka logs for unexpected warnings
185
- run: bin/verify_kafka_warnings
186
-
187
- - name: Check test topics naming convention
188
- run: bundle exec bin/verify_topics_naming
189
-
190
- integrations_pro:
191
- timeout-minutes: 45
192
- runs-on: ubuntu-latest
193
- env:
194
- BUNDLE_FORCE_RUBY_PLATFORM: ${{ matrix.force_ruby_platform }}
195
- strategy:
196
- fail-fast: false
197
- matrix:
198
- ruby:
199
- - '3.5.0-preview1'
200
- - '3.4'
201
- - '3.3'
202
- - '3.2'
203
- force_ruby_platform:
204
- - true
205
- - false
206
- parallel_group:
207
- - '0'
208
- - '1'
209
- steps:
210
- - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
211
- - name: Install package dependencies
212
- run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
213
-
214
- - name: Start Kafka with docker compose
215
- run: |
216
- docker compose up -d || (sleep 5 && docker compose up -d)
217
-
218
- - name: Set up Ruby
219
- uses: ruby/setup-ruby@d5126b9b3579e429dd52e51e68624dda2e05be25 # v1.267.0
220
- with:
221
- ruby-version: ${{matrix.ruby}}
222
- bundler: 'latest'
223
-
224
- - name: Install latest Bundler
225
- run: |
226
- gem install bundler --no-document
227
- gem update --system
228
- bundle config set without 'tools benchmarks docs'
229
-
230
- - name: Force Ruby platform for ffi gem on dev/preview Ruby
231
- if: contains(matrix.ruby, 'dev') || contains(matrix.ruby, 'preview') || contains(matrix.ruby, 'rc')
232
- run: bundle config set force_ruby_platform ffi
233
-
234
- - name: Bundle install
235
- run: |
236
- bundle config set without development
237
- bundle install --jobs 4 --retry 3
238
-
239
- - name: Wait for Kafka
240
- run: |
241
- bundle exec bin/wait_for_kafka
242
-
243
- - name: Run Pro integration tests
244
- env:
245
- KARAFKA_PRO_LICENSE_TOKEN: ${{ secrets.KARAFKA_PRO_LICENSE_TOKEN }}
246
- KARAFKA_PRO_USERNAME: ${{ secrets.KARAFKA_PRO_USERNAME }}
247
- KARAFKA_PRO_PASSWORD: ${{ secrets.KARAFKA_PRO_PASSWORD }}
248
- KARAFKA_PRO_VERSION: ${{ secrets.KARAFKA_PRO_VERSION }}
249
- KARAFKA_PRO_LICENSE_CHECKSUM: ${{ secrets.KARAFKA_PRO_LICENSE_CHECKSUM }}
250
- SPECS_SEED: ${{ github.run_id }}
251
- SPECS_GROUP: ${{ matrix.parallel_group }}
252
- run: |
253
- bin/integrations '/pro'
254
-
255
- - name: Check Kafka logs for unexpected warnings
256
- run: bin/verify_kafka_warnings
257
-
258
- - name: Check test topics naming convention
259
- run: bin/verify_topics_naming
260
-
261
- ci-success:
262
- name: CI Linux Ubuntu x86_64 GNU Success
263
- runs-on: ubuntu-latest
264
- if: always()
265
- needs:
266
- - karafka-checksum
267
- - coditsu
268
- - specs
269
- - integrations_oss
270
- - integrations_pro
271
- steps:
272
- - name: Check all jobs passed
273
- if: |
274
- contains(needs.*.result, 'failure') ||
275
- contains(needs.*.result, 'cancelled') ||
276
- contains(needs.*.result, 'skipped')
277
- run: exit 1
278
- - run: echo "All CI checks passed!"
@@ -1,151 +0,0 @@
1
- name: CI macOS ARM64 Swarm
2
-
3
- concurrency:
4
- group: ${{ github.workflow }}-${{ github.ref }}
5
- cancel-in-progress: true
6
-
7
- on:
8
- pull_request:
9
- branches: [ master ]
10
- schedule:
11
- - cron: '0 2 * * *'
12
-
13
- permissions:
14
- contents: read
15
-
16
- env:
17
- BUNDLE_RETRY: 6
18
- BUNDLE_JOBS: 4
19
- CONFLUENT_VERSION: "8.0.0"
20
-
21
- jobs:
22
- swarm-macos:
23
- timeout-minutes: 30
24
- strategy:
25
- fail-fast: false
26
- matrix:
27
- ruby:
28
- - '3.5.0-preview1'
29
- - '3.4'
30
- - '3.3'
31
- - '3.2'
32
- macos-version:
33
- - 'macos-14'
34
- - 'macos-15'
35
- exclude:
36
- - ruby: '3.5.0-preview1'
37
- macos-version: 'macos-14'
38
- runs-on: ${{ matrix.macos-version }}
39
- steps:
40
- - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
41
-
42
- - name: Remove Gemfile.lock for Ruby preview versions
43
- if: contains(matrix.ruby, 'preview')
44
- run: rm -f Gemfile.lock
45
-
46
- - name: Install Bash 4+ and Kerberos
47
- run: |
48
- brew install bash
49
- brew list krb5 &>/dev/null || brew install krb5
50
- echo "/opt/homebrew/bin" >> $GITHUB_PATH
51
- - name: Install and Start Confluent Community Kafka (KRaft)
52
- run: |
53
- brew install openjdk@17
54
- export PATH="/opt/homebrew/opt/openjdk@17/bin:$PATH"
55
- export JAVA_HOME="/opt/homebrew/opt/openjdk@17"
56
-
57
- curl -O "https://packages.confluent.io/archive/8.0/confluent-community-${CONFLUENT_VERSION}.tar.gz"
58
- tar -xzf "confluent-community-${CONFLUENT_VERSION}.tar.gz"
59
-
60
- export CONFLUENT_HOME="$(pwd)/confluent-${CONFLUENT_VERSION}"
61
- export PATH="$CONFLUENT_HOME/bin:$PATH"
62
- cd "$CONFLUENT_HOME"
63
-
64
- # Find the correct server config
65
- KRAFT_CONFIG=""
66
- for config in "etc/kafka/kraft/server.properties" "config/kraft/server.properties" "etc/kafka/server.properties"; do
67
- if [ -f "$config" ]; then
68
- KRAFT_CONFIG="$config"
69
- echo "Found config: $KRAFT_CONFIG"
70
- break
71
- fi
72
- done
73
-
74
- if [ -z "$KRAFT_CONFIG" ]; then
75
- echo "❌ No server config found"
76
- exit 1
77
- fi
78
-
79
- # Configure KRaft for single-node setup
80
- cat >> "$KRAFT_CONFIG" << 'EOF'
81
-
82
- # KRaft mode configuration for single-node setup
83
- process.roles=broker,controller
84
- node.id=1
85
- controller.quorum.voters=1@127.0.0.1:9093
86
- listeners=PLAINTEXT://0.0.0.0:9092,CONTROLLER://127.0.0.1:9093
87
- advertised.listeners=PLAINTEXT://127.0.0.1:9092
88
- controller.listener.names=CONTROLLER
89
- inter.broker.listener.name=PLAINTEXT
90
- log.dirs=/tmp/kraft-combined-logs
91
-
92
- # Enable simple ACL authorization for testing
93
- authorizer.class.name=org.apache.kafka.metadata.authorizer.StandardAuthorizer
94
- super.users=User:ANONYMOUS
95
- allow.everyone.if.no.acl.found=true
96
- EOF
97
-
98
- echo "Updated KRaft configuration"
99
-
100
- CLUSTER_ID=$(bin/kafka-storage random-uuid)
101
- bin/kafka-storage format -t "$CLUSTER_ID" -c "$KRAFT_CONFIG"
102
- bin/kafka-server-start "$KRAFT_CONFIG" &
103
-
104
- sleep 20
105
-
106
- for i in {1..30}; do
107
- if bin/kafka-topics --bootstrap-server 127.0.0.1:9092 --list >/dev/null 2>&1; then
108
- echo "✅ Confluent Community ${CONFLUENT_VERSION} (KRaft) is ready!"
109
- break
110
- fi
111
- [ $i -eq 30 ] && { echo "❌ Kafka failed to start"; exit 1; }
112
- sleep 2
113
- done
114
-
115
- - name: Remove Gemfile.lock for Ruby dev/preview versions
116
- if: contains(matrix.ruby, 'dev') || contains(matrix.ruby, 'preview') || contains(matrix.ruby, 'rc')
117
- run: rm -f Gemfile.lock
118
-
119
- - name: Set up Ruby
120
- uses: ruby/setup-ruby@d5126b9b3579e429dd52e51e68624dda2e05be25 # v1.267.0
121
- with:
122
- ruby-version: ${{ matrix.ruby }}
123
- bundler-cache: true
124
- bundler: 'latest'
125
-
126
- - name: Run swarm unit specs
127
- run: bundle exec rspec spec/lib/karafka/swarm/ --tag mode:fork
128
-
129
- - name: Run all swarm integration specs
130
- env:
131
- KARAFKA_PRO_LICENSE_TOKEN: ${{ secrets.KARAFKA_PRO_LICENSE_TOKEN }}
132
- KARAFKA_PRO_USERNAME: ${{ secrets.KARAFKA_PRO_USERNAME }}
133
- KARAFKA_PRO_PASSWORD: ${{ secrets.KARAFKA_PRO_PASSWORD }}
134
- KARAFKA_PRO_VERSION: ${{ secrets.KARAFKA_PRO_VERSION }}
135
- KARAFKA_PRO_LICENSE_CHECKSUM: ${{ secrets.KARAFKA_PRO_LICENSE_CHECKSUM }}
136
- run: bin/integrations swarm
137
-
138
- ci-success-macos:
139
- name: CI macOS ARM64 Swarm Success
140
- runs-on: ubuntu-latest
141
- if: always()
142
- needs:
143
- - swarm-macos
144
- steps:
145
- - name: Check all jobs passed
146
- if: |
147
- contains(needs.*.result, 'failure') ||
148
- contains(needs.*.result, 'cancelled') ||
149
- contains(needs.*.result, 'skipped')
150
- run: exit 1
151
- - run: echo "All macOS ARM64 Swarm CI checks passed!"