karafka 2.5.5 → 2.5.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (215) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +20 -0
  3. data/LICENSE-COMM +4 -0
  4. data/README.md +2 -2
  5. data/certs/expired.txt +2 -0
  6. data/karafka.gemspec +23 -23
  7. data/lib/active_job/karafka.rb +2 -2
  8. data/lib/active_job/queue_adapters/karafka_adapter.rb +5 -5
  9. data/lib/karafka/active_job/consumer.rb +3 -3
  10. data/lib/karafka/active_job/current_attributes.rb +4 -4
  11. data/lib/karafka/active_job/job_options_contract.rb +2 -2
  12. data/lib/karafka/admin/acl.rb +3 -3
  13. data/lib/karafka/admin/configs/resource.rb +1 -1
  14. data/lib/karafka/admin/configs.rb +1 -1
  15. data/lib/karafka/admin/consumer_groups.rb +8 -8
  16. data/lib/karafka/admin/contracts/replication.rb +2 -2
  17. data/lib/karafka/admin/replication.rb +21 -21
  18. data/lib/karafka/admin/topics.rb +6 -6
  19. data/lib/karafka/admin.rb +4 -5
  20. data/lib/karafka/app.rb +3 -3
  21. data/lib/karafka/base_consumer.rb +34 -30
  22. data/lib/karafka/cli/base.rb +8 -8
  23. data/lib/karafka/cli/console.rb +1 -1
  24. data/lib/karafka/cli/contracts/server.rb +12 -12
  25. data/lib/karafka/cli/help.rb +2 -2
  26. data/lib/karafka/cli/info.rb +4 -4
  27. data/lib/karafka/cli/install.rb +11 -11
  28. data/lib/karafka/cli/server.rb +6 -6
  29. data/lib/karafka/cli/swarm.rb +1 -1
  30. data/lib/karafka/cli/topics/align.rb +4 -4
  31. data/lib/karafka/cli/topics/base.rb +5 -5
  32. data/lib/karafka/cli/topics/create.rb +2 -2
  33. data/lib/karafka/cli/topics/delete.rb +2 -2
  34. data/lib/karafka/cli/topics/help.rb +5 -1
  35. data/lib/karafka/cli/topics/plan.rb +16 -16
  36. data/lib/karafka/cli/topics/repartition.rb +3 -3
  37. data/lib/karafka/cli/topics.rb +22 -22
  38. data/lib/karafka/cli.rb +2 -2
  39. data/lib/karafka/connection/client.rb +17 -17
  40. data/lib/karafka/connection/listener.rb +6 -6
  41. data/lib/karafka/connection/mode.rb +1 -1
  42. data/lib/karafka/connection/proxy.rb +1 -1
  43. data/lib/karafka/connection/status.rb +2 -2
  44. data/lib/karafka/constraints.rb +3 -3
  45. data/lib/karafka/embedded.rb +3 -3
  46. data/lib/karafka/env.rb +4 -4
  47. data/lib/karafka/errors.rb +6 -1
  48. data/lib/karafka/execution_mode.rb +1 -1
  49. data/lib/karafka/helpers/config_importer.rb +2 -2
  50. data/lib/karafka/helpers/interval_runner.rb +4 -2
  51. data/lib/karafka/helpers/multi_delegator.rb +1 -1
  52. data/lib/karafka/instrumentation/assignments_tracker.rb +9 -9
  53. data/lib/karafka/instrumentation/callbacks/error.rb +5 -5
  54. data/lib/karafka/instrumentation/callbacks/oauthbearer_token_refresh.rb +4 -4
  55. data/lib/karafka/instrumentation/callbacks/rebalance.rb +6 -6
  56. data/lib/karafka/instrumentation/callbacks/statistics.rb +5 -5
  57. data/lib/karafka/instrumentation/logger.rb +7 -7
  58. data/lib/karafka/instrumentation/logger_listener.rb +76 -63
  59. data/lib/karafka/instrumentation/vendors/appsignal/base.rb +1 -1
  60. data/lib/karafka/instrumentation/vendors/appsignal/client.rb +1 -1
  61. data/lib/karafka/instrumentation/vendors/appsignal/errors_listener.rb +1 -1
  62. data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb +36 -36
  63. data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +33 -28
  64. data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +38 -38
  65. data/lib/karafka/instrumentation/vendors/kubernetes/base_listener.rb +5 -5
  66. data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +1 -1
  67. data/lib/karafka/instrumentation/vendors/kubernetes/swarm_liveness_listener.rb +1 -1
  68. data/lib/karafka/licenser.rb +115 -8
  69. data/lib/karafka/messages/builders/batch_metadata.rb +4 -2
  70. data/lib/karafka/messages/messages.rb +1 -1
  71. data/lib/karafka/patches/rdkafka/bindings.rb +2 -2
  72. data/lib/karafka/pro/active_job/job_options_contract.rb +2 -2
  73. data/lib/karafka/pro/cleaner/messages/messages.rb +10 -0
  74. data/lib/karafka/pro/cli/contracts/server.rb +12 -12
  75. data/lib/karafka/pro/cli/parallel_segments/base.rb +4 -4
  76. data/lib/karafka/pro/cli/parallel_segments/collapse.rb +5 -5
  77. data/lib/karafka/pro/cli/parallel_segments/distribute.rb +3 -3
  78. data/lib/karafka/pro/cli/parallel_segments.rb +7 -7
  79. data/lib/karafka/pro/cli/topics/health.rb +162 -0
  80. data/lib/karafka/pro/cli/topics.rb +52 -0
  81. data/lib/karafka/pro/connection/manager.rb +14 -14
  82. data/lib/karafka/pro/encryption/contracts/config.rb +2 -2
  83. data/lib/karafka/pro/encryption/messages/middleware.rb +2 -2
  84. data/lib/karafka/pro/encryption/messages/parser.rb +2 -2
  85. data/lib/karafka/pro/encryption/setup/config.rb +2 -2
  86. data/lib/karafka/pro/iterator/tpl_builder.rb +2 -2
  87. data/lib/karafka/pro/iterator.rb +1 -1
  88. data/lib/karafka/pro/loader.rb +2 -1
  89. data/lib/karafka/pro/processing/adaptive_iterator/consumer.rb +1 -1
  90. data/lib/karafka/pro/processing/coordinators/virtual_offset_manager.rb +24 -14
  91. data/lib/karafka/pro/processing/filters/base.rb +1 -1
  92. data/lib/karafka/pro/processing/filters/delayer.rb +2 -2
  93. data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb +1 -1
  94. data/lib/karafka/pro/processing/offset_metadata/consumer.rb +1 -1
  95. data/lib/karafka/pro/processing/parallel_segments/filters/base.rb +6 -6
  96. data/lib/karafka/pro/processing/partitioner.rb +3 -3
  97. data/lib/karafka/pro/processing/periodic_job/consumer.rb +6 -5
  98. data/lib/karafka/pro/processing/piping/consumer.rb +7 -7
  99. data/lib/karafka/pro/processing/schedulers/base.rb +5 -5
  100. data/lib/karafka/pro/processing/schedulers/default.rb +5 -5
  101. data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb +6 -3
  102. data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb +6 -3
  103. data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom_vp.rb +6 -3
  104. data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb +2 -2
  105. data/lib/karafka/pro/processing/strategies/default.rb +22 -22
  106. data/lib/karafka/pro/processing/strategies/dlq/default.rb +7 -7
  107. data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb +6 -3
  108. data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb +6 -3
  109. data/lib/karafka/pro/processing/strategies/ftr/default.rb +2 -2
  110. data/lib/karafka/pro/processing/strategies/lrj/default.rb +2 -2
  111. data/lib/karafka/pro/processing/strategies/lrj/ftr.rb +6 -3
  112. data/lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb +6 -3
  113. data/lib/karafka/pro/processing/strategies/lrj/mom.rb +2 -2
  114. data/lib/karafka/pro/recurring_tasks/consumer.rb +2 -2
  115. data/lib/karafka/pro/recurring_tasks/contracts/config.rb +2 -2
  116. data/lib/karafka/pro/recurring_tasks/contracts/task.rb +2 -2
  117. data/lib/karafka/pro/recurring_tasks/dispatcher.rb +2 -2
  118. data/lib/karafka/pro/recurring_tasks/listener.rb +1 -1
  119. data/lib/karafka/pro/recurring_tasks/matcher.rb +2 -2
  120. data/lib/karafka/pro/recurring_tasks/serializer.rb +5 -5
  121. data/lib/karafka/pro/recurring_tasks/setup/config.rb +3 -3
  122. data/lib/karafka/pro/recurring_tasks/task.rb +4 -4
  123. data/lib/karafka/pro/recurring_tasks.rb +4 -4
  124. data/lib/karafka/pro/routing/features/adaptive_iterator/contracts/topic.rb +2 -2
  125. data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +2 -2
  126. data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb +1 -1
  127. data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb +2 -2
  128. data/lib/karafka/pro/routing/features/direct_assignments/contracts/consumer_group.rb +2 -2
  129. data/lib/karafka/pro/routing/features/direct_assignments/contracts/topic.rb +2 -2
  130. data/lib/karafka/pro/routing/features/direct_assignments/topic.rb +1 -1
  131. data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb +2 -2
  132. data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb +2 -2
  133. data/lib/karafka/pro/routing/features/inline_insights/contracts/topic.rb +2 -2
  134. data/lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb +2 -2
  135. data/lib/karafka/pro/routing/features/long_running_job/topic.rb +1 -1
  136. data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb +2 -2
  137. data/lib/karafka/pro/routing/features/multiplexing.rb +5 -5
  138. data/lib/karafka/pro/routing/features/non_blocking_job/topic.rb +1 -1
  139. data/lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb +2 -2
  140. data/lib/karafka/pro/routing/features/offset_metadata/topic.rb +1 -1
  141. data/lib/karafka/pro/routing/features/offset_metadata.rb +1 -1
  142. data/lib/karafka/pro/routing/features/parallel_segments/consumer_group.rb +5 -5
  143. data/lib/karafka/pro/routing/features/parallel_segments/contracts/consumer_group.rb +2 -2
  144. data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb +2 -2
  145. data/lib/karafka/pro/routing/features/patterns/contracts/pattern.rb +3 -3
  146. data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb +2 -2
  147. data/lib/karafka/pro/routing/features/patterns/topic.rb +1 -1
  148. data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb +2 -2
  149. data/lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb +2 -2
  150. data/lib/karafka/pro/routing/features/periodic_job/topic.rb +1 -1
  151. data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb +7 -7
  152. data/lib/karafka/pro/routing/features/recurring_tasks/contracts/topic.rb +2 -2
  153. data/lib/karafka/pro/routing/features/scheduled_messages/builder.rb +13 -13
  154. data/lib/karafka/pro/routing/features/scheduled_messages/contracts/topic.rb +2 -2
  155. data/lib/karafka/pro/routing/features/swarm/contracts/routing.rb +2 -2
  156. data/lib/karafka/pro/routing/features/swarm/contracts/topic.rb +2 -2
  157. data/lib/karafka/pro/routing/features/swarm.rb +1 -1
  158. data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb +2 -2
  159. data/lib/karafka/pro/routing/features/virtual_partitions/config.rb +7 -7
  160. data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +2 -2
  161. data/lib/karafka/pro/scheduled_messages/consumer.rb +4 -4
  162. data/lib/karafka/pro/scheduled_messages/contracts/config.rb +2 -2
  163. data/lib/karafka/pro/scheduled_messages/contracts/message.rb +10 -10
  164. data/lib/karafka/pro/scheduled_messages/daily_buffer.rb +2 -2
  165. data/lib/karafka/pro/scheduled_messages/deserializers/headers.rb +4 -4
  166. data/lib/karafka/pro/scheduled_messages/dispatcher.rb +5 -5
  167. data/lib/karafka/pro/scheduled_messages/proxy.rb +8 -8
  168. data/lib/karafka/pro/scheduled_messages/schema_validator.rb +1 -1
  169. data/lib/karafka/pro/scheduled_messages/setup/config.rb +2 -2
  170. data/lib/karafka/pro/scheduled_messages/state.rb +1 -1
  171. data/lib/karafka/pro/scheduled_messages/tracker.rb +2 -2
  172. data/lib/karafka/pro/scheduled_messages.rb +2 -2
  173. data/lib/karafka/pro/swarm/liveness_listener.rb +2 -2
  174. data/lib/karafka/process.rb +1 -1
  175. data/lib/karafka/processing/coordinator.rb +1 -1
  176. data/lib/karafka/processing/inline_insights/consumer.rb +4 -4
  177. data/lib/karafka/processing/inline_insights/tracker.rb +6 -6
  178. data/lib/karafka/processing/jobs/base.rb +6 -4
  179. data/lib/karafka/processing/jobs_queue.rb +10 -0
  180. data/lib/karafka/processing/schedulers/default.rb +4 -4
  181. data/lib/karafka/processing/strategies/base.rb +6 -6
  182. data/lib/karafka/processing/strategies/default.rb +13 -13
  183. data/lib/karafka/processing/strategies/dlq.rb +1 -1
  184. data/lib/karafka/processing/worker.rb +5 -5
  185. data/lib/karafka/railtie.rb +11 -11
  186. data/lib/karafka/routing/builder.rb +3 -3
  187. data/lib/karafka/routing/contracts/consumer_group.rb +6 -6
  188. data/lib/karafka/routing/contracts/routing.rb +2 -2
  189. data/lib/karafka/routing/contracts/topic.rb +4 -4
  190. data/lib/karafka/routing/features/active_job/contracts/topic.rb +3 -3
  191. data/lib/karafka/routing/features/base/expander.rb +4 -4
  192. data/lib/karafka/routing/features/base.rb +8 -8
  193. data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +2 -2
  194. data/lib/karafka/routing/features/declaratives/contracts/topic.rb +2 -2
  195. data/lib/karafka/routing/features/deserializers/contracts/topic.rb +2 -2
  196. data/lib/karafka/routing/features/eofed/contracts/topic.rb +3 -3
  197. data/lib/karafka/routing/features/inline_insights/contracts/topic.rb +2 -2
  198. data/lib/karafka/routing/features/inline_insights.rb +7 -7
  199. data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb +2 -2
  200. data/lib/karafka/routing/subscription_group.rb +9 -9
  201. data/lib/karafka/runner.rb +3 -3
  202. data/lib/karafka/server.rb +14 -5
  203. data/lib/karafka/setup/attributes_map.rb +7 -7
  204. data/lib/karafka/setup/config.rb +11 -11
  205. data/lib/karafka/setup/contracts/config.rb +2 -2
  206. data/lib/karafka/setup/defaults_injector.rb +11 -11
  207. data/lib/karafka/swarm/manager.rb +6 -6
  208. data/lib/karafka/swarm/node.rb +8 -37
  209. data/lib/karafka/swarm/producer_replacer.rb +110 -0
  210. data/lib/karafka/swarm/supervisor.rb +9 -6
  211. data/lib/karafka/swarm.rb +1 -1
  212. data/lib/karafka/time_trackers/pause.rb +1 -1
  213. data/lib/karafka/version.rb +1 -1
  214. data/lib/karafka.rb +36 -36
  215. metadata +7 -3
@@ -47,7 +47,7 @@ module Karafka
47
47
  #
48
48
  # @param event [Karafka::Core::Monitoring::Event] event details including payload
49
49
  def on_worker_process(event)
50
- current_span = client.trace('karafka.consumer', service: service_name)
50
+ current_span = client.trace("karafka.consumer", service: service_name)
51
51
  push_tags
52
52
 
53
53
  job = event[:job]
@@ -69,7 +69,7 @@ module Karafka
69
69
 
70
70
  job = event[:job]
71
71
  time = event[:time]
72
- job_type = job.class.to_s.split('::').last
72
+ job_type = job.class.to_s.split("::").last
73
73
  consumer = job.executor.topic.consumer
74
74
  topic = job.executor.topic.name
75
75
 
@@ -90,55 +90,60 @@ module Karafka
90
90
  client.active_span&.set_error(error)
91
91
 
92
92
  case event[:type]
93
- when 'consumer.initialized.error'
93
+ when "consumer.initialized.error"
94
94
  error "Consumer initialized error: #{error}"
95
- when 'consumer.wrap.error'
95
+ when "consumer.wrap.error"
96
96
  error "Consumer wrap failed due to an error: #{error}"
97
- when 'consumer.consume.error'
97
+ when "consumer.consume.error"
98
98
  error "Consumer consuming error: #{error}"
99
- when 'consumer.revoked.error'
99
+ when "consumer.revoked.error"
100
100
  error "Consumer on revoked failed due to an error: #{error}"
101
- when 'consumer.idle.error'
101
+ when "consumer.idle.error"
102
102
  error "Consumer idle failed due to an error: #{error}"
103
- when 'consumer.shutdown.error'
103
+ when "consumer.shutdown.error"
104
104
  error "Consumer on shutdown failed due to an error: #{error}"
105
- when 'consumer.tick.error'
105
+ when "consumer.tick.error"
106
106
  error "Consumer on tick failed due to an error: #{error}"
107
- when 'consumer.eofed.error'
107
+ when "consumer.eofed.error"
108
108
  error "Consumer on eofed failed due to an error: #{error}"
109
- when 'consumer.after_consume.error'
109
+ when "consumer.after_consume.error"
110
110
  error "Consumer on after_consume failed due to an error: #{error}"
111
- when 'worker.process.error'
111
+ when "worker.process.error"
112
112
  fatal "Worker processing failed due to an error: #{error}"
113
- when 'connection.listener.fetch_loop.error'
113
+ when "connection.listener.fetch_loop.error"
114
114
  error "Listener fetch loop error: #{error}"
115
- when 'swarm.supervisor.error'
115
+ when "swarm.supervisor.error"
116
116
  fatal "Swarm supervisor crashed due to an error: #{error}"
117
- when 'runner.call.error'
117
+ when "runner.call.error"
118
118
  fatal "Runner crashed due to an error: #{error}"
119
- when 'app.stopping.error'
120
- error 'Forceful Karafka server stop'
121
- when 'app.forceful_stopping.error'
119
+ when "app.stopping.error"
120
+ active_listeners = event.payload[:active_listeners]
121
+ alive_workers = event.payload[:alive_workers]
122
+
123
+ error "Forceful Karafka server stop with: " \
124
+ "#{alive_workers.size} active workers and " \
125
+ "#{active_listeners.size} active listeners"
126
+ when "app.forceful_stopping.error"
122
127
  error "Forceful shutdown error occurred: #{error}"
123
- when 'librdkafka.error'
128
+ when "librdkafka.error"
124
129
  error "librdkafka internal error occurred: #{error}"
125
- when 'callbacks.statistics.error'
130
+ when "callbacks.statistics.error"
126
131
  error "callbacks.statistics processing failed due to an error: #{error}"
127
- when 'callbacks.error.error'
132
+ when "callbacks.error.error"
128
133
  error "callbacks.error processing failed due to an error: #{error}"
129
134
  # Those will only occur when retries in the client fail and when they did not stop
130
135
  # after back-offs
131
- when 'connection.client.poll.error'
136
+ when "connection.client.poll.error"
132
137
  error "Data polling error occurred: #{error}"
133
- when 'connection.client.rebalance_callback.error'
138
+ when "connection.client.rebalance_callback.error"
134
139
  error "Rebalance callback error occurred: #{error}"
135
- when 'connection.client.unsubscribe.error'
140
+ when "connection.client.unsubscribe.error"
136
141
  error "Client unsubscribe error occurred: #{error}"
137
- when 'parallel_segments.reducer.error'
142
+ when "parallel_segments.reducer.error"
138
143
  error "Parallel segments reducer error occurred: #{error}"
139
- when 'parallel_segments.partitioner.error'
144
+ when "parallel_segments.partitioner.error"
140
145
  error "Parallel segments partitioner error occurred: #{error}"
141
- when 'virtual_partitions.partitioner.error'
146
+ when "virtual_partitions.partitioner.error"
142
147
  error "Virtual partitions partitioner error occurred: #{error}"
143
148
  else
144
149
  error "#{event[:type]} error occurred: #{error}"
@@ -180,7 +185,7 @@ module Karafka
180
185
  # the assignment race condition is irrelevant here since the same value will be
181
186
  # assigned.
182
187
  def fetch_job_type(job_class)
183
- @job_types_cache[job_class] ||= job_class.to_s.split('::').last
188
+ @job_types_cache[job_class] ||= job_class.to_s.split("::").last
184
189
  end
185
190
  end
186
191
  end
@@ -22,7 +22,7 @@ module Karafka
22
22
  RdKafkaMetric = Struct.new(:type, :scope, :name, :key_location)
23
23
 
24
24
  # Namespace under which the DD metrics should be published
25
- setting :namespace, default: 'karafka'
25
+ setting :namespace, default: "karafka"
26
26
 
27
27
  # Datadog client that we should use to publish the metrics
28
28
  setting :client
@@ -37,22 +37,22 @@ module Karafka
37
37
  # Note, that the once with `_d` come from Karafka, not rdkafka or Kafka
38
38
  setting :rd_kafka_metrics, default: [
39
39
  # Client metrics
40
- RdKafkaMetric.new(:count, :root, 'messages.consumed', 'rxmsgs_d'),
41
- RdKafkaMetric.new(:count, :root, 'messages.consumed.bytes', 'rxmsg_bytes'),
40
+ RdKafkaMetric.new(:count, :root, "messages.consumed", "rxmsgs_d"),
41
+ RdKafkaMetric.new(:count, :root, "messages.consumed.bytes", "rxmsg_bytes"),
42
42
 
43
43
  # Broker metrics
44
- RdKafkaMetric.new(:count, :brokers, 'consume.attempts', 'txretries_d'),
45
- RdKafkaMetric.new(:count, :brokers, 'consume.errors', 'txerrs_d'),
46
- RdKafkaMetric.new(:count, :brokers, 'receive.errors', 'rxerrs_d'),
47
- RdKafkaMetric.new(:count, :brokers, 'connection.connects', 'connects_d'),
48
- RdKafkaMetric.new(:count, :brokers, 'connection.disconnects', 'disconnects_d'),
49
- RdKafkaMetric.new(:gauge, :brokers, 'network.latency.avg', %w[rtt avg]),
50
- RdKafkaMetric.new(:gauge, :brokers, 'network.latency.p95', %w[rtt p95]),
51
- RdKafkaMetric.new(:gauge, :brokers, 'network.latency.p99', %w[rtt p99]),
44
+ RdKafkaMetric.new(:count, :brokers, "consume.attempts", "txretries_d"),
45
+ RdKafkaMetric.new(:count, :brokers, "consume.errors", "txerrs_d"),
46
+ RdKafkaMetric.new(:count, :brokers, "receive.errors", "rxerrs_d"),
47
+ RdKafkaMetric.new(:count, :brokers, "connection.connects", "connects_d"),
48
+ RdKafkaMetric.new(:count, :brokers, "connection.disconnects", "disconnects_d"),
49
+ RdKafkaMetric.new(:gauge, :brokers, "network.latency.avg", %w[rtt avg]),
50
+ RdKafkaMetric.new(:gauge, :brokers, "network.latency.p95", %w[rtt p95]),
51
+ RdKafkaMetric.new(:gauge, :brokers, "network.latency.p99", %w[rtt p99]),
52
52
 
53
53
  # Topics metrics
54
- RdKafkaMetric.new(:gauge, :topics, 'consumer.lags', 'consumer_lag_stored'),
55
- RdKafkaMetric.new(:gauge, :topics, 'consumer.lags_delta', 'consumer_lag_stored_d')
54
+ RdKafkaMetric.new(:gauge, :topics, "consumer.lags", "consumer_lag_stored"),
55
+ RdKafkaMetric.new(:gauge, :topics, "consumer.lags_delta", "consumer_lag_stored_d")
56
56
  ].freeze
57
57
 
58
58
  # Whether histogram metrics should be sent as distributions or histograms.
@@ -101,7 +101,7 @@ module Karafka
101
101
  tags.concat(consumer_tags(event.payload[:caller]))
102
102
  end
103
103
 
104
- count('error_occurred', 1, tags: tags)
104
+ count("error_occurred", 1, tags: tags)
105
105
  end
106
106
 
107
107
  # Reports how many messages we've polled and how much time did we spend on it
@@ -116,8 +116,8 @@ module Karafka
116
116
  tags = ["consumer_group:#{consumer_group_id}"]
117
117
  tags.concat(default_tags)
118
118
 
119
- histogram('listener.polling.time_taken', time_taken, tags: tags)
120
- histogram('listener.polling.messages', messages_count, tags: tags)
119
+ histogram("listener.polling.time_taken", time_taken, tags: tags)
120
+ histogram("listener.polling.messages", messages_count, tags: tags)
121
121
  end
122
122
 
123
123
  # Here we report majority of things related to processing as we have access to the
@@ -131,13 +131,13 @@ module Karafka
131
131
  tags = consumer_tags(consumer)
132
132
  tags.concat(default_tags)
133
133
 
134
- count('consumer.messages', messages.size, tags: tags)
135
- count('consumer.batches', 1, tags: tags)
136
- gauge('consumer.offset', metadata.last_offset, tags: tags)
137
- histogram('consumer.consumed.time_taken', event[:time], tags: tags)
138
- histogram('consumer.batch_size', messages.size, tags: tags)
139
- histogram('consumer.processing_lag', metadata.processing_lag, tags: tags)
140
- histogram('consumer.consumption_lag', metadata.consumption_lag, tags: tags)
134
+ count("consumer.messages", messages.size, tags: tags)
135
+ count("consumer.batches", 1, tags: tags)
136
+ gauge("consumer.offset", metadata.last_offset, tags: tags)
137
+ histogram("consumer.consumed.time_taken", event[:time], tags: tags)
138
+ histogram("consumer.batch_size", messages.size, tags: tags)
139
+ histogram("consumer.processing_lag", metadata.processing_lag, tags: tags)
140
+ histogram("consumer.consumption_lag", metadata.consumption_lag, tags: tags)
141
141
  end
142
142
 
143
143
  {
@@ -164,9 +164,9 @@ module Karafka
164
164
  jq_stats = event[:jobs_queue].statistics
165
165
 
166
166
  tags = default_tags
167
- gauge('worker.total_threads', Karafka::App.config.concurrency, tags: tags)
168
- histogram('worker.processing', jq_stats[:busy], tags: tags)
169
- histogram('worker.enqueued_jobs', jq_stats[:enqueued], tags: tags)
167
+ gauge("worker.total_threads", Karafka::App.config.concurrency, tags: tags)
168
+ histogram("worker.processing", jq_stats[:busy], tags: tags)
169
+ histogram("worker.enqueued_jobs", jq_stats[:enqueued], tags: tags)
170
170
  end
171
171
 
172
172
  # We report this metric before and after processing for higher accuracy
@@ -175,7 +175,7 @@ module Karafka
175
175
  def on_worker_processed(event)
176
176
  jq_stats = event[:jobs_queue].statistics
177
177
 
178
- histogram('worker.processing', jq_stats[:busy], tags: default_tags)
178
+ histogram("worker.processing", jq_stats[:busy], tags: default_tags)
179
179
  end
180
180
 
181
181
  private
@@ -213,7 +213,7 @@ module Karafka
213
213
  else
214
214
  raise(
215
215
  ArgumentError,
216
- 'distribution_mode setting value must be either :histogram or :distribution'
216
+ "distribution_mode setting value must be either :histogram or :distribution"
217
217
  )
218
218
  end
219
219
  end
@@ -239,13 +239,13 @@ module Karafka
239
239
  tags: base_tags
240
240
  )
241
241
  when :brokers
242
- statistics.fetch('brokers').each_value do |broker_statistics|
242
+ statistics.fetch("brokers").each_value do |broker_statistics|
243
243
  # Skip bootstrap nodes
244
244
  # Bootstrap nodes have nodeid -1, other nodes have positive
245
245
  # node ids
246
- next if broker_statistics['nodeid'] == -1
246
+ next if broker_statistics["nodeid"] == -1
247
247
 
248
- tags = ["broker:#{broker_statistics['nodename']}"]
248
+ tags = ["broker:#{broker_statistics["nodename"]}"]
249
249
  tags.concat(base_tags)
250
250
 
251
251
  public_send(
@@ -256,16 +256,16 @@ module Karafka
256
256
  )
257
257
  end
258
258
  when :topics
259
- statistics.fetch('topics').each do |topic_name, topic_values|
260
- topic_values['partitions'].each do |partition_name, partition_statistics|
261
- next if partition_name == '-1'
259
+ statistics.fetch("topics").each do |topic_name, topic_values|
260
+ topic_values["partitions"].each do |partition_name, partition_statistics|
261
+ next if partition_name == "-1"
262
262
  # Skip until lag info is available
263
- next if partition_statistics['consumer_lag'] == -1
264
- next if partition_statistics['consumer_lag_stored'] == -1
263
+ next if partition_statistics["consumer_lag"] == -1
264
+ next if partition_statistics["consumer_lag_stored"] == -1
265
265
 
266
266
  # Skip if we do not own the fetch assignment
267
- next if partition_statistics['fetch_state'] == 'stopped'
268
- next if partition_statistics['fetch_state'] == 'none'
267
+ next if partition_statistics["fetch_state"] == "stopped"
268
+ next if partition_statistics["fetch_state"] == "none"
269
269
 
270
270
  tags = ["topic:#{topic_name}", "partition:#{partition_name}"]
271
271
  tags.concat(base_tags)
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'socket'
3
+ require "socket"
4
4
 
5
5
  module Karafka
6
6
  module Instrumentation
@@ -13,10 +13,10 @@ module Karafka
13
13
  include Karafka::Core::Helpers::Time
14
14
 
15
15
  # All good with Karafka
16
- OK_CODE = '200 OK'
16
+ OK_CODE = "200 OK"
17
17
 
18
18
  # Some timeouts, fail
19
- FAIL_CODE = '500 Internal Server Error'
19
+ FAIL_CODE = "500 Internal Server Error"
20
20
 
21
21
  private_constant :OK_CODE, :FAIL_CODE
22
22
 
@@ -32,7 +32,7 @@ module Karafka
32
32
 
33
33
  # @return [Boolean] true if all good, false if we should tell k8s to kill this process
34
34
  def healthy?
35
- raise NotImplementedError, 'Implement in a subclass'
35
+ raise NotImplementedError, "Implement in a subclass"
36
36
  end
37
37
 
38
38
  private
@@ -58,7 +58,7 @@ module Karafka
58
58
  # @return [Hash] hash that will be the response body
59
59
  def status_body
60
60
  {
61
- status: healthy? ? 'healthy' : 'unhealthy',
61
+ status: healthy? ? "healthy" : "unhealthy",
62
62
  timestamp: Time.now.to_i,
63
63
  port: @port,
64
64
  process_id: ::Process.pid
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'karafka/instrumentation/vendors/kubernetes/base_listener'
3
+ require "karafka/instrumentation/vendors/kubernetes/base_listener"
4
4
 
5
5
  module Karafka
6
6
  module Instrumentation
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'karafka/instrumentation/vendors/kubernetes/base_listener'
3
+ require "karafka/instrumentation/vendors/kubernetes/base_listener"
4
4
 
5
5
  module Karafka
6
6
  module Instrumentation
@@ -4,15 +4,25 @@ module Karafka
4
4
  # Checks the license presence for pro and loads pro components when needed (if any)
5
5
  class Licenser
6
6
  # Location in the gem where we store the public key
7
- PUBLIC_KEY_LOCATION = File.join(Karafka.gem_root, 'certs', 'karafka-pro.pem')
7
+ PUBLIC_KEY_LOCATION = File.join(Karafka.gem_root, "certs", "karafka-pro.pem")
8
+
9
+ # Location of file containing expired/revoked license checksums
10
+ EXPIRED_CHECKSUMS_LOCATION = File.join(Karafka.gem_root, "certs", "expired.txt")
8
11
 
9
12
  private_constant :PUBLIC_KEY_LOCATION
10
13
 
11
14
  class << self
12
15
  # Tries to load the license and yields if successful
13
16
  def detect
14
- # If required, do not require again
15
- require('karafka-license') unless const_defined?('::Karafka::License')
17
+ # If license module is already fully defined, don't touch it
18
+ # This allows users to define their own License module for testing/custom setups
19
+ unless license_fully_defined?
20
+ # Try safe approach first (no code execution)
21
+ loaded = safe_load_license || fallback_require_license
22
+
23
+ # If neither method succeeded, return false
24
+ return false unless loaded
25
+ end
16
26
 
17
27
  yield
18
28
 
@@ -26,7 +36,7 @@ module Karafka
26
36
  # @param license_config [Karafka::Core::Configurable::Node] config related to the licensing
27
37
  def prepare_and_verify(license_config)
28
38
  # If license is not loaded, nothing to do
29
- return unless const_defined?('::Karafka::License')
39
+ return unless const_defined?("::Karafka::License")
30
40
 
31
41
  prepare(license_config)
32
42
  verify(license_config)
@@ -34,6 +44,65 @@ module Karafka
34
44
 
35
45
  private
36
46
 
47
+ # Check if License module and required methods are already defined
48
+ # @return [Boolean]
49
+ def license_fully_defined?
50
+ return false unless const_defined?("::Karafka::License")
51
+
52
+ # Check if the required methods exist
53
+ ::Karafka::License.respond_to?(:token) &&
54
+ ::Karafka::License.respond_to?(:version)
55
+ end
56
+
57
+ # Attempt to safely load license without executing gem code
58
+ # @return [Boolean] true if successful, false otherwise
59
+ def safe_load_license
60
+ # First verify the gem exists and files are accessible
61
+ spec = Gem::Specification.find_by_name("karafka-license")
62
+ license_path = File.join(spec.gem_dir, "lib", "license.txt")
63
+ version_path = File.join(spec.gem_dir, "lib", "version.txt")
64
+
65
+ return false unless File.exist?(license_path)
66
+
67
+ # Read license data to ensure we can successfully load before removing any constants
68
+ license_token = File.read(license_path)
69
+ version_content = File.read(version_path)
70
+
71
+ # Only after confirming we have valid data, remove and replace the License constant
72
+ # This ensures we always use the actual license from the gem, overwriting any stale
73
+ # or incomplete License module that may have been defined elsewhere
74
+ ::Karafka.send(:remove_const, :License) if const_defined?("::Karafka::License")
75
+
76
+ ::Karafka.const_set(:License, Module.new)
77
+ ::Karafka::License.define_singleton_method(:token) { license_token }
78
+ ::Karafka::License.define_singleton_method(:version) { version_content }
79
+
80
+ true
81
+ rescue Gem::MissingSpecError, Errno::ENOENT
82
+ # Gem not found or files don't exist - don't touch existing License constant
83
+ false
84
+ end
85
+
86
+ # Fallback to traditional require if safe method fails
87
+ # @return [Boolean] true if successful, false otherwise
88
+ def fallback_require_license
89
+ # First check if we can actually load the gem before removing any constants
90
+ # Try to find the gem spec to see if it exists
91
+ begin
92
+ Gem::Specification.find_by_name("karafka-license")
93
+ rescue Gem::MissingSpecError
94
+ return false
95
+ end
96
+
97
+ # Gem exists, so we can safely remove and reload the License constant
98
+ ::Karafka.send(:remove_const, :License) if const_defined?("::Karafka::License")
99
+
100
+ require("karafka-license")
101
+ true
102
+ rescue LoadError
103
+ false
104
+ end
105
+
37
106
  # @param license_config [Karafka::Core::Configurable::Node] config related to the licensing
38
107
  def prepare(license_config)
39
108
  license_config.token = Karafka::License.token
@@ -45,8 +114,12 @@ module Karafka
45
114
  public_key = OpenSSL::PKey::RSA.new(File.read(PUBLIC_KEY_LOCATION))
46
115
 
47
116
  # We gsub and strip in case someone copy-pasted it as a multi line string
48
- formatted_token = license_config.token.strip.delete("\n").delete(' ')
49
- decoded_token = formatted_token.unpack1('m') # decode from base64
117
+ formatted_token = license_config.token.strip.delete("\n").delete(" ")
118
+
119
+ # Check if the license has been revoked/expired before attempting decryption
120
+ raise_expired_license_token(license_config) if expired?(formatted_token)
121
+
122
+ decoded_token = formatted_token.unpack1("m") # decode from base64
50
123
 
51
124
  begin
52
125
  data = public_key.public_decrypt(decoded_token)
@@ -56,7 +129,26 @@ module Karafka
56
129
 
57
130
  details = data ? JSON.parse(data) : raise_invalid_license_token(license_config)
58
131
 
59
- license_config.entity = details.fetch('entity')
132
+ license_config.entity = details.fetch("entity")
133
+ end
134
+
135
+ # Checks if the license token has been expired/revoked
136
+ # @param formatted_token [String] formatted license token
137
+ # @return [Boolean] true if the license is in the expired list
138
+ def expired?(formatted_token)
139
+ token_checksum = Digest::SHA256.hexdigest(formatted_token)
140
+
141
+ expired_checksums.include?(token_checksum)
142
+ end
143
+
144
+ # Loads the list of expired license checksums
145
+ # @return [Set<String>] set of expired license checksums
146
+ def expired_checksums
147
+ File
148
+ .readlines(EXPIRED_CHECKSUMS_LOCATION)
149
+ .map(&:strip)
150
+ .reject { |line| line.empty? || line.start_with?("#") }
151
+ .to_set
60
152
  end
61
153
 
62
154
  # Raises an error with info, that used token is invalid
@@ -67,12 +159,27 @@ module Karafka
67
159
 
68
160
  raise(
69
161
  Errors::InvalidLicenseTokenError,
70
- <<~MSG.tr("\n", ' ')
162
+ <<~MSG.tr("\n", " ")
71
163
  License key you provided is invalid.
72
164
  Please reach us at contact@karafka.io or visit https://karafka.io to obtain a valid one.
73
165
  MSG
74
166
  )
75
167
  end
168
+
169
+ # Raises an error with info, that used token has expired or been revoked
170
+ # @param license_config [Karafka::Core::Configurable::Node]
171
+ def raise_expired_license_token(license_config)
172
+ # We set it to false so `Karafka.pro?` method behaves as expected
173
+ license_config.token = false
174
+
175
+ raise(
176
+ Errors::ExpiredLicenseTokenError,
177
+ <<~MSG.tr("\n", " ")
178
+ License key you provided has expired or been revoked.
179
+ Please reach us at contact@karafka.io or visit https://karafka.io to obtain a valid one.
180
+ MSG
181
+ )
182
+ end
76
183
  end
77
184
  end
78
185
  end
@@ -17,16 +17,18 @@ module Karafka
17
17
  # @note We do not set `processed_at` as this needs to be assigned when the batch is
18
18
  # picked up for processing.
19
19
  def call(messages, topic, partition, scheduled_at)
20
+ last_message = messages.last
21
+
20
22
  Karafka::Messages::BatchMetadata.new(
21
23
  size: messages.size,
22
24
  first_offset: messages.first&.offset || -1001,
23
- last_offset: messages.last&.offset || -1001,
25
+ last_offset: last_message&.offset || -1001,
24
26
  deserializers: topic.deserializers,
25
27
  partition: partition,
26
28
  topic: topic.name,
27
29
  # We go with the assumption that the creation of the whole batch is the last message
28
30
  # creation time
29
- created_at: local_created_at(messages.last),
31
+ created_at: local_created_at(last_message),
30
32
  # When this batch was built and scheduled for execution
31
33
  scheduled_at: scheduled_at,
32
34
  # This needs to be set to a correct value prior to processing starting
@@ -96,7 +96,7 @@ module Karafka
96
96
  @messages_array
97
97
  end
98
98
 
99
- alias count size
99
+ alias_method :count, :size
100
100
  end
101
101
  end
102
102
  end
@@ -80,7 +80,7 @@ module Karafka
80
80
  tpl = ::Rdkafka::Consumer::TopicPartitionList.from_native_tpl(partitions_ptr).freeze
81
81
  opaque = ::Rdkafka::Config.opaques[opaque_ptr.to_i]
82
82
 
83
- if RB.rd_kafka_rebalance_protocol(client_ptr) == 'COOPERATIVE'
83
+ if RB.rd_kafka_rebalance_protocol(client_ptr) == "COOPERATIVE"
84
84
  pr.on_cooperative_rebalance(client_ptr, code, partitions_ptr, tpl, opaque)
85
85
  else
86
86
  pr.on_eager_rebalance(client_ptr, code, partitions_ptr, tpl, opaque)
@@ -95,7 +95,7 @@ end
95
95
  # At the moment there is no API in rdkafka-ruby to do so
96
96
  Rdkafka::Bindings.send(
97
97
  :remove_const,
98
- 'RebalanceCallback'
98
+ "RebalanceCallback"
99
99
  )
100
100
 
101
101
  Rdkafka::Bindings.const_set(
@@ -28,8 +28,8 @@ module Karafka
28
28
  class JobOptionsContract < Contracts::Base
29
29
  configure do |config|
30
30
  config.error_messages = YAML.safe_load_file(
31
- File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
32
- ).fetch('en').fetch('validations').fetch('job_options')
31
+ File.join(Karafka.gem_root, "config", "locales", "errors.yml")
32
+ ).fetch("en").fetch("validations").fetch("job_options")
33
33
  end
34
34
 
35
35
  optional(:producer) { |val| val.nil? || val.respond_to?(:call) }
@@ -53,6 +53,16 @@ module Karafka
53
53
  super(&)
54
54
  end
55
55
  end
56
+
57
+ # Cleans all messages in the batch
58
+ #
59
+ # @param metadata [Boolean] should we also clean metadata alongside the payload for
60
+ # each message. `true` by default.
61
+ #
62
+ # @note This is a convenience method that calls `clean!` on each message in the batch.
63
+ def clean!(metadata: true)
64
+ each { |message| message.clean!(metadata: metadata) }
65
+ end
56
66
  end
57
67
  end
58
68
  end
@@ -30,8 +30,8 @@ module Karafka
30
30
  class Server < Karafka::Cli::Contracts::Server
31
31
  configure do |config|
32
32
  config.error_messages = YAML.safe_load_file(
33
- File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
34
- ).fetch('en').fetch('validations').fetch('cli').fetch('server')
33
+ File.join(Karafka.gem_root, "config", "locales", "errors.yml")
34
+ ).fetch("en").fetch("validations").fetch("cli").fetch("server")
35
35
  end
36
36
 
37
37
  %i[
@@ -66,10 +66,10 @@ module Karafka
66
66
  next if value.empty?
67
67
 
68
68
  subscription_groups = Karafka::App
69
- .consumer_groups
70
- .map(&:subscription_groups)
71
- .flatten
72
- .map(&:name)
69
+ .consumer_groups
70
+ .map(&:subscription_groups)
71
+ .flatten
72
+ .map(&:name)
73
73
 
74
74
  next if (value - subscription_groups).empty?
75
75
 
@@ -87,12 +87,12 @@ module Karafka
87
87
  next if value.empty?
88
88
 
89
89
  topics = Karafka::App
90
- .consumer_groups
91
- .map(&:subscription_groups)
92
- .flatten
93
- .map(&:topics)
94
- .map { |gtopics| gtopics.map(&:name) }
95
- .flatten
90
+ .consumer_groups
91
+ .map(&:subscription_groups)
92
+ .flatten
93
+ .map(&:topics)
94
+ .map { |gtopics| gtopics.map(&:name) }
95
+ .flatten
96
96
 
97
97
  next if (value - topics).empty?
98
98
 
@@ -47,9 +47,9 @@ module Karafka
47
47
  requested_groups = options[:groups].dup || []
48
48
 
49
49
  workable_groups = Karafka::App
50
- .routes
51
- .select(&:parallel_segments?)
52
- .group_by(&:segment_origin)
50
+ .routes
51
+ .select(&:parallel_segments?)
52
+ .group_by(&:segment_origin)
53
53
 
54
54
  # Use all if none provided
55
55
  return workable_groups if requested_groups.empty?
@@ -85,7 +85,7 @@ module Karafka
85
85
  consumer_groups = [segment_origin, segments.map(&:name)].flatten
86
86
 
87
87
  consumer_groups_with_topics = consumer_groups
88
- .to_h { |name| [name, topics_names] }
88
+ .to_h { |name| [name, topics_names] }
89
89
 
90
90
  lags_with_offsets = Karafka::Admin.read_lags_with_offsets(
91
91
  consumer_groups_with_topics