karafka 2.5.5 → 2.5.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +20 -0
- data/LICENSE-COMM +4 -0
- data/README.md +2 -2
- data/certs/expired.txt +2 -0
- data/karafka.gemspec +23 -23
- data/lib/active_job/karafka.rb +2 -2
- data/lib/active_job/queue_adapters/karafka_adapter.rb +5 -5
- data/lib/karafka/active_job/consumer.rb +3 -3
- data/lib/karafka/active_job/current_attributes.rb +4 -4
- data/lib/karafka/active_job/job_options_contract.rb +2 -2
- data/lib/karafka/admin/acl.rb +3 -3
- data/lib/karafka/admin/configs/resource.rb +1 -1
- data/lib/karafka/admin/configs.rb +1 -1
- data/lib/karafka/admin/consumer_groups.rb +8 -8
- data/lib/karafka/admin/contracts/replication.rb +2 -2
- data/lib/karafka/admin/replication.rb +21 -21
- data/lib/karafka/admin/topics.rb +6 -6
- data/lib/karafka/admin.rb +4 -5
- data/lib/karafka/app.rb +3 -3
- data/lib/karafka/base_consumer.rb +34 -30
- data/lib/karafka/cli/base.rb +8 -8
- data/lib/karafka/cli/console.rb +1 -1
- data/lib/karafka/cli/contracts/server.rb +12 -12
- data/lib/karafka/cli/help.rb +2 -2
- data/lib/karafka/cli/info.rb +4 -4
- data/lib/karafka/cli/install.rb +11 -11
- data/lib/karafka/cli/server.rb +6 -6
- data/lib/karafka/cli/swarm.rb +1 -1
- data/lib/karafka/cli/topics/align.rb +4 -4
- data/lib/karafka/cli/topics/base.rb +5 -5
- data/lib/karafka/cli/topics/create.rb +2 -2
- data/lib/karafka/cli/topics/delete.rb +2 -2
- data/lib/karafka/cli/topics/help.rb +5 -1
- data/lib/karafka/cli/topics/plan.rb +16 -16
- data/lib/karafka/cli/topics/repartition.rb +3 -3
- data/lib/karafka/cli/topics.rb +22 -22
- data/lib/karafka/cli.rb +2 -2
- data/lib/karafka/connection/client.rb +17 -17
- data/lib/karafka/connection/listener.rb +6 -6
- data/lib/karafka/connection/mode.rb +1 -1
- data/lib/karafka/connection/proxy.rb +1 -1
- data/lib/karafka/connection/status.rb +2 -2
- data/lib/karafka/constraints.rb +3 -3
- data/lib/karafka/embedded.rb +3 -3
- data/lib/karafka/env.rb +4 -4
- data/lib/karafka/errors.rb +6 -1
- data/lib/karafka/execution_mode.rb +1 -1
- data/lib/karafka/helpers/config_importer.rb +2 -2
- data/lib/karafka/helpers/interval_runner.rb +4 -2
- data/lib/karafka/helpers/multi_delegator.rb +1 -1
- data/lib/karafka/instrumentation/assignments_tracker.rb +9 -9
- data/lib/karafka/instrumentation/callbacks/error.rb +5 -5
- data/lib/karafka/instrumentation/callbacks/oauthbearer_token_refresh.rb +4 -4
- data/lib/karafka/instrumentation/callbacks/rebalance.rb +6 -6
- data/lib/karafka/instrumentation/callbacks/statistics.rb +5 -5
- data/lib/karafka/instrumentation/logger.rb +7 -7
- data/lib/karafka/instrumentation/logger_listener.rb +76 -63
- data/lib/karafka/instrumentation/vendors/appsignal/base.rb +1 -1
- data/lib/karafka/instrumentation/vendors/appsignal/client.rb +1 -1
- data/lib/karafka/instrumentation/vendors/appsignal/errors_listener.rb +1 -1
- data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb +36 -36
- data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +33 -28
- data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +38 -38
- data/lib/karafka/instrumentation/vendors/kubernetes/base_listener.rb +5 -5
- data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +1 -1
- data/lib/karafka/instrumentation/vendors/kubernetes/swarm_liveness_listener.rb +1 -1
- data/lib/karafka/licenser.rb +115 -8
- data/lib/karafka/messages/builders/batch_metadata.rb +4 -2
- data/lib/karafka/messages/messages.rb +1 -1
- data/lib/karafka/patches/rdkafka/bindings.rb +2 -2
- data/lib/karafka/pro/active_job/job_options_contract.rb +2 -2
- data/lib/karafka/pro/cleaner/messages/messages.rb +10 -0
- data/lib/karafka/pro/cli/contracts/server.rb +12 -12
- data/lib/karafka/pro/cli/parallel_segments/base.rb +4 -4
- data/lib/karafka/pro/cli/parallel_segments/collapse.rb +5 -5
- data/lib/karafka/pro/cli/parallel_segments/distribute.rb +3 -3
- data/lib/karafka/pro/cli/parallel_segments.rb +7 -7
- data/lib/karafka/pro/cli/topics/health.rb +162 -0
- data/lib/karafka/pro/cli/topics.rb +52 -0
- data/lib/karafka/pro/connection/manager.rb +14 -14
- data/lib/karafka/pro/encryption/contracts/config.rb +2 -2
- data/lib/karafka/pro/encryption/messages/middleware.rb +2 -2
- data/lib/karafka/pro/encryption/messages/parser.rb +2 -2
- data/lib/karafka/pro/encryption/setup/config.rb +2 -2
- data/lib/karafka/pro/iterator/tpl_builder.rb +2 -2
- data/lib/karafka/pro/iterator.rb +1 -1
- data/lib/karafka/pro/loader.rb +2 -1
- data/lib/karafka/pro/processing/adaptive_iterator/consumer.rb +1 -1
- data/lib/karafka/pro/processing/coordinators/virtual_offset_manager.rb +24 -14
- data/lib/karafka/pro/processing/filters/base.rb +1 -1
- data/lib/karafka/pro/processing/filters/delayer.rb +2 -2
- data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb +1 -1
- data/lib/karafka/pro/processing/offset_metadata/consumer.rb +1 -1
- data/lib/karafka/pro/processing/parallel_segments/filters/base.rb +6 -6
- data/lib/karafka/pro/processing/partitioner.rb +3 -3
- data/lib/karafka/pro/processing/periodic_job/consumer.rb +6 -5
- data/lib/karafka/pro/processing/piping/consumer.rb +7 -7
- data/lib/karafka/pro/processing/schedulers/base.rb +5 -5
- data/lib/karafka/pro/processing/schedulers/default.rb +5 -5
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb +6 -3
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb +6 -3
- data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom_vp.rb +6 -3
- data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb +2 -2
- data/lib/karafka/pro/processing/strategies/default.rb +22 -22
- data/lib/karafka/pro/processing/strategies/dlq/default.rb +7 -7
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb +6 -3
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb +6 -3
- data/lib/karafka/pro/processing/strategies/ftr/default.rb +2 -2
- data/lib/karafka/pro/processing/strategies/lrj/default.rb +2 -2
- data/lib/karafka/pro/processing/strategies/lrj/ftr.rb +6 -3
- data/lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb +6 -3
- data/lib/karafka/pro/processing/strategies/lrj/mom.rb +2 -2
- data/lib/karafka/pro/recurring_tasks/consumer.rb +2 -2
- data/lib/karafka/pro/recurring_tasks/contracts/config.rb +2 -2
- data/lib/karafka/pro/recurring_tasks/contracts/task.rb +2 -2
- data/lib/karafka/pro/recurring_tasks/dispatcher.rb +2 -2
- data/lib/karafka/pro/recurring_tasks/listener.rb +1 -1
- data/lib/karafka/pro/recurring_tasks/matcher.rb +2 -2
- data/lib/karafka/pro/recurring_tasks/serializer.rb +5 -5
- data/lib/karafka/pro/recurring_tasks/setup/config.rb +3 -3
- data/lib/karafka/pro/recurring_tasks/task.rb +4 -4
- data/lib/karafka/pro/recurring_tasks.rb +4 -4
- data/lib/karafka/pro/routing/features/adaptive_iterator/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb +1 -1
- data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/direct_assignments/contracts/consumer_group.rb +2 -2
- data/lib/karafka/pro/routing/features/direct_assignments/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/direct_assignments/topic.rb +1 -1
- data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/inline_insights/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/long_running_job/topic.rb +1 -1
- data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/multiplexing.rb +5 -5
- data/lib/karafka/pro/routing/features/non_blocking_job/topic.rb +1 -1
- data/lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/offset_metadata/topic.rb +1 -1
- data/lib/karafka/pro/routing/features/offset_metadata.rb +1 -1
- data/lib/karafka/pro/routing/features/parallel_segments/consumer_group.rb +5 -5
- data/lib/karafka/pro/routing/features/parallel_segments/contracts/consumer_group.rb +2 -2
- data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb +2 -2
- data/lib/karafka/pro/routing/features/patterns/contracts/pattern.rb +3 -3
- data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/patterns/topic.rb +1 -1
- data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/periodic_job/topic.rb +1 -1
- data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb +7 -7
- data/lib/karafka/pro/routing/features/recurring_tasks/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/scheduled_messages/builder.rb +13 -13
- data/lib/karafka/pro/routing/features/scheduled_messages/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/swarm/contracts/routing.rb +2 -2
- data/lib/karafka/pro/routing/features/swarm/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/swarm.rb +1 -1
- data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/virtual_partitions/config.rb +7 -7
- data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +2 -2
- data/lib/karafka/pro/scheduled_messages/consumer.rb +4 -4
- data/lib/karafka/pro/scheduled_messages/contracts/config.rb +2 -2
- data/lib/karafka/pro/scheduled_messages/contracts/message.rb +10 -10
- data/lib/karafka/pro/scheduled_messages/daily_buffer.rb +2 -2
- data/lib/karafka/pro/scheduled_messages/deserializers/headers.rb +4 -4
- data/lib/karafka/pro/scheduled_messages/dispatcher.rb +5 -5
- data/lib/karafka/pro/scheduled_messages/proxy.rb +8 -8
- data/lib/karafka/pro/scheduled_messages/schema_validator.rb +1 -1
- data/lib/karafka/pro/scheduled_messages/setup/config.rb +2 -2
- data/lib/karafka/pro/scheduled_messages/state.rb +1 -1
- data/lib/karafka/pro/scheduled_messages/tracker.rb +2 -2
- data/lib/karafka/pro/scheduled_messages.rb +2 -2
- data/lib/karafka/pro/swarm/liveness_listener.rb +2 -2
- data/lib/karafka/process.rb +1 -1
- data/lib/karafka/processing/coordinator.rb +1 -1
- data/lib/karafka/processing/inline_insights/consumer.rb +4 -4
- data/lib/karafka/processing/inline_insights/tracker.rb +6 -6
- data/lib/karafka/processing/jobs/base.rb +6 -4
- data/lib/karafka/processing/jobs_queue.rb +10 -0
- data/lib/karafka/processing/schedulers/default.rb +4 -4
- data/lib/karafka/processing/strategies/base.rb +6 -6
- data/lib/karafka/processing/strategies/default.rb +13 -13
- data/lib/karafka/processing/strategies/dlq.rb +1 -1
- data/lib/karafka/processing/worker.rb +5 -5
- data/lib/karafka/railtie.rb +11 -11
- data/lib/karafka/routing/builder.rb +3 -3
- data/lib/karafka/routing/contracts/consumer_group.rb +6 -6
- data/lib/karafka/routing/contracts/routing.rb +2 -2
- data/lib/karafka/routing/contracts/topic.rb +4 -4
- data/lib/karafka/routing/features/active_job/contracts/topic.rb +3 -3
- data/lib/karafka/routing/features/base/expander.rb +4 -4
- data/lib/karafka/routing/features/base.rb +8 -8
- data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +2 -2
- data/lib/karafka/routing/features/declaratives/contracts/topic.rb +2 -2
- data/lib/karafka/routing/features/deserializers/contracts/topic.rb +2 -2
- data/lib/karafka/routing/features/eofed/contracts/topic.rb +3 -3
- data/lib/karafka/routing/features/inline_insights/contracts/topic.rb +2 -2
- data/lib/karafka/routing/features/inline_insights.rb +7 -7
- data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb +2 -2
- data/lib/karafka/routing/subscription_group.rb +9 -9
- data/lib/karafka/runner.rb +3 -3
- data/lib/karafka/server.rb +14 -5
- data/lib/karafka/setup/attributes_map.rb +7 -7
- data/lib/karafka/setup/config.rb +11 -11
- data/lib/karafka/setup/contracts/config.rb +2 -2
- data/lib/karafka/setup/defaults_injector.rb +11 -11
- data/lib/karafka/swarm/manager.rb +6 -6
- data/lib/karafka/swarm/node.rb +8 -37
- data/lib/karafka/swarm/producer_replacer.rb +110 -0
- data/lib/karafka/swarm/supervisor.rb +9 -6
- data/lib/karafka/swarm.rb +1 -1
- data/lib/karafka/time_trackers/pause.rb +1 -1
- data/lib/karafka/version.rb +1 -1
- data/lib/karafka.rb +36 -36
- metadata +7 -3
|
@@ -47,7 +47,7 @@ module Karafka
|
|
|
47
47
|
#
|
|
48
48
|
# @param event [Karafka::Core::Monitoring::Event] event details including payload
|
|
49
49
|
def on_worker_process(event)
|
|
50
|
-
current_span = client.trace(
|
|
50
|
+
current_span = client.trace("karafka.consumer", service: service_name)
|
|
51
51
|
push_tags
|
|
52
52
|
|
|
53
53
|
job = event[:job]
|
|
@@ -69,7 +69,7 @@ module Karafka
|
|
|
69
69
|
|
|
70
70
|
job = event[:job]
|
|
71
71
|
time = event[:time]
|
|
72
|
-
job_type = job.class.to_s.split(
|
|
72
|
+
job_type = job.class.to_s.split("::").last
|
|
73
73
|
consumer = job.executor.topic.consumer
|
|
74
74
|
topic = job.executor.topic.name
|
|
75
75
|
|
|
@@ -90,55 +90,60 @@ module Karafka
|
|
|
90
90
|
client.active_span&.set_error(error)
|
|
91
91
|
|
|
92
92
|
case event[:type]
|
|
93
|
-
when
|
|
93
|
+
when "consumer.initialized.error"
|
|
94
94
|
error "Consumer initialized error: #{error}"
|
|
95
|
-
when
|
|
95
|
+
when "consumer.wrap.error"
|
|
96
96
|
error "Consumer wrap failed due to an error: #{error}"
|
|
97
|
-
when
|
|
97
|
+
when "consumer.consume.error"
|
|
98
98
|
error "Consumer consuming error: #{error}"
|
|
99
|
-
when
|
|
99
|
+
when "consumer.revoked.error"
|
|
100
100
|
error "Consumer on revoked failed due to an error: #{error}"
|
|
101
|
-
when
|
|
101
|
+
when "consumer.idle.error"
|
|
102
102
|
error "Consumer idle failed due to an error: #{error}"
|
|
103
|
-
when
|
|
103
|
+
when "consumer.shutdown.error"
|
|
104
104
|
error "Consumer on shutdown failed due to an error: #{error}"
|
|
105
|
-
when
|
|
105
|
+
when "consumer.tick.error"
|
|
106
106
|
error "Consumer on tick failed due to an error: #{error}"
|
|
107
|
-
when
|
|
107
|
+
when "consumer.eofed.error"
|
|
108
108
|
error "Consumer on eofed failed due to an error: #{error}"
|
|
109
|
-
when
|
|
109
|
+
when "consumer.after_consume.error"
|
|
110
110
|
error "Consumer on after_consume failed due to an error: #{error}"
|
|
111
|
-
when
|
|
111
|
+
when "worker.process.error"
|
|
112
112
|
fatal "Worker processing failed due to an error: #{error}"
|
|
113
|
-
when
|
|
113
|
+
when "connection.listener.fetch_loop.error"
|
|
114
114
|
error "Listener fetch loop error: #{error}"
|
|
115
|
-
when
|
|
115
|
+
when "swarm.supervisor.error"
|
|
116
116
|
fatal "Swarm supervisor crashed due to an error: #{error}"
|
|
117
|
-
when
|
|
117
|
+
when "runner.call.error"
|
|
118
118
|
fatal "Runner crashed due to an error: #{error}"
|
|
119
|
-
when
|
|
120
|
-
|
|
121
|
-
|
|
119
|
+
when "app.stopping.error"
|
|
120
|
+
active_listeners = event.payload[:active_listeners]
|
|
121
|
+
alive_workers = event.payload[:alive_workers]
|
|
122
|
+
|
|
123
|
+
error "Forceful Karafka server stop with: " \
|
|
124
|
+
"#{alive_workers.size} active workers and " \
|
|
125
|
+
"#{active_listeners.size} active listeners"
|
|
126
|
+
when "app.forceful_stopping.error"
|
|
122
127
|
error "Forceful shutdown error occurred: #{error}"
|
|
123
|
-
when
|
|
128
|
+
when "librdkafka.error"
|
|
124
129
|
error "librdkafka internal error occurred: #{error}"
|
|
125
|
-
when
|
|
130
|
+
when "callbacks.statistics.error"
|
|
126
131
|
error "callbacks.statistics processing failed due to an error: #{error}"
|
|
127
|
-
when
|
|
132
|
+
when "callbacks.error.error"
|
|
128
133
|
error "callbacks.error processing failed due to an error: #{error}"
|
|
129
134
|
# Those will only occur when retries in the client fail and when they did not stop
|
|
130
135
|
# after back-offs
|
|
131
|
-
when
|
|
136
|
+
when "connection.client.poll.error"
|
|
132
137
|
error "Data polling error occurred: #{error}"
|
|
133
|
-
when
|
|
138
|
+
when "connection.client.rebalance_callback.error"
|
|
134
139
|
error "Rebalance callback error occurred: #{error}"
|
|
135
|
-
when
|
|
140
|
+
when "connection.client.unsubscribe.error"
|
|
136
141
|
error "Client unsubscribe error occurred: #{error}"
|
|
137
|
-
when
|
|
142
|
+
when "parallel_segments.reducer.error"
|
|
138
143
|
error "Parallel segments reducer error occurred: #{error}"
|
|
139
|
-
when
|
|
144
|
+
when "parallel_segments.partitioner.error"
|
|
140
145
|
error "Parallel segments partitioner error occurred: #{error}"
|
|
141
|
-
when
|
|
146
|
+
when "virtual_partitions.partitioner.error"
|
|
142
147
|
error "Virtual partitions partitioner error occurred: #{error}"
|
|
143
148
|
else
|
|
144
149
|
error "#{event[:type]} error occurred: #{error}"
|
|
@@ -180,7 +185,7 @@ module Karafka
|
|
|
180
185
|
# the assignment race condition is irrelevant here since the same value will be
|
|
181
186
|
# assigned.
|
|
182
187
|
def fetch_job_type(job_class)
|
|
183
|
-
@job_types_cache[job_class] ||= job_class.to_s.split(
|
|
188
|
+
@job_types_cache[job_class] ||= job_class.to_s.split("::").last
|
|
184
189
|
end
|
|
185
190
|
end
|
|
186
191
|
end
|
|
@@ -22,7 +22,7 @@ module Karafka
|
|
|
22
22
|
RdKafkaMetric = Struct.new(:type, :scope, :name, :key_location)
|
|
23
23
|
|
|
24
24
|
# Namespace under which the DD metrics should be published
|
|
25
|
-
setting :namespace, default:
|
|
25
|
+
setting :namespace, default: "karafka"
|
|
26
26
|
|
|
27
27
|
# Datadog client that we should use to publish the metrics
|
|
28
28
|
setting :client
|
|
@@ -37,22 +37,22 @@ module Karafka
|
|
|
37
37
|
# Note, that the once with `_d` come from Karafka, not rdkafka or Kafka
|
|
38
38
|
setting :rd_kafka_metrics, default: [
|
|
39
39
|
# Client metrics
|
|
40
|
-
RdKafkaMetric.new(:count, :root,
|
|
41
|
-
RdKafkaMetric.new(:count, :root,
|
|
40
|
+
RdKafkaMetric.new(:count, :root, "messages.consumed", "rxmsgs_d"),
|
|
41
|
+
RdKafkaMetric.new(:count, :root, "messages.consumed.bytes", "rxmsg_bytes"),
|
|
42
42
|
|
|
43
43
|
# Broker metrics
|
|
44
|
-
RdKafkaMetric.new(:count, :brokers,
|
|
45
|
-
RdKafkaMetric.new(:count, :brokers,
|
|
46
|
-
RdKafkaMetric.new(:count, :brokers,
|
|
47
|
-
RdKafkaMetric.new(:count, :brokers,
|
|
48
|
-
RdKafkaMetric.new(:count, :brokers,
|
|
49
|
-
RdKafkaMetric.new(:gauge, :brokers,
|
|
50
|
-
RdKafkaMetric.new(:gauge, :brokers,
|
|
51
|
-
RdKafkaMetric.new(:gauge, :brokers,
|
|
44
|
+
RdKafkaMetric.new(:count, :brokers, "consume.attempts", "txretries_d"),
|
|
45
|
+
RdKafkaMetric.new(:count, :brokers, "consume.errors", "txerrs_d"),
|
|
46
|
+
RdKafkaMetric.new(:count, :brokers, "receive.errors", "rxerrs_d"),
|
|
47
|
+
RdKafkaMetric.new(:count, :brokers, "connection.connects", "connects_d"),
|
|
48
|
+
RdKafkaMetric.new(:count, :brokers, "connection.disconnects", "disconnects_d"),
|
|
49
|
+
RdKafkaMetric.new(:gauge, :brokers, "network.latency.avg", %w[rtt avg]),
|
|
50
|
+
RdKafkaMetric.new(:gauge, :brokers, "network.latency.p95", %w[rtt p95]),
|
|
51
|
+
RdKafkaMetric.new(:gauge, :brokers, "network.latency.p99", %w[rtt p99]),
|
|
52
52
|
|
|
53
53
|
# Topics metrics
|
|
54
|
-
RdKafkaMetric.new(:gauge, :topics,
|
|
55
|
-
RdKafkaMetric.new(:gauge, :topics,
|
|
54
|
+
RdKafkaMetric.new(:gauge, :topics, "consumer.lags", "consumer_lag_stored"),
|
|
55
|
+
RdKafkaMetric.new(:gauge, :topics, "consumer.lags_delta", "consumer_lag_stored_d")
|
|
56
56
|
].freeze
|
|
57
57
|
|
|
58
58
|
# Whether histogram metrics should be sent as distributions or histograms.
|
|
@@ -101,7 +101,7 @@ module Karafka
|
|
|
101
101
|
tags.concat(consumer_tags(event.payload[:caller]))
|
|
102
102
|
end
|
|
103
103
|
|
|
104
|
-
count(
|
|
104
|
+
count("error_occurred", 1, tags: tags)
|
|
105
105
|
end
|
|
106
106
|
|
|
107
107
|
# Reports how many messages we've polled and how much time did we spend on it
|
|
@@ -116,8 +116,8 @@ module Karafka
|
|
|
116
116
|
tags = ["consumer_group:#{consumer_group_id}"]
|
|
117
117
|
tags.concat(default_tags)
|
|
118
118
|
|
|
119
|
-
histogram(
|
|
120
|
-
histogram(
|
|
119
|
+
histogram("listener.polling.time_taken", time_taken, tags: tags)
|
|
120
|
+
histogram("listener.polling.messages", messages_count, tags: tags)
|
|
121
121
|
end
|
|
122
122
|
|
|
123
123
|
# Here we report majority of things related to processing as we have access to the
|
|
@@ -131,13 +131,13 @@ module Karafka
|
|
|
131
131
|
tags = consumer_tags(consumer)
|
|
132
132
|
tags.concat(default_tags)
|
|
133
133
|
|
|
134
|
-
count(
|
|
135
|
-
count(
|
|
136
|
-
gauge(
|
|
137
|
-
histogram(
|
|
138
|
-
histogram(
|
|
139
|
-
histogram(
|
|
140
|
-
histogram(
|
|
134
|
+
count("consumer.messages", messages.size, tags: tags)
|
|
135
|
+
count("consumer.batches", 1, tags: tags)
|
|
136
|
+
gauge("consumer.offset", metadata.last_offset, tags: tags)
|
|
137
|
+
histogram("consumer.consumed.time_taken", event[:time], tags: tags)
|
|
138
|
+
histogram("consumer.batch_size", messages.size, tags: tags)
|
|
139
|
+
histogram("consumer.processing_lag", metadata.processing_lag, tags: tags)
|
|
140
|
+
histogram("consumer.consumption_lag", metadata.consumption_lag, tags: tags)
|
|
141
141
|
end
|
|
142
142
|
|
|
143
143
|
{
|
|
@@ -164,9 +164,9 @@ module Karafka
|
|
|
164
164
|
jq_stats = event[:jobs_queue].statistics
|
|
165
165
|
|
|
166
166
|
tags = default_tags
|
|
167
|
-
gauge(
|
|
168
|
-
histogram(
|
|
169
|
-
histogram(
|
|
167
|
+
gauge("worker.total_threads", Karafka::App.config.concurrency, tags: tags)
|
|
168
|
+
histogram("worker.processing", jq_stats[:busy], tags: tags)
|
|
169
|
+
histogram("worker.enqueued_jobs", jq_stats[:enqueued], tags: tags)
|
|
170
170
|
end
|
|
171
171
|
|
|
172
172
|
# We report this metric before and after processing for higher accuracy
|
|
@@ -175,7 +175,7 @@ module Karafka
|
|
|
175
175
|
def on_worker_processed(event)
|
|
176
176
|
jq_stats = event[:jobs_queue].statistics
|
|
177
177
|
|
|
178
|
-
histogram(
|
|
178
|
+
histogram("worker.processing", jq_stats[:busy], tags: default_tags)
|
|
179
179
|
end
|
|
180
180
|
|
|
181
181
|
private
|
|
@@ -213,7 +213,7 @@ module Karafka
|
|
|
213
213
|
else
|
|
214
214
|
raise(
|
|
215
215
|
ArgumentError,
|
|
216
|
-
|
|
216
|
+
"distribution_mode setting value must be either :histogram or :distribution"
|
|
217
217
|
)
|
|
218
218
|
end
|
|
219
219
|
end
|
|
@@ -239,13 +239,13 @@ module Karafka
|
|
|
239
239
|
tags: base_tags
|
|
240
240
|
)
|
|
241
241
|
when :brokers
|
|
242
|
-
statistics.fetch(
|
|
242
|
+
statistics.fetch("brokers").each_value do |broker_statistics|
|
|
243
243
|
# Skip bootstrap nodes
|
|
244
244
|
# Bootstrap nodes have nodeid -1, other nodes have positive
|
|
245
245
|
# node ids
|
|
246
|
-
next if broker_statistics[
|
|
246
|
+
next if broker_statistics["nodeid"] == -1
|
|
247
247
|
|
|
248
|
-
tags = ["broker:#{broker_statistics[
|
|
248
|
+
tags = ["broker:#{broker_statistics["nodename"]}"]
|
|
249
249
|
tags.concat(base_tags)
|
|
250
250
|
|
|
251
251
|
public_send(
|
|
@@ -256,16 +256,16 @@ module Karafka
|
|
|
256
256
|
)
|
|
257
257
|
end
|
|
258
258
|
when :topics
|
|
259
|
-
statistics.fetch(
|
|
260
|
-
topic_values[
|
|
261
|
-
next if partition_name ==
|
|
259
|
+
statistics.fetch("topics").each do |topic_name, topic_values|
|
|
260
|
+
topic_values["partitions"].each do |partition_name, partition_statistics|
|
|
261
|
+
next if partition_name == "-1"
|
|
262
262
|
# Skip until lag info is available
|
|
263
|
-
next if partition_statistics[
|
|
264
|
-
next if partition_statistics[
|
|
263
|
+
next if partition_statistics["consumer_lag"] == -1
|
|
264
|
+
next if partition_statistics["consumer_lag_stored"] == -1
|
|
265
265
|
|
|
266
266
|
# Skip if we do not own the fetch assignment
|
|
267
|
-
next if partition_statistics[
|
|
268
|
-
next if partition_statistics[
|
|
267
|
+
next if partition_statistics["fetch_state"] == "stopped"
|
|
268
|
+
next if partition_statistics["fetch_state"] == "none"
|
|
269
269
|
|
|
270
270
|
tags = ["topic:#{topic_name}", "partition:#{partition_name}"]
|
|
271
271
|
tags.concat(base_tags)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require
|
|
3
|
+
require "socket"
|
|
4
4
|
|
|
5
5
|
module Karafka
|
|
6
6
|
module Instrumentation
|
|
@@ -13,10 +13,10 @@ module Karafka
|
|
|
13
13
|
include Karafka::Core::Helpers::Time
|
|
14
14
|
|
|
15
15
|
# All good with Karafka
|
|
16
|
-
OK_CODE =
|
|
16
|
+
OK_CODE = "200 OK"
|
|
17
17
|
|
|
18
18
|
# Some timeouts, fail
|
|
19
|
-
FAIL_CODE =
|
|
19
|
+
FAIL_CODE = "500 Internal Server Error"
|
|
20
20
|
|
|
21
21
|
private_constant :OK_CODE, :FAIL_CODE
|
|
22
22
|
|
|
@@ -32,7 +32,7 @@ module Karafka
|
|
|
32
32
|
|
|
33
33
|
# @return [Boolean] true if all good, false if we should tell k8s to kill this process
|
|
34
34
|
def healthy?
|
|
35
|
-
raise NotImplementedError,
|
|
35
|
+
raise NotImplementedError, "Implement in a subclass"
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
private
|
|
@@ -58,7 +58,7 @@ module Karafka
|
|
|
58
58
|
# @return [Hash] hash that will be the response body
|
|
59
59
|
def status_body
|
|
60
60
|
{
|
|
61
|
-
status: healthy? ?
|
|
61
|
+
status: healthy? ? "healthy" : "unhealthy",
|
|
62
62
|
timestamp: Time.now.to_i,
|
|
63
63
|
port: @port,
|
|
64
64
|
process_id: ::Process.pid
|
data/lib/karafka/licenser.rb
CHANGED
|
@@ -4,15 +4,25 @@ module Karafka
|
|
|
4
4
|
# Checks the license presence for pro and loads pro components when needed (if any)
|
|
5
5
|
class Licenser
|
|
6
6
|
# Location in the gem where we store the public key
|
|
7
|
-
PUBLIC_KEY_LOCATION = File.join(Karafka.gem_root,
|
|
7
|
+
PUBLIC_KEY_LOCATION = File.join(Karafka.gem_root, "certs", "karafka-pro.pem")
|
|
8
|
+
|
|
9
|
+
# Location of file containing expired/revoked license checksums
|
|
10
|
+
EXPIRED_CHECKSUMS_LOCATION = File.join(Karafka.gem_root, "certs", "expired.txt")
|
|
8
11
|
|
|
9
12
|
private_constant :PUBLIC_KEY_LOCATION
|
|
10
13
|
|
|
11
14
|
class << self
|
|
12
15
|
# Tries to load the license and yields if successful
|
|
13
16
|
def detect
|
|
14
|
-
# If
|
|
15
|
-
|
|
17
|
+
# If license module is already fully defined, don't touch it
|
|
18
|
+
# This allows users to define their own License module for testing/custom setups
|
|
19
|
+
unless license_fully_defined?
|
|
20
|
+
# Try safe approach first (no code execution)
|
|
21
|
+
loaded = safe_load_license || fallback_require_license
|
|
22
|
+
|
|
23
|
+
# If neither method succeeded, return false
|
|
24
|
+
return false unless loaded
|
|
25
|
+
end
|
|
16
26
|
|
|
17
27
|
yield
|
|
18
28
|
|
|
@@ -26,7 +36,7 @@ module Karafka
|
|
|
26
36
|
# @param license_config [Karafka::Core::Configurable::Node] config related to the licensing
|
|
27
37
|
def prepare_and_verify(license_config)
|
|
28
38
|
# If license is not loaded, nothing to do
|
|
29
|
-
return unless const_defined?(
|
|
39
|
+
return unless const_defined?("::Karafka::License")
|
|
30
40
|
|
|
31
41
|
prepare(license_config)
|
|
32
42
|
verify(license_config)
|
|
@@ -34,6 +44,65 @@ module Karafka
|
|
|
34
44
|
|
|
35
45
|
private
|
|
36
46
|
|
|
47
|
+
# Check if License module and required methods are already defined
|
|
48
|
+
# @return [Boolean]
|
|
49
|
+
def license_fully_defined?
|
|
50
|
+
return false unless const_defined?("::Karafka::License")
|
|
51
|
+
|
|
52
|
+
# Check if the required methods exist
|
|
53
|
+
::Karafka::License.respond_to?(:token) &&
|
|
54
|
+
::Karafka::License.respond_to?(:version)
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Attempt to safely load license without executing gem code
|
|
58
|
+
# @return [Boolean] true if successful, false otherwise
|
|
59
|
+
def safe_load_license
|
|
60
|
+
# First verify the gem exists and files are accessible
|
|
61
|
+
spec = Gem::Specification.find_by_name("karafka-license")
|
|
62
|
+
license_path = File.join(spec.gem_dir, "lib", "license.txt")
|
|
63
|
+
version_path = File.join(spec.gem_dir, "lib", "version.txt")
|
|
64
|
+
|
|
65
|
+
return false unless File.exist?(license_path)
|
|
66
|
+
|
|
67
|
+
# Read license data to ensure we can successfully load before removing any constants
|
|
68
|
+
license_token = File.read(license_path)
|
|
69
|
+
version_content = File.read(version_path)
|
|
70
|
+
|
|
71
|
+
# Only after confirming we have valid data, remove and replace the License constant
|
|
72
|
+
# This ensures we always use the actual license from the gem, overwriting any stale
|
|
73
|
+
# or incomplete License module that may have been defined elsewhere
|
|
74
|
+
::Karafka.send(:remove_const, :License) if const_defined?("::Karafka::License")
|
|
75
|
+
|
|
76
|
+
::Karafka.const_set(:License, Module.new)
|
|
77
|
+
::Karafka::License.define_singleton_method(:token) { license_token }
|
|
78
|
+
::Karafka::License.define_singleton_method(:version) { version_content }
|
|
79
|
+
|
|
80
|
+
true
|
|
81
|
+
rescue Gem::MissingSpecError, Errno::ENOENT
|
|
82
|
+
# Gem not found or files don't exist - don't touch existing License constant
|
|
83
|
+
false
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# Fallback to traditional require if safe method fails
|
|
87
|
+
# @return [Boolean] true if successful, false otherwise
|
|
88
|
+
def fallback_require_license
|
|
89
|
+
# First check if we can actually load the gem before removing any constants
|
|
90
|
+
# Try to find the gem spec to see if it exists
|
|
91
|
+
begin
|
|
92
|
+
Gem::Specification.find_by_name("karafka-license")
|
|
93
|
+
rescue Gem::MissingSpecError
|
|
94
|
+
return false
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# Gem exists, so we can safely remove and reload the License constant
|
|
98
|
+
::Karafka.send(:remove_const, :License) if const_defined?("::Karafka::License")
|
|
99
|
+
|
|
100
|
+
require("karafka-license")
|
|
101
|
+
true
|
|
102
|
+
rescue LoadError
|
|
103
|
+
false
|
|
104
|
+
end
|
|
105
|
+
|
|
37
106
|
# @param license_config [Karafka::Core::Configurable::Node] config related to the licensing
|
|
38
107
|
def prepare(license_config)
|
|
39
108
|
license_config.token = Karafka::License.token
|
|
@@ -45,8 +114,12 @@ module Karafka
|
|
|
45
114
|
public_key = OpenSSL::PKey::RSA.new(File.read(PUBLIC_KEY_LOCATION))
|
|
46
115
|
|
|
47
116
|
# We gsub and strip in case someone copy-pasted it as a multi line string
|
|
48
|
-
formatted_token = license_config.token.strip.delete("\n").delete(
|
|
49
|
-
|
|
117
|
+
formatted_token = license_config.token.strip.delete("\n").delete(" ")
|
|
118
|
+
|
|
119
|
+
# Check if the license has been revoked/expired before attempting decryption
|
|
120
|
+
raise_expired_license_token(license_config) if expired?(formatted_token)
|
|
121
|
+
|
|
122
|
+
decoded_token = formatted_token.unpack1("m") # decode from base64
|
|
50
123
|
|
|
51
124
|
begin
|
|
52
125
|
data = public_key.public_decrypt(decoded_token)
|
|
@@ -56,7 +129,26 @@ module Karafka
|
|
|
56
129
|
|
|
57
130
|
details = data ? JSON.parse(data) : raise_invalid_license_token(license_config)
|
|
58
131
|
|
|
59
|
-
license_config.entity = details.fetch(
|
|
132
|
+
license_config.entity = details.fetch("entity")
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
# Checks if the license token has been expired/revoked
|
|
136
|
+
# @param formatted_token [String] formatted license token
|
|
137
|
+
# @return [Boolean] true if the license is in the expired list
|
|
138
|
+
def expired?(formatted_token)
|
|
139
|
+
token_checksum = Digest::SHA256.hexdigest(formatted_token)
|
|
140
|
+
|
|
141
|
+
expired_checksums.include?(token_checksum)
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# Loads the list of expired license checksums
|
|
145
|
+
# @return [Set<String>] set of expired license checksums
|
|
146
|
+
def expired_checksums
|
|
147
|
+
File
|
|
148
|
+
.readlines(EXPIRED_CHECKSUMS_LOCATION)
|
|
149
|
+
.map(&:strip)
|
|
150
|
+
.reject { |line| line.empty? || line.start_with?("#") }
|
|
151
|
+
.to_set
|
|
60
152
|
end
|
|
61
153
|
|
|
62
154
|
# Raises an error with info, that used token is invalid
|
|
@@ -67,12 +159,27 @@ module Karafka
|
|
|
67
159
|
|
|
68
160
|
raise(
|
|
69
161
|
Errors::InvalidLicenseTokenError,
|
|
70
|
-
<<~MSG.tr("\n",
|
|
162
|
+
<<~MSG.tr("\n", " ")
|
|
71
163
|
License key you provided is invalid.
|
|
72
164
|
Please reach us at contact@karafka.io or visit https://karafka.io to obtain a valid one.
|
|
73
165
|
MSG
|
|
74
166
|
)
|
|
75
167
|
end
|
|
168
|
+
|
|
169
|
+
# Raises an error with info, that used token has expired or been revoked
|
|
170
|
+
# @param license_config [Karafka::Core::Configurable::Node]
|
|
171
|
+
def raise_expired_license_token(license_config)
|
|
172
|
+
# We set it to false so `Karafka.pro?` method behaves as expected
|
|
173
|
+
license_config.token = false
|
|
174
|
+
|
|
175
|
+
raise(
|
|
176
|
+
Errors::ExpiredLicenseTokenError,
|
|
177
|
+
<<~MSG.tr("\n", " ")
|
|
178
|
+
License key you provided has expired or been revoked.
|
|
179
|
+
Please reach us at contact@karafka.io or visit https://karafka.io to obtain a valid one.
|
|
180
|
+
MSG
|
|
181
|
+
)
|
|
182
|
+
end
|
|
76
183
|
end
|
|
77
184
|
end
|
|
78
185
|
end
|
|
@@ -17,16 +17,18 @@ module Karafka
|
|
|
17
17
|
# @note We do not set `processed_at` as this needs to be assigned when the batch is
|
|
18
18
|
# picked up for processing.
|
|
19
19
|
def call(messages, topic, partition, scheduled_at)
|
|
20
|
+
last_message = messages.last
|
|
21
|
+
|
|
20
22
|
Karafka::Messages::BatchMetadata.new(
|
|
21
23
|
size: messages.size,
|
|
22
24
|
first_offset: messages.first&.offset || -1001,
|
|
23
|
-
last_offset:
|
|
25
|
+
last_offset: last_message&.offset || -1001,
|
|
24
26
|
deserializers: topic.deserializers,
|
|
25
27
|
partition: partition,
|
|
26
28
|
topic: topic.name,
|
|
27
29
|
# We go with the assumption that the creation of the whole batch is the last message
|
|
28
30
|
# creation time
|
|
29
|
-
created_at: local_created_at(
|
|
31
|
+
created_at: local_created_at(last_message),
|
|
30
32
|
# When this batch was built and scheduled for execution
|
|
31
33
|
scheduled_at: scheduled_at,
|
|
32
34
|
# This needs to be set to a correct value prior to processing starting
|
|
@@ -80,7 +80,7 @@ module Karafka
|
|
|
80
80
|
tpl = ::Rdkafka::Consumer::TopicPartitionList.from_native_tpl(partitions_ptr).freeze
|
|
81
81
|
opaque = ::Rdkafka::Config.opaques[opaque_ptr.to_i]
|
|
82
82
|
|
|
83
|
-
if RB.rd_kafka_rebalance_protocol(client_ptr) ==
|
|
83
|
+
if RB.rd_kafka_rebalance_protocol(client_ptr) == "COOPERATIVE"
|
|
84
84
|
pr.on_cooperative_rebalance(client_ptr, code, partitions_ptr, tpl, opaque)
|
|
85
85
|
else
|
|
86
86
|
pr.on_eager_rebalance(client_ptr, code, partitions_ptr, tpl, opaque)
|
|
@@ -95,7 +95,7 @@ end
|
|
|
95
95
|
# At the moment there is no API in rdkafka-ruby to do so
|
|
96
96
|
Rdkafka::Bindings.send(
|
|
97
97
|
:remove_const,
|
|
98
|
-
|
|
98
|
+
"RebalanceCallback"
|
|
99
99
|
)
|
|
100
100
|
|
|
101
101
|
Rdkafka::Bindings.const_set(
|
|
@@ -28,8 +28,8 @@ module Karafka
|
|
|
28
28
|
class JobOptionsContract < Contracts::Base
|
|
29
29
|
configure do |config|
|
|
30
30
|
config.error_messages = YAML.safe_load_file(
|
|
31
|
-
File.join(Karafka.gem_root,
|
|
32
|
-
).fetch(
|
|
31
|
+
File.join(Karafka.gem_root, "config", "locales", "errors.yml")
|
|
32
|
+
).fetch("en").fetch("validations").fetch("job_options")
|
|
33
33
|
end
|
|
34
34
|
|
|
35
35
|
optional(:producer) { |val| val.nil? || val.respond_to?(:call) }
|
|
@@ -53,6 +53,16 @@ module Karafka
|
|
|
53
53
|
super(&)
|
|
54
54
|
end
|
|
55
55
|
end
|
|
56
|
+
|
|
57
|
+
# Cleans all messages in the batch
|
|
58
|
+
#
|
|
59
|
+
# @param metadata [Boolean] should we also clean metadata alongside the payload for
|
|
60
|
+
# each message. `true` by default.
|
|
61
|
+
#
|
|
62
|
+
# @note This is a convenience method that calls `clean!` on each message in the batch.
|
|
63
|
+
def clean!(metadata: true)
|
|
64
|
+
each { |message| message.clean!(metadata: metadata) }
|
|
65
|
+
end
|
|
56
66
|
end
|
|
57
67
|
end
|
|
58
68
|
end
|
|
@@ -30,8 +30,8 @@ module Karafka
|
|
|
30
30
|
class Server < Karafka::Cli::Contracts::Server
|
|
31
31
|
configure do |config|
|
|
32
32
|
config.error_messages = YAML.safe_load_file(
|
|
33
|
-
File.join(Karafka.gem_root,
|
|
34
|
-
).fetch(
|
|
33
|
+
File.join(Karafka.gem_root, "config", "locales", "errors.yml")
|
|
34
|
+
).fetch("en").fetch("validations").fetch("cli").fetch("server")
|
|
35
35
|
end
|
|
36
36
|
|
|
37
37
|
%i[
|
|
@@ -66,10 +66,10 @@ module Karafka
|
|
|
66
66
|
next if value.empty?
|
|
67
67
|
|
|
68
68
|
subscription_groups = Karafka::App
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
69
|
+
.consumer_groups
|
|
70
|
+
.map(&:subscription_groups)
|
|
71
|
+
.flatten
|
|
72
|
+
.map(&:name)
|
|
73
73
|
|
|
74
74
|
next if (value - subscription_groups).empty?
|
|
75
75
|
|
|
@@ -87,12 +87,12 @@ module Karafka
|
|
|
87
87
|
next if value.empty?
|
|
88
88
|
|
|
89
89
|
topics = Karafka::App
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
90
|
+
.consumer_groups
|
|
91
|
+
.map(&:subscription_groups)
|
|
92
|
+
.flatten
|
|
93
|
+
.map(&:topics)
|
|
94
|
+
.map { |gtopics| gtopics.map(&:name) }
|
|
95
|
+
.flatten
|
|
96
96
|
|
|
97
97
|
next if (value - topics).empty?
|
|
98
98
|
|
|
@@ -47,9 +47,9 @@ module Karafka
|
|
|
47
47
|
requested_groups = options[:groups].dup || []
|
|
48
48
|
|
|
49
49
|
workable_groups = Karafka::App
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
50
|
+
.routes
|
|
51
|
+
.select(&:parallel_segments?)
|
|
52
|
+
.group_by(&:segment_origin)
|
|
53
53
|
|
|
54
54
|
# Use all if none provided
|
|
55
55
|
return workable_groups if requested_groups.empty?
|
|
@@ -85,7 +85,7 @@ module Karafka
|
|
|
85
85
|
consumer_groups = [segment_origin, segments.map(&:name)].flatten
|
|
86
86
|
|
|
87
87
|
consumer_groups_with_topics = consumer_groups
|
|
88
|
-
|
|
88
|
+
.to_h { |name| [name, topics_names] }
|
|
89
89
|
|
|
90
90
|
lags_with_offsets = Karafka::Admin.read_lags_with_offsets(
|
|
91
91
|
consumer_groups_with_topics
|