karafka 2.5.1 → 2.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci_linux_ubuntu_x86_64_gnu.yml +21 -29
- data/.github/workflows/ci_macos_arm64.yml +1 -1
- data/.github/workflows/push.yml +2 -2
- data/.github/workflows/trigger-wiki-refresh.yml +1 -1
- data/.ruby-version +1 -1
- data/.yard-lint.yml +174 -0
- data/CHANGELOG.md +20 -4
- data/Gemfile +1 -2
- data/Gemfile.lock +45 -41
- data/bin/integrations +2 -1
- data/bin/rspecs +4 -0
- data/config/locales/errors.yml +6 -4
- data/config/locales/pro_errors.yml +5 -4
- data/docker-compose.yml +1 -1
- data/examples/payloads/json/sample_set_02/download.json +191 -0
- data/examples/payloads/json/sample_set_03/event_type_1.json +18 -0
- data/examples/payloads/json/sample_set_03/event_type_2.json +263 -0
- data/examples/payloads/json/sample_set_03/event_type_3.json +41 -0
- data/karafka.gemspec +3 -3
- data/lib/active_job/queue_adapters/karafka_adapter.rb +3 -3
- data/lib/karafka/active_job/consumer.rb +7 -3
- data/lib/karafka/active_job/current_attributes/job_wrapper.rb +45 -0
- data/lib/karafka/active_job/current_attributes/loading.rb +1 -1
- data/lib/karafka/active_job/current_attributes/persistence.rb +19 -7
- data/lib/karafka/active_job/current_attributes.rb +3 -2
- data/lib/karafka/active_job/deserializer.rb +61 -0
- data/lib/karafka/active_job/dispatcher.rb +34 -14
- data/lib/karafka/active_job/job_options_contract.rb +2 -4
- data/lib/karafka/admin/acl.rb +8 -4
- data/lib/karafka/admin/configs/config.rb +6 -4
- data/lib/karafka/admin/configs/resource.rb +7 -1
- data/lib/karafka/admin/consumer_groups.rb +80 -12
- data/lib/karafka/admin/topics.rb +43 -9
- data/lib/karafka/admin.rb +23 -14
- data/lib/karafka/app.rb +3 -3
- data/lib/karafka/base_consumer.rb +6 -6
- data/lib/karafka/cli/base.rb +2 -2
- data/lib/karafka/cli/console.rb +1 -1
- data/lib/karafka/cli/contracts/server.rb +3 -5
- data/lib/karafka/cli/help.rb +1 -1
- data/lib/karafka/cli/install.rb +3 -2
- data/lib/karafka/cli/server.rb +1 -1
- data/lib/karafka/cli/swarm.rb +1 -1
- data/lib/karafka/cli/topics/align.rb +1 -1
- data/lib/karafka/cli/topics/repartition.rb +2 -2
- data/lib/karafka/connection/client.rb +30 -19
- data/lib/karafka/connection/listeners_batch.rb +2 -3
- data/lib/karafka/connection/manager.rb +1 -0
- data/lib/karafka/connection/proxy.rb +12 -8
- data/lib/karafka/connection/rebalance_manager.rb +1 -1
- data/lib/karafka/connection/status.rb +1 -0
- data/lib/karafka/constraints.rb +1 -1
- data/lib/karafka/contracts/base.rb +1 -1
- data/lib/karafka/deserializers/payload.rb +1 -1
- data/lib/karafka/env.rb +1 -2
- data/lib/karafka/helpers/async.rb +1 -1
- data/lib/karafka/helpers/config_importer.rb +3 -3
- data/lib/karafka/helpers/interval_runner.rb +4 -1
- data/lib/karafka/helpers/multi_delegator.rb +3 -0
- data/lib/karafka/instrumentation/assignments_tracker.rb +19 -1
- data/lib/karafka/instrumentation/callbacks/error.rb +2 -2
- data/lib/karafka/instrumentation/callbacks/statistics.rb +3 -3
- data/lib/karafka/instrumentation/logger.rb +6 -6
- data/lib/karafka/instrumentation/monitor.rb +3 -3
- data/lib/karafka/instrumentation/notifications.rb +1 -0
- data/lib/karafka/instrumentation/vendors/appsignal/base.rb +3 -4
- data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +3 -4
- data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +10 -11
- data/lib/karafka/instrumentation/vendors/kubernetes/base_listener.rb +1 -1
- data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +5 -18
- data/lib/karafka/messages/builders/batch_metadata.rb +2 -2
- data/lib/karafka/messages/builders/message.rb +1 -1
- data/lib/karafka/messages/messages.rb +2 -3
- data/lib/karafka/patches/rdkafka/bindings.rb +6 -6
- data/lib/karafka/patches/rdkafka/opaque.rb +1 -1
- data/lib/karafka/pro/active_job/consumer.rb +2 -2
- data/lib/karafka/pro/active_job/dispatcher.rb +10 -6
- data/lib/karafka/pro/active_job/job_options_contract.rb +2 -4
- data/lib/karafka/pro/cleaner/messages/messages.rb +2 -3
- data/lib/karafka/pro/cleaner.rb +3 -3
- data/lib/karafka/pro/cli/contracts/server.rb +3 -5
- data/lib/karafka/pro/cli/parallel_segments/base.rb +5 -5
- data/lib/karafka/pro/cli/parallel_segments/collapse.rb +3 -3
- data/lib/karafka/pro/cli/parallel_segments/distribute.rb +3 -3
- data/lib/karafka/pro/cli/parallel_segments.rb +1 -1
- data/lib/karafka/pro/connection/manager.rb +3 -4
- data/lib/karafka/pro/connection/multiplexing/listener.rb +1 -0
- data/lib/karafka/pro/contracts/base.rb +1 -1
- data/lib/karafka/pro/encryption/cipher.rb +3 -2
- data/lib/karafka/pro/encryption/contracts/config.rb +5 -7
- data/lib/karafka/pro/encryption/messages/parser.rb +4 -4
- data/lib/karafka/pro/encryption/setup/config.rb +1 -1
- data/lib/karafka/pro/instrumentation/performance_tracker.rb +3 -3
- data/lib/karafka/pro/iterator/expander.rb +1 -1
- data/lib/karafka/pro/iterator/tpl_builder.rb +2 -2
- data/lib/karafka/pro/iterator.rb +3 -3
- data/lib/karafka/pro/loader.rb +1 -1
- data/lib/karafka/pro/processing/coordinator.rb +1 -1
- data/lib/karafka/pro/processing/coordinators/errors_tracker.rb +2 -3
- data/lib/karafka/pro/processing/coordinators/filters_applier.rb +3 -3
- data/lib/karafka/pro/processing/filters/base.rb +1 -0
- data/lib/karafka/pro/processing/filters/delayer.rb +1 -1
- data/lib/karafka/pro/processing/filters/expirer.rb +1 -1
- data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb +1 -1
- data/lib/karafka/pro/processing/filters/throttler.rb +1 -1
- data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb +1 -1
- data/lib/karafka/pro/processing/jobs/eofed_non_blocking.rb +1 -1
- data/lib/karafka/pro/processing/jobs/periodic.rb +1 -1
- data/lib/karafka/pro/processing/jobs/revoked_non_blocking.rb +1 -1
- data/lib/karafka/pro/processing/jobs_builder.rb +1 -1
- data/lib/karafka/pro/processing/offset_metadata/fetcher.rb +1 -0
- data/lib/karafka/pro/processing/partitioner.rb +1 -1
- data/lib/karafka/pro/processing/schedulers/default.rb +2 -4
- data/lib/karafka/pro/processing/strategies/base.rb +1 -1
- data/lib/karafka/pro/processing/strategies/default.rb +2 -2
- data/lib/karafka/pro/processing/strategies/lrj/default.rb +2 -4
- data/lib/karafka/pro/processing/strategies/vp/default.rb +2 -4
- data/lib/karafka/pro/processing/strategy_selector.rb +1 -0
- data/lib/karafka/pro/processing/subscription_groups_coordinator.rb +2 -3
- data/lib/karafka/pro/processing/virtual_partitions/distributors/balanced.rb +4 -2
- data/lib/karafka/pro/processing/virtual_partitions/distributors/consistent.rb +4 -2
- data/lib/karafka/pro/recurring_tasks/consumer.rb +3 -2
- data/lib/karafka/pro/recurring_tasks/contracts/config.rb +4 -6
- data/lib/karafka/pro/recurring_tasks/contracts/task.rb +3 -5
- data/lib/karafka/pro/recurring_tasks/deserializer.rb +1 -1
- data/lib/karafka/pro/recurring_tasks/dispatcher.rb +7 -6
- data/lib/karafka/pro/recurring_tasks/executor.rb +2 -1
- data/lib/karafka/pro/recurring_tasks/schedule.rb +9 -8
- data/lib/karafka/pro/recurring_tasks/serializer.rb +6 -5
- data/lib/karafka/pro/recurring_tasks/setup/config.rb +2 -2
- data/lib/karafka/pro/recurring_tasks/task.rb +1 -1
- data/lib/karafka/pro/recurring_tasks.rb +8 -5
- data/lib/karafka/pro/routing/features/adaptive_iterator/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb +3 -0
- data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/delaying/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/direct_assignments/contracts/consumer_group.rb +4 -8
- data/lib/karafka/pro/routing/features/direct_assignments/contracts/topic.rb +5 -7
- data/lib/karafka/pro/routing/features/direct_assignments/subscription_group.rb +7 -6
- data/lib/karafka/pro/routing/features/direct_assignments/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/expiring/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/filtering/topic.rb +2 -3
- data/lib/karafka/pro/routing/features/inline_insights/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb +3 -5
- data/lib/karafka/pro/routing/features/multiplexing/subscription_groups_builder.rb +1 -1
- data/lib/karafka/pro/routing/features/multiplexing.rb +5 -5
- data/lib/karafka/pro/routing/features/non_blocking_job/topic.rb +3 -3
- data/lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/offset_metadata.rb +4 -4
- data/lib/karafka/pro/routing/features/parallel_segments/builder.rb +1 -1
- data/lib/karafka/pro/routing/features/parallel_segments/contracts/consumer_group.rb +2 -4
- data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb +3 -5
- data/lib/karafka/pro/routing/features/patterns/contracts/pattern.rb +2 -4
- data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/patterns/patterns.rb +1 -1
- data/lib/karafka/pro/routing/features/pausing/config.rb +26 -0
- data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb +17 -11
- data/lib/karafka/pro/routing/features/pausing/topic.rb +69 -8
- data/lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/periodic_job/topic.rb +1 -1
- data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb +1 -1
- data/lib/karafka/pro/routing/features/recurring_tasks/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/scheduled_messages/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/swarm/contracts/routing.rb +2 -4
- data/lib/karafka/pro/routing/features/swarm/contracts/topic.rb +6 -8
- data/lib/karafka/pro/routing/features/swarm.rb +1 -1
- data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/throttling/topic.rb +3 -1
- data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +2 -4
- data/lib/karafka/pro/scheduled_messages/consumer.rb +1 -1
- data/lib/karafka/pro/scheduled_messages/contracts/config.rb +4 -6
- data/lib/karafka/pro/scheduled_messages/contracts/message.rb +3 -5
- data/lib/karafka/pro/scheduled_messages/daily_buffer.rb +3 -2
- data/lib/karafka/pro/scheduled_messages/day.rb +1 -0
- data/lib/karafka/pro/scheduled_messages/deserializers/headers.rb +1 -1
- data/lib/karafka/pro/scheduled_messages/deserializers/payload.rb +1 -1
- data/lib/karafka/pro/scheduled_messages/max_epoch.rb +1 -0
- data/lib/karafka/pro/scheduled_messages/proxy.rb +1 -1
- data/lib/karafka/pro/scheduled_messages/serializer.rb +3 -3
- data/lib/karafka/pro/scheduled_messages/setup/config.rb +2 -2
- data/lib/karafka/pro/scheduled_messages/state.rb +1 -0
- data/lib/karafka/pro/scheduled_messages/tracker.rb +1 -0
- data/lib/karafka/pro/scheduled_messages.rb +4 -6
- data/lib/karafka/pro/swarm/liveness_listener.rb +2 -2
- data/lib/karafka/process.rb +4 -4
- data/lib/karafka/processing/coordinator.rb +2 -4
- data/lib/karafka/processing/coordinators_buffer.rb +2 -3
- data/lib/karafka/processing/executor.rb +3 -4
- data/lib/karafka/processing/inline_insights/tracker.rb +1 -0
- data/lib/karafka/processing/jobs/base.rb +2 -3
- data/lib/karafka/processing/jobs_queue.rb +1 -1
- data/lib/karafka/processing/result.rb +1 -0
- data/lib/karafka/processing/strategy_selector.rb +1 -0
- data/lib/karafka/processing/workers_batch.rb +2 -3
- data/lib/karafka/railtie.rb +1 -0
- data/lib/karafka/routing/activity_manager.rb +3 -2
- data/lib/karafka/routing/builder.rb +8 -8
- data/lib/karafka/routing/consumer_group.rb +4 -6
- data/lib/karafka/routing/contracts/consumer_group.rb +6 -7
- data/lib/karafka/routing/contracts/routing.rb +2 -4
- data/lib/karafka/routing/contracts/topic.rb +7 -6
- data/lib/karafka/routing/features/active_job/contracts/topic.rb +2 -4
- data/lib/karafka/routing/features/active_job/topic.rb +6 -0
- data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +3 -5
- data/lib/karafka/routing/features/declaratives/contracts/topic.rb +3 -5
- data/lib/karafka/routing/features/declaratives/topic.rb +5 -2
- data/lib/karafka/routing/features/deserializers/contracts/topic.rb +2 -4
- data/lib/karafka/routing/features/deserializers/topic.rb +3 -3
- data/lib/karafka/routing/features/eofed/contracts/topic.rb +2 -4
- data/lib/karafka/routing/features/inline_insights/contracts/topic.rb +2 -4
- data/lib/karafka/routing/features/inline_insights.rb +5 -5
- data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb +2 -4
- data/lib/karafka/routing/router.rb +1 -1
- data/lib/karafka/routing/subscription_group.rb +1 -1
- data/lib/karafka/routing/subscription_groups_builder.rb +1 -0
- data/lib/karafka/routing/topic.rb +3 -3
- data/lib/karafka/routing/topics.rb +4 -9
- data/lib/karafka/server.rb +2 -2
- data/lib/karafka/setup/attributes_map.rb +4 -2
- data/lib/karafka/setup/config.rb +85 -17
- data/lib/karafka/setup/config_proxy.rb +209 -0
- data/lib/karafka/setup/contracts/config.rb +13 -11
- data/lib/karafka/setup/defaults_injector.rb +3 -2
- data/lib/karafka/setup/dsl.rb +2 -3
- data/lib/karafka/swarm/liveness_listener.rb +3 -3
- data/lib/karafka/swarm/manager.rb +7 -6
- data/lib/karafka/swarm/node.rb +1 -1
- data/lib/karafka/swarm/supervisor.rb +2 -1
- data/lib/karafka/time_trackers/base.rb +1 -1
- data/lib/karafka/version.rb +1 -1
- data/lib/karafka.rb +4 -4
- metadata +14 -6
- data/.diffend.yml +0 -3
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require 'active_support/current_attributes'
|
|
4
|
+
require_relative 'current_attributes/job_wrapper'
|
|
4
5
|
require_relative 'current_attributes/loading'
|
|
5
6
|
require_relative 'current_attributes/persistence'
|
|
6
7
|
|
|
@@ -22,8 +23,8 @@ module Karafka
|
|
|
22
23
|
.each { |expandable| expandable.class_attribute :_cattr_klasses, default: {} }
|
|
23
24
|
|
|
24
25
|
# Do not double inject in case of running persist multiple times
|
|
25
|
-
Dispatcher.prepend(Persistence) unless Dispatcher
|
|
26
|
-
Consumer.prepend(Loading) unless Consumer
|
|
26
|
+
Dispatcher.prepend(Persistence) unless Dispatcher <= Persistence
|
|
27
|
+
Consumer.prepend(Loading) unless Consumer <= Loading
|
|
27
28
|
|
|
28
29
|
klasses.map(&:to_s).each do |stringified_klass|
|
|
29
30
|
# Prevent registering same klass multiple times
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Karafka
|
|
4
|
+
module ActiveJob
|
|
5
|
+
# Default deserializer for ActiveJob jobs
|
|
6
|
+
#
|
|
7
|
+
# @note Despite the name, this class handles both serialization (job to Kafka payload) and
|
|
8
|
+
# deserialization (Kafka message to job). It's called "Deserializer" to align with Karafka's
|
|
9
|
+
# naming conventions where message consumption is the primary concern.
|
|
10
|
+
#
|
|
11
|
+
# This class can be inherited and its methods can be overridden to support
|
|
12
|
+
# custom payload formats (e.g., Avro, Protobuf, MessagePack)
|
|
13
|
+
#
|
|
14
|
+
# @example Wrapping jobs in a custom envelope with metadata
|
|
15
|
+
# class EnvelopedJobDeserializer < Karafka::ActiveJob::Deserializer
|
|
16
|
+
# def serialize(job)
|
|
17
|
+
# # Wrap the job in an envelope with additional metadata
|
|
18
|
+
# envelope = {
|
|
19
|
+
# version: 1,
|
|
20
|
+
# produced_at: Time.now.iso8601,
|
|
21
|
+
# producer: 'my-app',
|
|
22
|
+
# payload: job.serialize
|
|
23
|
+
# }
|
|
24
|
+
# ::ActiveSupport::JSON.encode(envelope)
|
|
25
|
+
# end
|
|
26
|
+
#
|
|
27
|
+
# def deserialize(message)
|
|
28
|
+
# # Extract the job from the envelope
|
|
29
|
+
# envelope = ::ActiveSupport::JSON.decode(message.raw_payload)
|
|
30
|
+
#
|
|
31
|
+
# # Could validate envelope version, log metadata, etc.
|
|
32
|
+
# raise 'Unsupported version' if envelope['version'] != 1
|
|
33
|
+
#
|
|
34
|
+
# # Return the actual job data
|
|
35
|
+
# envelope['payload']
|
|
36
|
+
# end
|
|
37
|
+
# end
|
|
38
|
+
#
|
|
39
|
+
# # Configure in Karafka
|
|
40
|
+
# Karafka::App.config.internal.active_job.deserializer = EnvelopedJobDeserializer.new
|
|
41
|
+
class Deserializer
|
|
42
|
+
# Serializes an ActiveJob job into a string payload for Kafka
|
|
43
|
+
#
|
|
44
|
+
# @param job [ActiveJob::Base, #serialize] job to serialize. The job must respond to
|
|
45
|
+
# #serialize which returns a Hash of job attributes. When CurrentAttributes are used,
|
|
46
|
+
# this may be a JobWrapper instance instead of the original ::ActiveJob::Base.
|
|
47
|
+
# @return [String] serialized job payload
|
|
48
|
+
def serialize(job)
|
|
49
|
+
::ActiveSupport::JSON.encode(job.serialize)
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Deserializes a Kafka message payload into an ActiveJob job hash
|
|
53
|
+
#
|
|
54
|
+
# @param message [Karafka::Messages::Message] message containing the job
|
|
55
|
+
# @return [Hash] deserialized job hash
|
|
56
|
+
def deserialize(message)
|
|
57
|
+
::ActiveSupport::JSON.decode(message.raw_payload)
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
@@ -4,6 +4,10 @@ module Karafka
|
|
|
4
4
|
module ActiveJob
|
|
5
5
|
# Dispatcher that sends the ActiveJob job to a proper topic based on the queue name
|
|
6
6
|
class Dispatcher
|
|
7
|
+
include Helpers::ConfigImporter.new(
|
|
8
|
+
deserializer: %i[internal active_job deserializer]
|
|
9
|
+
)
|
|
10
|
+
|
|
7
11
|
# Defaults for dispatching
|
|
8
12
|
# The can be updated by using `#karafka_options` on the job
|
|
9
13
|
DEFAULTS = {
|
|
@@ -15,10 +19,10 @@ module Karafka
|
|
|
15
19
|
|
|
16
20
|
# @param job [ActiveJob::Base] job
|
|
17
21
|
def dispatch(job)
|
|
18
|
-
|
|
22
|
+
Karafka.producer.public_send(
|
|
19
23
|
fetch_option(job, :dispatch_method, DEFAULTS),
|
|
20
24
|
topic: job.queue_name,
|
|
21
|
-
payload:
|
|
25
|
+
payload: serialize_job(job)
|
|
22
26
|
)
|
|
23
27
|
end
|
|
24
28
|
|
|
@@ -34,12 +38,12 @@ module Karafka
|
|
|
34
38
|
|
|
35
39
|
dispatches[d_method] << {
|
|
36
40
|
topic: job.queue_name,
|
|
37
|
-
payload:
|
|
41
|
+
payload: serialize_job(job)
|
|
38
42
|
}
|
|
39
43
|
end
|
|
40
44
|
|
|
41
45
|
dispatches.each do |type, messages|
|
|
42
|
-
|
|
46
|
+
Karafka.producer.public_send(
|
|
43
47
|
type,
|
|
44
48
|
messages
|
|
45
49
|
)
|
|
@@ -48,14 +52,18 @@ module Karafka
|
|
|
48
52
|
|
|
49
53
|
# Raises info, that Karafka backend does not support scheduling jobs if someone wants to
|
|
50
54
|
# schedule jobs in the future. It works for past and present because we want to support
|
|
51
|
-
# things like continuation and `#retry_on` API with no wait and no jitter
|
|
55
|
+
# things like continuation and `#retry_on` API with no wait and no jitter.
|
|
52
56
|
#
|
|
53
57
|
# @param job [Object] job we cannot enqueue
|
|
54
58
|
# @param timestamp [Time] time when job should run
|
|
55
59
|
#
|
|
56
|
-
# @note Karafka Pro supports future jobs
|
|
60
|
+
# @note Karafka Pro supports future jobs via the Scheduled Messages feature
|
|
61
|
+
#
|
|
62
|
+
# @note For ActiveJob Continuation to work without Pro, configure your continuable jobs:
|
|
63
|
+
# self.resume_options = { wait: 0 }
|
|
57
64
|
#
|
|
58
|
-
# @note
|
|
65
|
+
# @note For `#retry_on` to work without Pro, configure with:
|
|
66
|
+
# retry_on SomeError, wait: 0, jitter: 0
|
|
59
67
|
def dispatch_at(job, timestamp)
|
|
60
68
|
# Dispatch at is used by some of the ActiveJob features that actually do not back-off
|
|
61
69
|
# but things go via this API nonetheless.
|
|
@@ -64,13 +72,31 @@ module Karafka
|
|
|
64
72
|
else
|
|
65
73
|
raise NotImplementedError, <<~ERROR_MESSAGE
|
|
66
74
|
This queueing backend does not support scheduling future jobs.
|
|
67
|
-
|
|
75
|
+
|
|
76
|
+
If you're using ActiveJob Continuation, configure your jobs with:
|
|
77
|
+
self.resume_options = { wait: 0 }
|
|
78
|
+
|
|
79
|
+
If you're using retry_on, configure with:
|
|
80
|
+
retry_on SomeError, wait: 0, jitter: 0
|
|
81
|
+
|
|
82
|
+
For full support of delayed job execution, consider using Karafka Pro with Scheduled Messages.
|
|
68
83
|
ERROR_MESSAGE
|
|
69
84
|
end
|
|
70
85
|
end
|
|
71
86
|
|
|
72
87
|
private
|
|
73
88
|
|
|
89
|
+
# Serializes a job using the configured deserializer
|
|
90
|
+
# This method serves as an extension point and can be wrapped by modules like
|
|
91
|
+
# CurrentAttributes::Persistence
|
|
92
|
+
#
|
|
93
|
+
# @param job [ActiveJob::Base, CurrentAttributes::Persistence::JobWrapper] job to serialize.
|
|
94
|
+
# When CurrentAttributes are used, this may be a JobWrapper instead of the original job.
|
|
95
|
+
# @return [String] serialized job payload
|
|
96
|
+
def serialize_job(job)
|
|
97
|
+
deserializer.serialize(job)
|
|
98
|
+
end
|
|
99
|
+
|
|
74
100
|
# @param job [ActiveJob::Base] job
|
|
75
101
|
# @param key [Symbol] key we want to fetch
|
|
76
102
|
# @param defaults [Hash]
|
|
@@ -81,12 +107,6 @@ module Karafka
|
|
|
81
107
|
.karafka_options
|
|
82
108
|
.fetch(key, defaults.fetch(key))
|
|
83
109
|
end
|
|
84
|
-
|
|
85
|
-
# @param job [ActiveJob::Base] job
|
|
86
|
-
# @return [Hash] json representation of the job
|
|
87
|
-
def serialize_job(job)
|
|
88
|
-
job.serialize
|
|
89
|
-
end
|
|
90
110
|
end
|
|
91
111
|
end
|
|
92
112
|
end
|
|
@@ -8,10 +8,8 @@ module Karafka
|
|
|
8
8
|
# all in the same place
|
|
9
9
|
class JobOptionsContract < Contracts::Base
|
|
10
10
|
configure do |config|
|
|
11
|
-
config.error_messages = YAML.
|
|
12
|
-
File.
|
|
13
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
|
|
14
|
-
)
|
|
11
|
+
config.error_messages = YAML.safe_load_file(
|
|
12
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
|
|
15
13
|
).fetch('en').fetch('validations').fetch('job_options')
|
|
16
14
|
end
|
|
17
15
|
|
data/lib/karafka/admin/acl.rb
CHANGED
|
@@ -98,8 +98,10 @@ module Karafka
|
|
|
98
98
|
PERMISSION_TYPES_MAP
|
|
99
99
|
].freeze
|
|
100
100
|
|
|
101
|
-
private_constant
|
|
102
|
-
|
|
101
|
+
private_constant(
|
|
102
|
+
:RESOURCE_TYPES_MAP, :RESOURCE_PATTERNS_TYPE_MAP, :OPERATIONS_MAP, :PERMISSION_TYPES_MAP,
|
|
103
|
+
:ALL_MAPS
|
|
104
|
+
)
|
|
103
105
|
|
|
104
106
|
# Class level APIs that operate on Acl instances and/or return Acl instances.
|
|
105
107
|
# @note For the sake of consistency all methods from this API return array of Acls
|
|
@@ -187,8 +189,10 @@ module Karafka
|
|
|
187
189
|
end
|
|
188
190
|
end
|
|
189
191
|
|
|
190
|
-
attr_reader
|
|
191
|
-
|
|
192
|
+
attr_reader(
|
|
193
|
+
:resource_type, :resource_name, :resource_pattern_type, :principal, :host, :operation,
|
|
194
|
+
:permission_type
|
|
195
|
+
)
|
|
192
196
|
|
|
193
197
|
# Initializes a new Acl instance with specified attributes.
|
|
194
198
|
#
|
|
@@ -71,10 +71,12 @@ module Karafka
|
|
|
71
71
|
def synonym? = @synonym.positive?
|
|
72
72
|
|
|
73
73
|
# @return [Hash] hash that we can use to operate with rdkafka
|
|
74
|
-
def to_native_hash
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
74
|
+
def to_native_hash
|
|
75
|
+
{
|
|
76
|
+
name: name,
|
|
77
|
+
value: value
|
|
78
|
+
}.freeze
|
|
79
|
+
end
|
|
78
80
|
end
|
|
79
81
|
end
|
|
80
82
|
end
|
|
@@ -40,10 +40,16 @@ module Karafka
|
|
|
40
40
|
OPERATIONS_TYPES_MAP.each do |op_name, op_value|
|
|
41
41
|
# Adds an outgoing operation to a given resource of a given type
|
|
42
42
|
# Useful since we alter in batches and not one at a time
|
|
43
|
+
#
|
|
44
|
+
# For example, when op_name is :set and op_value is 0:
|
|
45
|
+
# def set(name, value)
|
|
46
|
+
# @operations[0] << Config.new(name: name, value: value.to_s)
|
|
47
|
+
# end
|
|
48
|
+
default_value = op_name == :delete ? ' = nil' : ''
|
|
43
49
|
class_eval <<~RUBY, __FILE__, __LINE__ + 1
|
|
44
50
|
# @param name [String] name of the config to alter
|
|
45
51
|
# @param value [String] value of the config
|
|
46
|
-
def #{op_name}(name, value
|
|
52
|
+
def #{op_name}(name, value#{default_value})
|
|
47
53
|
@operations[#{op_value}] << Config.new(name: name, value: value.to_s)
|
|
48
54
|
end
|
|
49
55
|
RUBY
|
|
@@ -80,15 +80,13 @@ module Karafka
|
|
|
80
80
|
case casted_position
|
|
81
81
|
# Earliest is not always 0. When compacting/deleting it can be much later, that's why
|
|
82
82
|
# we fetch the oldest possible offset
|
|
83
|
-
|
|
83
|
+
# false is treated the same as 'earliest'
|
|
84
|
+
when 'earliest', false
|
|
84
85
|
LONG_TIME_AGO
|
|
85
86
|
# Latest will always be the high-watermark offset and we can get it just by getting
|
|
86
87
|
# a future position
|
|
87
88
|
when 'latest'
|
|
88
89
|
Time.now + DAY_IN_SECONDS
|
|
89
|
-
# Same as `'earliest'`
|
|
90
|
-
when false
|
|
91
|
-
LONG_TIME_AGO
|
|
92
90
|
# Regular offset case
|
|
93
91
|
else
|
|
94
92
|
position
|
|
@@ -234,17 +232,87 @@ module Karafka
|
|
|
234
232
|
end
|
|
235
233
|
end
|
|
236
234
|
|
|
235
|
+
# Triggers a rebalance for the specified consumer group by briefly joining and leaving
|
|
236
|
+
#
|
|
237
|
+
# @param consumer_group_id [String] consumer group id to trigger rebalance for
|
|
238
|
+
#
|
|
239
|
+
# @return [void]
|
|
240
|
+
#
|
|
241
|
+
# @raise [Karafka::Errors::InvalidConfigurationError] when consumer group is not found in
|
|
242
|
+
# routing or has no topics
|
|
243
|
+
#
|
|
244
|
+
# @note This method creates a temporary "fake" consumer that joins the consumer group,
|
|
245
|
+
# triggering a rebalance when it joins and another when it leaves. This should only be
|
|
246
|
+
# used for operational/testing purposes as it causes two rebalances.
|
|
247
|
+
#
|
|
248
|
+
# @note The consumer group does not need to be running for this to work, but if it is,
|
|
249
|
+
# it will experience rebalances.
|
|
250
|
+
#
|
|
251
|
+
# @note The behavior follows the configured rebalance protocol. For cooperative sticky
|
|
252
|
+
# rebalancing or KIP-848 based protocols, there may be no immediate reaction to the
|
|
253
|
+
# rebalance trigger as these protocols allow incremental partition reassignments without
|
|
254
|
+
# stopping all consumers.
|
|
255
|
+
#
|
|
256
|
+
# @note Topics are always detected from the routing configuration. The consumer settings
|
|
257
|
+
# (kafka config) are taken from the first topic in the consumer group to ensure
|
|
258
|
+
# consistency with the actual consumer configuration.
|
|
259
|
+
#
|
|
260
|
+
# @example Trigger rebalance for a consumer group
|
|
261
|
+
# Karafka::Admin::ConsumerGroups.trigger_rebalance('my-group')
|
|
262
|
+
def trigger_rebalance(consumer_group_id)
|
|
263
|
+
consumer_group = Karafka::App.routes.find { |cg| cg.id == consumer_group_id }
|
|
264
|
+
|
|
265
|
+
unless consumer_group
|
|
266
|
+
raise(
|
|
267
|
+
Errors::InvalidConfigurationError,
|
|
268
|
+
"Consumer group '#{consumer_group_id}' not found in routing"
|
|
269
|
+
)
|
|
270
|
+
end
|
|
271
|
+
|
|
272
|
+
topics = consumer_group.topics.map(&:name)
|
|
273
|
+
|
|
274
|
+
if topics.empty?
|
|
275
|
+
raise(
|
|
276
|
+
Errors::InvalidConfigurationError,
|
|
277
|
+
"Consumer group '#{consumer_group_id}' has no topics"
|
|
278
|
+
)
|
|
279
|
+
end
|
|
280
|
+
|
|
281
|
+
# Get the first topic to extract kafka settings
|
|
282
|
+
first_topic = consumer_group.topics.first
|
|
283
|
+
|
|
284
|
+
# Build consumer settings using the consumer group's kafka config from first topic
|
|
285
|
+
# This ensures we use the same settings as the actual consumers
|
|
286
|
+
# Following the same pattern as in Karafka::Connection::Client#build_kafka
|
|
287
|
+
consumer_settings = Setup::AttributesMap.consumer(first_topic.kafka.dup)
|
|
288
|
+
consumer_settings[:'group.id'] = consumer_group.id
|
|
289
|
+
consumer_settings[:'enable.auto.offset.store'] = false
|
|
290
|
+
consumer_settings[:'auto.offset.reset'] ||= first_topic.initial_offset
|
|
291
|
+
|
|
292
|
+
with_consumer(consumer_settings) do |consumer|
|
|
293
|
+
# Subscribe to the topics - this triggers the first rebalance
|
|
294
|
+
consumer.subscribe(*topics)
|
|
295
|
+
|
|
296
|
+
# Wait briefly (100ms) to allow the rebalance to initiate
|
|
297
|
+
# The actual rebalance happens asynchronously, so we just need to give it a moment
|
|
298
|
+
sleep(0.1)
|
|
299
|
+
|
|
300
|
+
# Unsubscribe - this will trigger the second rebalance when the consumer closes
|
|
301
|
+
# The ensure block in with_consumer will handle the unsubscribe and close
|
|
302
|
+
end
|
|
303
|
+
end
|
|
304
|
+
|
|
237
305
|
# Reads lags and offsets for given topics in the context of consumer groups defined in the
|
|
238
306
|
# routing
|
|
239
307
|
#
|
|
240
|
-
# @param consumer_groups_with_topics [Hash
|
|
308
|
+
# @param consumer_groups_with_topics [Hash{String => Array<String>}] hash with consumer
|
|
241
309
|
# groups names with array of topics to query per consumer group inside
|
|
242
310
|
# @param active_topics_only [Boolean] if set to false, when we use routing topics, will
|
|
243
311
|
# select also topics that are marked as inactive in routing
|
|
244
312
|
#
|
|
245
|
-
# @return [Hash
|
|
246
|
-
# the consumer groups and values are hashes with topics and inside
|
|
247
|
-
# and offsets
|
|
313
|
+
# @return [Hash{String => Hash{Integer => Hash{Integer => Object}}}] hash where the top
|
|
314
|
+
# level keys are the consumer groups and values are hashes with topics and inside
|
|
315
|
+
# partitions with lags and offsets
|
|
248
316
|
#
|
|
249
317
|
# @note For topics that do not exist, topic details will be set to an empty hash
|
|
250
318
|
#
|
|
@@ -257,19 +325,19 @@ module Karafka
|
|
|
257
325
|
# We first fetch all the topics with partitions count that exist in the cluster so we
|
|
258
326
|
# do not query for topics that do not exist and so we can get partitions count for all
|
|
259
327
|
# the topics we may need. The non-existent and not consumed will be filled at the end
|
|
260
|
-
existing_topics = cluster_info.topics.
|
|
328
|
+
existing_topics = cluster_info.topics.to_h do |topic|
|
|
261
329
|
[topic[:topic_name], topic[:partition_count]]
|
|
262
|
-
end.
|
|
330
|
+
end.freeze
|
|
263
331
|
|
|
264
332
|
# If no expected CGs, we use all from routing that have active topics
|
|
265
333
|
if consumer_groups_with_topics.empty?
|
|
266
|
-
consumer_groups_with_topics = Karafka::App.routes.
|
|
334
|
+
consumer_groups_with_topics = Karafka::App.routes.to_h do |cg|
|
|
267
335
|
cg_topics = cg.topics.select do |cg_topic|
|
|
268
336
|
active_topics_only ? cg_topic.active? : true
|
|
269
337
|
end
|
|
270
338
|
|
|
271
339
|
[cg.id, cg_topics.map(&:name)]
|
|
272
|
-
end
|
|
340
|
+
end
|
|
273
341
|
end
|
|
274
342
|
|
|
275
343
|
# We make a copy because we will remove once with non-existing topics
|
data/lib/karafka/admin/topics.rb
CHANGED
|
@@ -36,7 +36,7 @@ module Karafka
|
|
|
36
36
|
|
|
37
37
|
# Build the requested range - since first element is on the start offset we need to
|
|
38
38
|
# subtract one from requested count to end up with expected number of elements
|
|
39
|
-
requested_range = (start_offset..start_offset +
|
|
39
|
+
requested_range = (start_offset..(start_offset + count - 1))
|
|
40
40
|
# Establish theoretical available range. Note, that this does not handle cases related
|
|
41
41
|
# to log retention or compaction
|
|
42
42
|
available_range = (low_offset..(high_offset - 1))
|
|
@@ -75,7 +75,7 @@ module Karafka
|
|
|
75
75
|
# Use topic from routes if we can match it or create a dummy one
|
|
76
76
|
# Dummy one is used in case we cannot match the topic with routes. This can happen
|
|
77
77
|
# when admin API is used to read topics that are not part of the routing
|
|
78
|
-
topic =
|
|
78
|
+
topic = Karafka::Routing::Router.find_or_initialize_by_name(name)
|
|
79
79
|
|
|
80
80
|
messages.map! do |message|
|
|
81
81
|
Messages::Builders::Message.call(
|
|
@@ -139,15 +139,49 @@ module Karafka
|
|
|
139
139
|
end
|
|
140
140
|
end
|
|
141
141
|
|
|
142
|
-
# Fetches the watermark offsets for a given topic partition
|
|
142
|
+
# Fetches the watermark offsets for a given topic partition or multiple topics and
|
|
143
|
+
# partitions
|
|
143
144
|
#
|
|
144
|
-
# @param
|
|
145
|
-
# @param partition [Integer] partition
|
|
146
|
-
#
|
|
147
|
-
|
|
145
|
+
# @param name_or_hash [String, Symbol, Hash] topic name or hash with topics and partitions
|
|
146
|
+
# @param partition [Integer, nil] partition number
|
|
147
|
+
# (required when first param is topic name)
|
|
148
|
+
#
|
|
149
|
+
# @return [Array<Integer, Integer>, Hash] when querying single partition returns array with
|
|
150
|
+
# low and high watermark offsets, when querying multiple returns nested hash
|
|
151
|
+
#
|
|
152
|
+
# @example Query single partition
|
|
153
|
+
# Karafka::Admin::Topics.read_watermark_offsets('events', 0)
|
|
154
|
+
# # => [0, 100]
|
|
155
|
+
#
|
|
156
|
+
# @example Query specific partitions across multiple topics
|
|
157
|
+
# Karafka::Admin::Topics.read_watermark_offsets(
|
|
158
|
+
# { 'events' => [0, 1], 'logs' => [0] }
|
|
159
|
+
# )
|
|
160
|
+
# # => {
|
|
161
|
+
# # 'events' => {
|
|
162
|
+
# # 0 => [0, 100],
|
|
163
|
+
# # 1 => [0, 150]
|
|
164
|
+
# # },
|
|
165
|
+
# # 'logs' => {
|
|
166
|
+
# # 0 => [0, 50]
|
|
167
|
+
# # }
|
|
168
|
+
# # }
|
|
169
|
+
def read_watermark_offsets(name_or_hash, partition = nil)
|
|
170
|
+
# Normalize input to hash format
|
|
171
|
+
topics_with_partitions = partition ? { name_or_hash => [partition] } : name_or_hash
|
|
172
|
+
|
|
173
|
+
result = Hash.new { |h, k| h[k] = {} }
|
|
174
|
+
|
|
148
175
|
with_consumer do |consumer|
|
|
149
|
-
|
|
176
|
+
topics_with_partitions.each do |topic, partitions|
|
|
177
|
+
partitions.each do |partition_id|
|
|
178
|
+
result[topic][partition_id] = consumer.query_watermark_offsets(topic, partition_id)
|
|
179
|
+
end
|
|
180
|
+
end
|
|
150
181
|
end
|
|
182
|
+
|
|
183
|
+
# Return single array for single partition query, hash for multiple
|
|
184
|
+
partition ? result.dig(name_or_hash, partition) : result
|
|
151
185
|
end
|
|
152
186
|
|
|
153
187
|
# Returns basic topic metadata
|
|
@@ -184,7 +218,7 @@ module Karafka
|
|
|
184
218
|
# @return [Integer] expected offset
|
|
185
219
|
def resolve_offset(consumer, name, partition, offset)
|
|
186
220
|
if offset.is_a?(Time)
|
|
187
|
-
tpl =
|
|
221
|
+
tpl = Rdkafka::Consumer::TopicPartitionList.new
|
|
188
222
|
tpl.add_topic_and_partitions_with_offsets(
|
|
189
223
|
name, partition => offset
|
|
190
224
|
)
|
data/lib/karafka/admin.rb
CHANGED
|
@@ -59,11 +59,11 @@ module Karafka
|
|
|
59
59
|
Topics.create_partitions(name, partitions)
|
|
60
60
|
end
|
|
61
61
|
|
|
62
|
-
# @param
|
|
63
|
-
# @param partition [Integer] partition
|
|
62
|
+
# @param name_or_hash [String, Symbol, Hash] topic name or hash with topics and partitions
|
|
63
|
+
# @param partition [Integer, nil] partition (nil when using hash format)
|
|
64
64
|
# @see Topics.read_watermark_offsets
|
|
65
|
-
def read_watermark_offsets(
|
|
66
|
-
Topics.read_watermark_offsets(
|
|
65
|
+
def read_watermark_offsets(name_or_hash, partition = nil)
|
|
66
|
+
Topics.read_watermark_offsets(name_or_hash, partition)
|
|
67
67
|
end
|
|
68
68
|
|
|
69
69
|
# @param topic_name [String] name of the topic we're interested in
|
|
@@ -113,15 +113,24 @@ module Karafka
|
|
|
113
113
|
ConsumerGroups.delete(consumer_group_id)
|
|
114
114
|
end
|
|
115
115
|
|
|
116
|
+
# Triggers a rebalance for the specified consumer group
|
|
117
|
+
#
|
|
118
|
+
# @param consumer_group_id [String] consumer group id to trigger rebalance for
|
|
119
|
+
# @see ConsumerGroups.trigger_rebalance
|
|
120
|
+
# @note This API should be used only for development.
|
|
121
|
+
def trigger_rebalance(consumer_group_id)
|
|
122
|
+
ConsumerGroups.trigger_rebalance(consumer_group_id)
|
|
123
|
+
end
|
|
124
|
+
|
|
116
125
|
# Reads lags and offsets for given topics in the context of consumer groups defined in the
|
|
117
126
|
# routing
|
|
118
|
-
# @param consumer_groups_with_topics [Hash
|
|
119
|
-
# names with array of topics to query per consumer group inside
|
|
127
|
+
# @param consumer_groups_with_topics [Hash{String => Array<String>}] hash with consumer
|
|
128
|
+
# groups names with array of topics to query per consumer group inside
|
|
120
129
|
# @param active_topics_only [Boolean] if set to false, when we use routing topics, will
|
|
121
130
|
# select also topics that are marked as inactive in routing
|
|
122
|
-
# @return [Hash
|
|
123
|
-
# the consumer groups and values are hashes with topics and inside
|
|
124
|
-
# and offsets
|
|
131
|
+
# @return [Hash{String => Hash{Integer => Hash{Integer => Object}}}] hash where the top
|
|
132
|
+
# level keys are the consumer groups and values are hashes with topics and inside
|
|
133
|
+
# partitions with lags and offsets
|
|
125
134
|
# @see ConsumerGroups.read_lags_with_offsets
|
|
126
135
|
def read_lags_with_offsets(consumer_groups_with_topics = {}, active_topics_only: true)
|
|
127
136
|
ConsumerGroups.read_lags_with_offsets(
|
|
@@ -149,7 +158,7 @@ module Karafka
|
|
|
149
158
|
bind_oauth(bind_id, consumer)
|
|
150
159
|
|
|
151
160
|
consumer.start
|
|
152
|
-
proxy =
|
|
161
|
+
proxy = Karafka::Connection::Proxy.new(consumer)
|
|
153
162
|
yield(proxy)
|
|
154
163
|
ensure
|
|
155
164
|
# Always unsubscribe consumer just to be sure, that no metadata requests are running
|
|
@@ -179,7 +188,7 @@ module Karafka
|
|
|
179
188
|
bind_oauth(bind_id, admin)
|
|
180
189
|
|
|
181
190
|
admin.start
|
|
182
|
-
proxy =
|
|
191
|
+
proxy = Karafka::Connection::Proxy.new(admin)
|
|
183
192
|
yield(proxy)
|
|
184
193
|
ensure
|
|
185
194
|
admin&.close
|
|
@@ -202,7 +211,7 @@ module Karafka
|
|
|
202
211
|
# @param instance [Rdkafka::Consumer, Rdkafka::Admin] rdkafka instance to be used to set
|
|
203
212
|
# appropriate oauth token when needed
|
|
204
213
|
def bind_oauth(id, instance)
|
|
205
|
-
|
|
214
|
+
Karafka::Core::Instrumentation.oauthbearer_token_refresh_callbacks.add(
|
|
206
215
|
id,
|
|
207
216
|
Instrumentation::Callbacks::OauthbearerTokenRefresh.new(
|
|
208
217
|
instance
|
|
@@ -215,7 +224,7 @@ module Karafka
|
|
|
215
224
|
# @param id [String, Symbol] unique (for the lifetime of instance) id that we use for
|
|
216
225
|
# callback referencing
|
|
217
226
|
def unbind_oauth(id)
|
|
218
|
-
|
|
227
|
+
Karafka::Core::Instrumentation.oauthbearer_token_refresh_callbacks.delete(id)
|
|
219
228
|
end
|
|
220
229
|
|
|
221
230
|
# There are some cases where rdkafka admin operations finish successfully but without the
|
|
@@ -260,7 +269,7 @@ module Karafka
|
|
|
260
269
|
# consumer group or do something similar
|
|
261
270
|
.merge!(settings)
|
|
262
271
|
.then { |config| Karafka::Setup::AttributesMap.public_send(type, config) }
|
|
263
|
-
.then { |config|
|
|
272
|
+
.then { |config| Rdkafka::Config.new(config) }
|
|
264
273
|
end
|
|
265
274
|
end
|
|
266
275
|
end
|
data/lib/karafka/app.rb
CHANGED
|
@@ -52,7 +52,7 @@ module Karafka
|
|
|
52
52
|
|
|
53
53
|
# Returns current assignments of this process. Both topics and partitions
|
|
54
54
|
#
|
|
55
|
-
# @return [Hash
|
|
55
|
+
# @return [Hash{Karafka::Routing::Topic => Array<Integer>}]
|
|
56
56
|
def assignments
|
|
57
57
|
Instrumentation::AssignmentsTracker.instance.current
|
|
58
58
|
end
|
|
@@ -102,8 +102,8 @@ module Karafka
|
|
|
102
102
|
#
|
|
103
103
|
# @param contexts [String] librdkafka low level debug contexts for granular debugging
|
|
104
104
|
def debug!(contexts = 'all')
|
|
105
|
-
logger.level =
|
|
106
|
-
producer.config.logger.level =
|
|
105
|
+
logger.level = Logger::DEBUG
|
|
106
|
+
producer.config.logger.level = Logger::DEBUG
|
|
107
107
|
|
|
108
108
|
config.kafka[:debug] = contexts
|
|
109
109
|
producer.config.kafka[:debug] = contexts
|
|
@@ -5,7 +5,7 @@ module Karafka
|
|
|
5
5
|
# Base consumer from which all Karafka consumers should inherit
|
|
6
6
|
class BaseConsumer
|
|
7
7
|
# Allow for consumer instance tagging for instrumentation
|
|
8
|
-
include
|
|
8
|
+
include Karafka::Core::Taggable
|
|
9
9
|
include Helpers::ConfigImporter.new(
|
|
10
10
|
monitor: %i[monitor]
|
|
11
11
|
)
|
|
@@ -14,8 +14,9 @@ module Karafka
|
|
|
14
14
|
|
|
15
15
|
def_delegators :@coordinator, :topic, :partition, :eofed?, :seek_offset, :seek_offset=
|
|
16
16
|
|
|
17
|
-
def_delegators
|
|
18
|
-
|
|
17
|
+
def_delegators(
|
|
18
|
+
:producer, :produce_async, :produce_sync, :produce_many_async, :produce_many_sync
|
|
19
|
+
)
|
|
19
20
|
|
|
20
21
|
def_delegators :messages, :each
|
|
21
22
|
|
|
@@ -81,9 +82,8 @@ module Karafka
|
|
|
81
82
|
# @private
|
|
82
83
|
#
|
|
83
84
|
# @param action [Symbol]
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
handle_wrap(action, &block)
|
|
85
|
+
def on_wrap(action, &)
|
|
86
|
+
handle_wrap(action, &)
|
|
87
87
|
rescue StandardError => e
|
|
88
88
|
monitor.instrument(
|
|
89
89
|
'error.occurred',
|
data/lib/karafka/cli/base.rb
CHANGED
|
@@ -68,11 +68,11 @@ module Karafka
|
|
|
68
68
|
|
|
69
69
|
# However when it is unavailable, we still want to be able to run help command
|
|
70
70
|
# and install command as they don't require configured app itself to run
|
|
71
|
-
return if %w[-h install].any?
|
|
71
|
+
return if %w[-h install].any?(ARGV[0])
|
|
72
72
|
|
|
73
73
|
# All other commands except help and install do require an existing boot file if it was
|
|
74
74
|
# declared
|
|
75
|
-
raise
|
|
75
|
+
raise Karafka::Errors::MissingBootFileError, Karafka.boot_file
|
|
76
76
|
end
|
|
77
77
|
|
|
78
78
|
# Allows to set options for Thor cli
|
data/lib/karafka/cli/console.rb
CHANGED
|
@@ -5,12 +5,10 @@ module Karafka
|
|
|
5
5
|
# CLI related contracts
|
|
6
6
|
module Contracts
|
|
7
7
|
# Contract for validating correctness of the server cli command options.
|
|
8
|
-
class Server <
|
|
8
|
+
class Server < Karafka::Contracts::Base
|
|
9
9
|
configure do |config|
|
|
10
|
-
config.error_messages = YAML.
|
|
11
|
-
File.
|
|
12
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
|
|
13
|
-
)
|
|
10
|
+
config.error_messages = YAML.safe_load_file(
|
|
11
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
|
|
14
12
|
).fetch('en').fetch('validations').fetch('cli').fetch('server')
|
|
15
13
|
end
|
|
16
14
|
|