karafka 2.5.5 → 2.5.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +20 -0
- data/LICENSE-COMM +4 -0
- data/README.md +2 -2
- data/certs/expired.txt +2 -0
- data/karafka.gemspec +23 -23
- data/lib/active_job/karafka.rb +2 -2
- data/lib/active_job/queue_adapters/karafka_adapter.rb +5 -5
- data/lib/karafka/active_job/consumer.rb +3 -3
- data/lib/karafka/active_job/current_attributes.rb +4 -4
- data/lib/karafka/active_job/job_options_contract.rb +2 -2
- data/lib/karafka/admin/acl.rb +3 -3
- data/lib/karafka/admin/configs/resource.rb +1 -1
- data/lib/karafka/admin/configs.rb +1 -1
- data/lib/karafka/admin/consumer_groups.rb +8 -8
- data/lib/karafka/admin/contracts/replication.rb +2 -2
- data/lib/karafka/admin/replication.rb +21 -21
- data/lib/karafka/admin/topics.rb +6 -6
- data/lib/karafka/admin.rb +4 -5
- data/lib/karafka/app.rb +3 -3
- data/lib/karafka/base_consumer.rb +34 -30
- data/lib/karafka/cli/base.rb +8 -8
- data/lib/karafka/cli/console.rb +1 -1
- data/lib/karafka/cli/contracts/server.rb +12 -12
- data/lib/karafka/cli/help.rb +2 -2
- data/lib/karafka/cli/info.rb +4 -4
- data/lib/karafka/cli/install.rb +11 -11
- data/lib/karafka/cli/server.rb +6 -6
- data/lib/karafka/cli/swarm.rb +1 -1
- data/lib/karafka/cli/topics/align.rb +4 -4
- data/lib/karafka/cli/topics/base.rb +5 -5
- data/lib/karafka/cli/topics/create.rb +2 -2
- data/lib/karafka/cli/topics/delete.rb +2 -2
- data/lib/karafka/cli/topics/help.rb +5 -1
- data/lib/karafka/cli/topics/plan.rb +16 -16
- data/lib/karafka/cli/topics/repartition.rb +3 -3
- data/lib/karafka/cli/topics.rb +22 -22
- data/lib/karafka/cli.rb +2 -2
- data/lib/karafka/connection/client.rb +17 -17
- data/lib/karafka/connection/listener.rb +6 -6
- data/lib/karafka/connection/mode.rb +1 -1
- data/lib/karafka/connection/proxy.rb +1 -1
- data/lib/karafka/connection/status.rb +2 -2
- data/lib/karafka/constraints.rb +3 -3
- data/lib/karafka/embedded.rb +3 -3
- data/lib/karafka/env.rb +4 -4
- data/lib/karafka/errors.rb +6 -1
- data/lib/karafka/execution_mode.rb +1 -1
- data/lib/karafka/helpers/config_importer.rb +2 -2
- data/lib/karafka/helpers/interval_runner.rb +4 -2
- data/lib/karafka/helpers/multi_delegator.rb +1 -1
- data/lib/karafka/instrumentation/assignments_tracker.rb +9 -9
- data/lib/karafka/instrumentation/callbacks/error.rb +5 -5
- data/lib/karafka/instrumentation/callbacks/oauthbearer_token_refresh.rb +4 -4
- data/lib/karafka/instrumentation/callbacks/rebalance.rb +6 -6
- data/lib/karafka/instrumentation/callbacks/statistics.rb +5 -5
- data/lib/karafka/instrumentation/logger.rb +7 -7
- data/lib/karafka/instrumentation/logger_listener.rb +76 -63
- data/lib/karafka/instrumentation/vendors/appsignal/base.rb +1 -1
- data/lib/karafka/instrumentation/vendors/appsignal/client.rb +1 -1
- data/lib/karafka/instrumentation/vendors/appsignal/errors_listener.rb +1 -1
- data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb +36 -36
- data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +33 -28
- data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +38 -38
- data/lib/karafka/instrumentation/vendors/kubernetes/base_listener.rb +5 -5
- data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +1 -1
- data/lib/karafka/instrumentation/vendors/kubernetes/swarm_liveness_listener.rb +1 -1
- data/lib/karafka/licenser.rb +115 -8
- data/lib/karafka/messages/builders/batch_metadata.rb +4 -2
- data/lib/karafka/messages/messages.rb +1 -1
- data/lib/karafka/patches/rdkafka/bindings.rb +2 -2
- data/lib/karafka/pro/active_job/job_options_contract.rb +2 -2
- data/lib/karafka/pro/cleaner/messages/messages.rb +10 -0
- data/lib/karafka/pro/cli/contracts/server.rb +12 -12
- data/lib/karafka/pro/cli/parallel_segments/base.rb +4 -4
- data/lib/karafka/pro/cli/parallel_segments/collapse.rb +5 -5
- data/lib/karafka/pro/cli/parallel_segments/distribute.rb +3 -3
- data/lib/karafka/pro/cli/parallel_segments.rb +7 -7
- data/lib/karafka/pro/cli/topics/health.rb +162 -0
- data/lib/karafka/pro/cli/topics.rb +52 -0
- data/lib/karafka/pro/connection/manager.rb +14 -14
- data/lib/karafka/pro/encryption/contracts/config.rb +2 -2
- data/lib/karafka/pro/encryption/messages/middleware.rb +2 -2
- data/lib/karafka/pro/encryption/messages/parser.rb +2 -2
- data/lib/karafka/pro/encryption/setup/config.rb +2 -2
- data/lib/karafka/pro/iterator/tpl_builder.rb +2 -2
- data/lib/karafka/pro/iterator.rb +1 -1
- data/lib/karafka/pro/loader.rb +2 -1
- data/lib/karafka/pro/processing/adaptive_iterator/consumer.rb +1 -1
- data/lib/karafka/pro/processing/coordinators/virtual_offset_manager.rb +24 -14
- data/lib/karafka/pro/processing/filters/base.rb +1 -1
- data/lib/karafka/pro/processing/filters/delayer.rb +2 -2
- data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb +1 -1
- data/lib/karafka/pro/processing/offset_metadata/consumer.rb +1 -1
- data/lib/karafka/pro/processing/parallel_segments/filters/base.rb +6 -6
- data/lib/karafka/pro/processing/partitioner.rb +3 -3
- data/lib/karafka/pro/processing/periodic_job/consumer.rb +6 -5
- data/lib/karafka/pro/processing/piping/consumer.rb +7 -7
- data/lib/karafka/pro/processing/schedulers/base.rb +5 -5
- data/lib/karafka/pro/processing/schedulers/default.rb +5 -5
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb +6 -3
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb +6 -3
- data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom_vp.rb +6 -3
- data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb +2 -2
- data/lib/karafka/pro/processing/strategies/default.rb +22 -22
- data/lib/karafka/pro/processing/strategies/dlq/default.rb +7 -7
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb +6 -3
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb +6 -3
- data/lib/karafka/pro/processing/strategies/ftr/default.rb +2 -2
- data/lib/karafka/pro/processing/strategies/lrj/default.rb +2 -2
- data/lib/karafka/pro/processing/strategies/lrj/ftr.rb +6 -3
- data/lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb +6 -3
- data/lib/karafka/pro/processing/strategies/lrj/mom.rb +2 -2
- data/lib/karafka/pro/recurring_tasks/consumer.rb +2 -2
- data/lib/karafka/pro/recurring_tasks/contracts/config.rb +2 -2
- data/lib/karafka/pro/recurring_tasks/contracts/task.rb +2 -2
- data/lib/karafka/pro/recurring_tasks/dispatcher.rb +2 -2
- data/lib/karafka/pro/recurring_tasks/listener.rb +1 -1
- data/lib/karafka/pro/recurring_tasks/matcher.rb +2 -2
- data/lib/karafka/pro/recurring_tasks/serializer.rb +5 -5
- data/lib/karafka/pro/recurring_tasks/setup/config.rb +3 -3
- data/lib/karafka/pro/recurring_tasks/task.rb +4 -4
- data/lib/karafka/pro/recurring_tasks.rb +4 -4
- data/lib/karafka/pro/routing/features/adaptive_iterator/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb +1 -1
- data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/direct_assignments/contracts/consumer_group.rb +2 -2
- data/lib/karafka/pro/routing/features/direct_assignments/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/direct_assignments/topic.rb +1 -1
- data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/inline_insights/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/long_running_job/topic.rb +1 -1
- data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/multiplexing.rb +5 -5
- data/lib/karafka/pro/routing/features/non_blocking_job/topic.rb +1 -1
- data/lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/offset_metadata/topic.rb +1 -1
- data/lib/karafka/pro/routing/features/offset_metadata.rb +1 -1
- data/lib/karafka/pro/routing/features/parallel_segments/consumer_group.rb +5 -5
- data/lib/karafka/pro/routing/features/parallel_segments/contracts/consumer_group.rb +2 -2
- data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb +2 -2
- data/lib/karafka/pro/routing/features/patterns/contracts/pattern.rb +3 -3
- data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/patterns/topic.rb +1 -1
- data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/periodic_job/topic.rb +1 -1
- data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb +7 -7
- data/lib/karafka/pro/routing/features/recurring_tasks/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/scheduled_messages/builder.rb +13 -13
- data/lib/karafka/pro/routing/features/scheduled_messages/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/swarm/contracts/routing.rb +2 -2
- data/lib/karafka/pro/routing/features/swarm/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/swarm.rb +1 -1
- data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/virtual_partitions/config.rb +7 -7
- data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +2 -2
- data/lib/karafka/pro/scheduled_messages/consumer.rb +4 -4
- data/lib/karafka/pro/scheduled_messages/contracts/config.rb +2 -2
- data/lib/karafka/pro/scheduled_messages/contracts/message.rb +10 -10
- data/lib/karafka/pro/scheduled_messages/daily_buffer.rb +2 -2
- data/lib/karafka/pro/scheduled_messages/deserializers/headers.rb +4 -4
- data/lib/karafka/pro/scheduled_messages/dispatcher.rb +5 -5
- data/lib/karafka/pro/scheduled_messages/proxy.rb +8 -8
- data/lib/karafka/pro/scheduled_messages/schema_validator.rb +1 -1
- data/lib/karafka/pro/scheduled_messages/setup/config.rb +2 -2
- data/lib/karafka/pro/scheduled_messages/state.rb +1 -1
- data/lib/karafka/pro/scheduled_messages/tracker.rb +2 -2
- data/lib/karafka/pro/scheduled_messages.rb +2 -2
- data/lib/karafka/pro/swarm/liveness_listener.rb +2 -2
- data/lib/karafka/process.rb +1 -1
- data/lib/karafka/processing/coordinator.rb +1 -1
- data/lib/karafka/processing/inline_insights/consumer.rb +4 -4
- data/lib/karafka/processing/inline_insights/tracker.rb +6 -6
- data/lib/karafka/processing/jobs/base.rb +6 -4
- data/lib/karafka/processing/jobs_queue.rb +10 -0
- data/lib/karafka/processing/schedulers/default.rb +4 -4
- data/lib/karafka/processing/strategies/base.rb +6 -6
- data/lib/karafka/processing/strategies/default.rb +13 -13
- data/lib/karafka/processing/strategies/dlq.rb +1 -1
- data/lib/karafka/processing/worker.rb +5 -5
- data/lib/karafka/railtie.rb +11 -11
- data/lib/karafka/routing/builder.rb +3 -3
- data/lib/karafka/routing/contracts/consumer_group.rb +6 -6
- data/lib/karafka/routing/contracts/routing.rb +2 -2
- data/lib/karafka/routing/contracts/topic.rb +4 -4
- data/lib/karafka/routing/features/active_job/contracts/topic.rb +3 -3
- data/lib/karafka/routing/features/base/expander.rb +4 -4
- data/lib/karafka/routing/features/base.rb +8 -8
- data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +2 -2
- data/lib/karafka/routing/features/declaratives/contracts/topic.rb +2 -2
- data/lib/karafka/routing/features/deserializers/contracts/topic.rb +2 -2
- data/lib/karafka/routing/features/eofed/contracts/topic.rb +3 -3
- data/lib/karafka/routing/features/inline_insights/contracts/topic.rb +2 -2
- data/lib/karafka/routing/features/inline_insights.rb +7 -7
- data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb +2 -2
- data/lib/karafka/routing/subscription_group.rb +9 -9
- data/lib/karafka/runner.rb +3 -3
- data/lib/karafka/server.rb +14 -5
- data/lib/karafka/setup/attributes_map.rb +7 -7
- data/lib/karafka/setup/config.rb +11 -11
- data/lib/karafka/setup/contracts/config.rb +2 -2
- data/lib/karafka/setup/defaults_injector.rb +11 -11
- data/lib/karafka/swarm/manager.rb +6 -6
- data/lib/karafka/swarm/node.rb +8 -37
- data/lib/karafka/swarm/producer_replacer.rb +110 -0
- data/lib/karafka/swarm/supervisor.rb +9 -6
- data/lib/karafka/swarm.rb +1 -1
- data/lib/karafka/time_trackers/pause.rb +1 -1
- data/lib/karafka/version.rb +1 -1
- data/lib/karafka.rb +36 -36
- metadata +7 -3
|
@@ -143,8 +143,8 @@ module Karafka
|
|
|
143
143
|
# are always considered immediate as they indicate, that a message with a given key
|
|
144
144
|
# was already dispatched or that user decided not to dispatch and cancelled the dispatch
|
|
145
145
|
# via tombstone publishing.
|
|
146
|
-
if message.headers[
|
|
147
|
-
time = message.headers[
|
|
146
|
+
if message.headers["schedule_source_type"] == "schedule"
|
|
147
|
+
time = message.headers["schedule_target_epoch"]
|
|
148
148
|
|
|
149
149
|
# Do not track historical below today as those will be reflected in the daily buffer
|
|
150
150
|
@tracker.future(message) if time >= @today.starts_at
|
|
@@ -162,8 +162,8 @@ module Karafka
|
|
|
162
162
|
# that we've got that far in the dispatching time. This allows us (with a certain buffer)
|
|
163
163
|
# to quickly reject older messages (older in sense of being scheduled for previous times)
|
|
164
164
|
# instead of loading them into memory until they are expired
|
|
165
|
-
if message.headers[
|
|
166
|
-
@max_epoch.update(message.headers[
|
|
165
|
+
if message.headers["schedule_source_type"] == "tombstone"
|
|
166
|
+
@max_epoch.update(message.headers["schedule_target_epoch"])
|
|
167
167
|
end
|
|
168
168
|
|
|
169
169
|
# Add to buffer all tombstones and messages for the same day
|
|
@@ -29,8 +29,8 @@ module Karafka
|
|
|
29
29
|
class Config < Karafka::Contracts::Base
|
|
30
30
|
configure do |config|
|
|
31
31
|
config.error_messages = YAML.safe_load_file(
|
|
32
|
-
File.join(Karafka.gem_root,
|
|
33
|
-
).fetch(
|
|
32
|
+
File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
|
|
33
|
+
).fetch("en").fetch("validations").fetch("setup").fetch("config")
|
|
34
34
|
end
|
|
35
35
|
|
|
36
36
|
nested(:scheduled_messages) do
|
|
@@ -31,8 +31,8 @@ module Karafka
|
|
|
31
31
|
class Message < Karafka::Contracts::Base
|
|
32
32
|
configure do |config|
|
|
33
33
|
config.error_messages = YAML.safe_load_file(
|
|
34
|
-
File.join(Karafka.gem_root,
|
|
35
|
-
).fetch(
|
|
34
|
+
File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
|
|
35
|
+
).fetch("en").fetch("validations").fetch("scheduled_messages_message")
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
# Headers we expect in each message of type "schedule" that goes to our scheduled
|
|
@@ -56,7 +56,7 @@ module Karafka
|
|
|
56
56
|
required(:headers) do |val|
|
|
57
57
|
next false unless val.is_a?(Hash)
|
|
58
58
|
|
|
59
|
-
if val[
|
|
59
|
+
if val["schedule_source_type"] == "message"
|
|
60
60
|
(val.keys & EXPECTED_SCHEDULE_HEADERS).size >= 4
|
|
61
61
|
else
|
|
62
62
|
(val.keys & EXPECTED_CANCEL_HEADERS).size >= 2
|
|
@@ -70,9 +70,9 @@ module Karafka
|
|
|
70
70
|
next unless errors.empty?
|
|
71
71
|
|
|
72
72
|
# Validate epoch only for schedules
|
|
73
|
-
next unless data[:headers][
|
|
73
|
+
next unless data[:headers]["schedule_source_type"] == "schedule"
|
|
74
74
|
|
|
75
|
-
epoch_time = data[:headers].fetch(
|
|
75
|
+
epoch_time = data[:headers].fetch("schedule_target_epoch").to_i
|
|
76
76
|
|
|
77
77
|
# We allow for small lag as those will be dispatched but we should prevent dispatching
|
|
78
78
|
# in the past in general as often it is a source of errors
|
|
@@ -86,11 +86,11 @@ module Karafka
|
|
|
86
86
|
next unless errors.empty?
|
|
87
87
|
|
|
88
88
|
scheduled_topics = Karafka::App
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
89
|
+
.routes
|
|
90
|
+
.flat_map(&:topics)
|
|
91
|
+
.flat_map(&:to_a)
|
|
92
|
+
.select(&:scheduled_messages?)
|
|
93
|
+
.map(&:name)
|
|
94
94
|
|
|
95
95
|
next if scheduled_topics.include?(data[:topic].to_s)
|
|
96
96
|
|
|
@@ -49,12 +49,12 @@ module Karafka
|
|
|
49
49
|
# @note Only messages for a given day should be added here.
|
|
50
50
|
def <<(message)
|
|
51
51
|
# Non schedule are only tombstones and cancellations
|
|
52
|
-
schedule = message.headers[
|
|
52
|
+
schedule = message.headers["schedule_source_type"] == "schedule"
|
|
53
53
|
|
|
54
54
|
key = message.key
|
|
55
55
|
|
|
56
56
|
if schedule
|
|
57
|
-
epoch = message.headers[
|
|
57
|
+
epoch = message.headers["schedule_target_epoch"]
|
|
58
58
|
@accu[key] = [epoch, message]
|
|
59
59
|
else
|
|
60
60
|
@accu.delete(key)
|
|
@@ -38,18 +38,18 @@ module Karafka
|
|
|
38
38
|
def call(metadata)
|
|
39
39
|
raw_headers = metadata.raw_headers
|
|
40
40
|
|
|
41
|
-
type = raw_headers.fetch(
|
|
41
|
+
type = raw_headers.fetch("schedule_source_type")
|
|
42
42
|
|
|
43
43
|
# tombstone and cancellation events are not operable, thus we do not have to cast any
|
|
44
44
|
# of the headers pieces
|
|
45
45
|
return raw_headers unless WORKABLE_TYPES.include?(type)
|
|
46
46
|
|
|
47
47
|
headers = raw_headers.dup
|
|
48
|
-
headers[
|
|
48
|
+
headers["schedule_target_epoch"] = headers["schedule_target_epoch"].to_i
|
|
49
49
|
|
|
50
50
|
# This attribute is optional, this is why we have to check for its existence
|
|
51
|
-
if headers.key?(
|
|
52
|
-
headers[
|
|
51
|
+
if headers.key?("schedule_target_partition")
|
|
52
|
+
headers["schedule_target_partition"] = headers["schedule_target_partition"].to_i
|
|
53
53
|
end
|
|
54
54
|
|
|
55
55
|
headers
|
|
@@ -54,10 +54,10 @@ module Karafka
|
|
|
54
54
|
# @note It also produces needed tombstone event as well as an audit log message
|
|
55
55
|
def <<(message)
|
|
56
56
|
target_headers = message.raw_headers.merge(
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
57
|
+
"schedule_source_topic" => @topic,
|
|
58
|
+
"schedule_source_partition" => @partition.to_s,
|
|
59
|
+
"schedule_source_offset" => message.offset.to_s,
|
|
60
|
+
"schedule_source_key" => message.key
|
|
61
61
|
).compact
|
|
62
62
|
|
|
63
63
|
target = {
|
|
@@ -90,7 +90,7 @@ module Karafka
|
|
|
90
90
|
# We use the state as a key, so we always have one state transition data available
|
|
91
91
|
key: "#{tracker.state}_state",
|
|
92
92
|
partition: @partition,
|
|
93
|
-
headers: {
|
|
93
|
+
headers: { "zlib" => "true" }
|
|
94
94
|
)
|
|
95
95
|
end
|
|
96
96
|
|
|
@@ -84,9 +84,9 @@ module Karafka
|
|
|
84
84
|
)
|
|
85
85
|
|
|
86
86
|
headers = (message[:headers] || {}).merge(
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
87
|
+
"schedule_schema_version" => ScheduledMessages::SCHEMA_VERSION,
|
|
88
|
+
"schedule_target_epoch" => epoch.to_i.to_s,
|
|
89
|
+
"schedule_source_type" => "schedule"
|
|
90
90
|
)
|
|
91
91
|
|
|
92
92
|
export(headers, message, :topic)
|
|
@@ -118,8 +118,8 @@ module Karafka
|
|
|
118
118
|
key: key,
|
|
119
119
|
payload: nil,
|
|
120
120
|
headers: {
|
|
121
|
-
|
|
122
|
-
|
|
121
|
+
"schedule_schema_version" => ScheduledMessages::SCHEMA_VERSION,
|
|
122
|
+
"schedule_source_type" => "cancel"
|
|
123
123
|
}
|
|
124
124
|
}.merge(envelope)
|
|
125
125
|
|
|
@@ -140,9 +140,9 @@ module Karafka
|
|
|
140
140
|
topic: message.topic,
|
|
141
141
|
partition: message.partition,
|
|
142
142
|
headers: message.raw_headers.merge(
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
143
|
+
"schedule_schema_version" => ScheduledMessages::SCHEMA_VERSION,
|
|
144
|
+
"schedule_source_type" => "tombstone",
|
|
145
|
+
"schedule_source_offset" => message.offset.to_s
|
|
146
146
|
)
|
|
147
147
|
}
|
|
148
148
|
end
|
|
@@ -33,7 +33,7 @@ module Karafka
|
|
|
33
33
|
#
|
|
34
34
|
# @param message [Karafka::Messages::Message]
|
|
35
35
|
def call(message)
|
|
36
|
-
message_version = message.headers[
|
|
36
|
+
message_version = message.headers["schedule_schema_version"]
|
|
37
37
|
|
|
38
38
|
return if message_version <= ScheduledMessages::SCHEMA_VERSION
|
|
39
39
|
|
|
@@ -30,7 +30,7 @@ module Karafka
|
|
|
30
30
|
extend Karafka::Core::Configurable
|
|
31
31
|
|
|
32
32
|
setting(:consumer_class, default: Consumer)
|
|
33
|
-
setting(:group_id, default:
|
|
33
|
+
setting(:group_id, default: "karafka_scheduled_messages")
|
|
34
34
|
|
|
35
35
|
# By default we will run the scheduling every 15 seconds since we provide a minute-based
|
|
36
36
|
# precision. Can be increased when having dedicated processes to run this. Lower values
|
|
@@ -52,7 +52,7 @@ module Karafka
|
|
|
52
52
|
setting(:dispatcher_class, default: Dispatcher)
|
|
53
53
|
|
|
54
54
|
# Postfix for the states topic to build the states based on the group name + postfix
|
|
55
|
-
setting(:states_postfix, default:
|
|
55
|
+
setting(:states_postfix, default: "_states")
|
|
56
56
|
|
|
57
57
|
setting(:deserializers) do
|
|
58
58
|
# Deserializer for schedules messages to convert epochs
|
|
@@ -40,7 +40,7 @@ module Karafka
|
|
|
40
40
|
@daily = Hash.new { |h, k| h[k] = 0 }
|
|
41
41
|
@started_at = Time.now.to_i
|
|
42
42
|
@offsets = { low: -1, high: -1 }
|
|
43
|
-
@state =
|
|
43
|
+
@state = "fresh"
|
|
44
44
|
@reloads = 0
|
|
45
45
|
end
|
|
46
46
|
|
|
@@ -71,7 +71,7 @@ module Karafka
|
|
|
71
71
|
# drastically increase complexity. For given day we use the accurate counter and for
|
|
72
72
|
# future days we use estimates.
|
|
73
73
|
def future(message)
|
|
74
|
-
epoch = message.headers[
|
|
74
|
+
epoch = message.headers["schedule_target_epoch"]
|
|
75
75
|
|
|
76
76
|
@daily[epoch_to_date(epoch)] += 1
|
|
77
77
|
end
|
|
@@ -33,11 +33,11 @@ module Karafka
|
|
|
33
33
|
# Version of the schema we use for envelops in scheduled messages.
|
|
34
34
|
# We use it to detect any potential upgrades similar to other components of Karafka and to
|
|
35
35
|
# stop processing of incompatible versions
|
|
36
|
-
SCHEMA_VERSION =
|
|
36
|
+
SCHEMA_VERSION = "1.0.0"
|
|
37
37
|
|
|
38
38
|
# Version of the states schema. Used to publish per partition simple aggregated metrics
|
|
39
39
|
# that can be used for schedules reporting
|
|
40
|
-
STATES_SCHEMA_VERSION =
|
|
40
|
+
STATES_SCHEMA_VERSION = "1.0.0"
|
|
41
41
|
|
|
42
42
|
class << self
|
|
43
43
|
# Runs the `Proxy.call`
|
|
@@ -181,7 +181,7 @@ module Karafka
|
|
|
181
181
|
|
|
182
182
|
# @return [Integer] RSS in MB for the current process
|
|
183
183
|
def rss_mb
|
|
184
|
-
RUBY_PLATFORM.include?(
|
|
184
|
+
RUBY_PLATFORM.include?("linux") ? rss_mb_linux : rss_mb_macos
|
|
185
185
|
end
|
|
186
186
|
|
|
187
187
|
# @return [Integer] RSS in MB for the current process on Linux
|
|
@@ -189,7 +189,7 @@ module Karafka
|
|
|
189
189
|
kb_rss = 0
|
|
190
190
|
|
|
191
191
|
File.readlines("/proc/#{node.pid}/status").each do |line|
|
|
192
|
-
next unless line.start_with?(
|
|
192
|
+
next unless line.start_with?("VmRSS:")
|
|
193
193
|
|
|
194
194
|
kb_rss = line.split[1].to_i
|
|
195
195
|
break
|
data/lib/karafka/process.rb
CHANGED
|
@@ -98,7 +98,7 @@ module Karafka
|
|
|
98
98
|
# Informs monitoring about trapped signal
|
|
99
99
|
# @param signal [Symbol] signal type that we received
|
|
100
100
|
def notice_signal(signal)
|
|
101
|
-
Karafka.monitor.instrument(
|
|
101
|
+
Karafka.monitor.instrument("process.notice_signal", caller: self, signal: signal)
|
|
102
102
|
end
|
|
103
103
|
end
|
|
104
104
|
end
|
|
@@ -100,7 +100,7 @@ module Karafka
|
|
|
100
100
|
|
|
101
101
|
# This should never happen. If it does, something is heavily out of sync. Please reach
|
|
102
102
|
# out to us if you encounter this
|
|
103
|
-
raise Karafka::Errors::InvalidCoordinatorStateError,
|
|
103
|
+
raise Karafka::Errors::InvalidCoordinatorStateError, "Was zero before decrementation"
|
|
104
104
|
end
|
|
105
105
|
end
|
|
106
106
|
|
|
@@ -33,10 +33,10 @@ module Karafka
|
|
|
33
33
|
!insights.empty?
|
|
34
34
|
end
|
|
35
35
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
36
|
+
alias_method :statistics, :insights
|
|
37
|
+
alias_method :statistics?, :insights?
|
|
38
|
+
alias_method :inline_insights, :insights
|
|
39
|
+
alias_method :inline_insights?, :insights?
|
|
40
40
|
end
|
|
41
41
|
end
|
|
42
42
|
end
|
|
@@ -68,8 +68,8 @@ module Karafka
|
|
|
68
68
|
# @param statistics [Hash] librdkafka enriched statistics
|
|
69
69
|
def add(consumer_group_id, statistics)
|
|
70
70
|
@mutex.synchronize do
|
|
71
|
-
statistics.fetch(
|
|
72
|
-
t_details.fetch(
|
|
71
|
+
statistics.fetch("topics", EMPTY_HASH).each do |topic_name, t_details|
|
|
72
|
+
t_details.fetch("partitions", EMPTY_HASH).each do |partition_id, p_details|
|
|
73
73
|
next unless track?(partition_id, p_details)
|
|
74
74
|
|
|
75
75
|
key = "#{consumer_group_id}_#{topic_name}_#{partition_id}"
|
|
@@ -114,12 +114,12 @@ module Karafka
|
|
|
114
114
|
# @param p_details [Hash] partition statistics details
|
|
115
115
|
# @return [Boolean] true if we should track given partition
|
|
116
116
|
def track?(partition_id, p_details)
|
|
117
|
-
return false if partition_id ==
|
|
117
|
+
return false if partition_id == "-1"
|
|
118
118
|
|
|
119
|
-
fetch_state = p_details.fetch(
|
|
119
|
+
fetch_state = p_details.fetch("fetch_state")
|
|
120
120
|
|
|
121
|
-
return false if fetch_state ==
|
|
122
|
-
return false if fetch_state ==
|
|
121
|
+
return false if fetch_state == "stopped"
|
|
122
|
+
return false if fetch_state == "none"
|
|
123
123
|
|
|
124
124
|
true
|
|
125
125
|
end
|
|
@@ -38,19 +38,21 @@ module Karafka
|
|
|
38
38
|
# When redefined can run any code prior to the job being scheduled
|
|
39
39
|
# @note This will run in the listener thread and not in the worker
|
|
40
40
|
def before_schedule
|
|
41
|
-
raise NotImplementedError,
|
|
41
|
+
raise NotImplementedError, "Please implement in a subclass"
|
|
42
42
|
end
|
|
43
43
|
|
|
44
44
|
# When redefined can run any code that should run before executing the proper code
|
|
45
|
-
def before_call
|
|
45
|
+
def before_call
|
|
46
|
+
end
|
|
46
47
|
|
|
47
48
|
# The main entry-point of a job
|
|
48
49
|
def call
|
|
49
|
-
raise NotImplementedError,
|
|
50
|
+
raise NotImplementedError, "Please implement in a subclass"
|
|
50
51
|
end
|
|
51
52
|
|
|
52
53
|
# When redefined can run any code that should run after executing the proper code
|
|
53
|
-
def after_call
|
|
54
|
+
def after_call
|
|
55
|
+
end
|
|
54
56
|
|
|
55
57
|
# @return [Boolean] is this a non-blocking job
|
|
56
58
|
#
|
|
@@ -180,6 +180,16 @@ module Karafka
|
|
|
180
180
|
end
|
|
181
181
|
end
|
|
182
182
|
|
|
183
|
+
# Returns a snapshot of all jobs currently in processing per group.
|
|
184
|
+
# Useful for diagnostics during forceful shutdown to understand what is blocking.
|
|
185
|
+
#
|
|
186
|
+
# @return [Hash{String => Array<Jobs::Base>}] hash mapping group ids to arrays of jobs
|
|
187
|
+
def in_processing
|
|
188
|
+
@mutex.synchronize do
|
|
189
|
+
@in_processing.transform_values(&:dup).freeze
|
|
190
|
+
end
|
|
191
|
+
end
|
|
192
|
+
|
|
183
193
|
private
|
|
184
194
|
|
|
185
195
|
# @param group_id [String] id of the group in which jobs we're interested.
|
|
@@ -21,10 +21,10 @@ module Karafka
|
|
|
21
21
|
end
|
|
22
22
|
|
|
23
23
|
# Revocation, shutdown and idle jobs can also run in fifo by default
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
24
|
+
alias_method :on_schedule_revocation, :on_schedule_consumption
|
|
25
|
+
alias_method :on_schedule_shutdown, :on_schedule_consumption
|
|
26
|
+
alias_method :on_schedule_idle, :on_schedule_consumption
|
|
27
|
+
alias_method :on_schedule_eofed, :on_schedule_consumption
|
|
28
28
|
|
|
29
29
|
# This scheduler does not have anything to manage as it is a pass through and has no state
|
|
30
30
|
def on_manage
|
|
@@ -28,32 +28,32 @@ module Karafka
|
|
|
28
28
|
|
|
29
29
|
# What should happen before we kick in the processing
|
|
30
30
|
def handle_before_consume
|
|
31
|
-
raise NotImplementedError,
|
|
31
|
+
raise NotImplementedError, "Implement in a subclass"
|
|
32
32
|
end
|
|
33
33
|
|
|
34
34
|
# What should happen in the processing
|
|
35
35
|
def handle_consume
|
|
36
|
-
raise NotImplementedError,
|
|
36
|
+
raise NotImplementedError, "Implement in a subclass"
|
|
37
37
|
end
|
|
38
38
|
|
|
39
39
|
# Post-consumption handling
|
|
40
40
|
def handle_after_consume
|
|
41
|
-
raise NotImplementedError,
|
|
41
|
+
raise NotImplementedError, "Implement in a subclass"
|
|
42
42
|
end
|
|
43
43
|
|
|
44
44
|
# Idle run handling
|
|
45
45
|
def handle_idle
|
|
46
|
-
raise NotImplementedError,
|
|
46
|
+
raise NotImplementedError, "Implement in a subclass"
|
|
47
47
|
end
|
|
48
48
|
|
|
49
49
|
# Revocation handling
|
|
50
50
|
def handle_revoked
|
|
51
|
-
raise NotImplementedError,
|
|
51
|
+
raise NotImplementedError, "Implement in a subclass"
|
|
52
52
|
end
|
|
53
53
|
|
|
54
54
|
# Shutdown handling
|
|
55
55
|
def handle_shutdown
|
|
56
|
-
raise NotImplementedError,
|
|
56
|
+
raise NotImplementedError, "Implement in a subclass"
|
|
57
57
|
end
|
|
58
58
|
end
|
|
59
59
|
end
|
|
@@ -35,8 +35,8 @@ module Karafka
|
|
|
35
35
|
# @note It runs in the listener loop. Should **not** be used for anything heavy or
|
|
36
36
|
# with any potential errors. Mostly for initialization of states, etc.
|
|
37
37
|
def handle_initialized
|
|
38
|
-
monitor.instrument(
|
|
39
|
-
monitor.instrument(
|
|
38
|
+
monitor.instrument("consumer.initialize", caller: self)
|
|
39
|
+
monitor.instrument("consumer.initialized", caller: self) do
|
|
40
40
|
initialized
|
|
41
41
|
end
|
|
42
42
|
end
|
|
@@ -120,22 +120,22 @@ module Karafka
|
|
|
120
120
|
# @param action [Symbol]
|
|
121
121
|
# @param block [Proc]
|
|
122
122
|
def handle_wrap(action, &block)
|
|
123
|
-
monitor.instrument(
|
|
124
|
-
monitor.instrument(
|
|
123
|
+
monitor.instrument("consumer.wrap", caller: self)
|
|
124
|
+
monitor.instrument("consumer.wrapped", caller: self) do
|
|
125
125
|
wrap(action, &block)
|
|
126
126
|
end
|
|
127
127
|
end
|
|
128
128
|
|
|
129
129
|
# Run the user consumption code
|
|
130
130
|
def handle_consume
|
|
131
|
-
monitor.instrument(
|
|
132
|
-
monitor.instrument(
|
|
131
|
+
monitor.instrument("consumer.consume", caller: self)
|
|
132
|
+
monitor.instrument("consumer.consumed", caller: self) do
|
|
133
133
|
consume
|
|
134
134
|
end
|
|
135
135
|
|
|
136
136
|
# Mark job as successful
|
|
137
137
|
coordinator.success!(self)
|
|
138
|
-
rescue
|
|
138
|
+
rescue => e
|
|
139
139
|
coordinator.failure!(self, e)
|
|
140
140
|
|
|
141
141
|
# Re-raise so reported in the consumer
|
|
@@ -175,8 +175,8 @@ module Karafka
|
|
|
175
175
|
|
|
176
176
|
# Runs the consumer `#eofed` method with reporting
|
|
177
177
|
def handle_eofed
|
|
178
|
-
monitor.instrument(
|
|
179
|
-
monitor.instrument(
|
|
178
|
+
monitor.instrument("consumer.eof", caller: self)
|
|
179
|
+
monitor.instrument("consumer.eofed", caller: self) do
|
|
180
180
|
eofed
|
|
181
181
|
end
|
|
182
182
|
ensure
|
|
@@ -191,8 +191,8 @@ module Karafka
|
|
|
191
191
|
|
|
192
192
|
coordinator.revoke
|
|
193
193
|
|
|
194
|
-
monitor.instrument(
|
|
195
|
-
monitor.instrument(
|
|
194
|
+
monitor.instrument("consumer.revoke", caller: self)
|
|
195
|
+
monitor.instrument("consumer.revoked", caller: self) do
|
|
196
196
|
revoked
|
|
197
197
|
end
|
|
198
198
|
ensure
|
|
@@ -201,8 +201,8 @@ module Karafka
|
|
|
201
201
|
|
|
202
202
|
# Runs the shutdown code
|
|
203
203
|
def handle_shutdown
|
|
204
|
-
monitor.instrument(
|
|
205
|
-
monitor.instrument(
|
|
204
|
+
monitor.instrument("consumer.shutting_down", caller: self)
|
|
205
|
+
monitor.instrument("consumer.shutdown", caller: self) do
|
|
206
206
|
shutdown
|
|
207
207
|
end
|
|
208
208
|
ensure
|
|
@@ -55,9 +55,9 @@ module Karafka
|
|
|
55
55
|
|
|
56
56
|
if job
|
|
57
57
|
job.wrap do
|
|
58
|
-
monitor.instrument(
|
|
58
|
+
monitor.instrument("worker.process", instrument_details)
|
|
59
59
|
|
|
60
|
-
monitor.instrument(
|
|
60
|
+
monitor.instrument("worker.processed", instrument_details) do
|
|
61
61
|
job.before_call
|
|
62
62
|
|
|
63
63
|
# If a job is marked as non blocking, we can run a tick in the job queue and if there
|
|
@@ -87,12 +87,12 @@ module Karafka
|
|
|
87
87
|
rescue Exception => e
|
|
88
88
|
# rubocop:enable Lint/RescueException
|
|
89
89
|
monitor.instrument(
|
|
90
|
-
|
|
90
|
+
"error.occurred",
|
|
91
91
|
caller: self,
|
|
92
92
|
job: job,
|
|
93
93
|
jobs_queue: @jobs_queue,
|
|
94
94
|
error: e,
|
|
95
|
-
type:
|
|
95
|
+
type: "worker.process.error"
|
|
96
96
|
)
|
|
97
97
|
ensure
|
|
98
98
|
# job can be nil when the queue is being closed
|
|
@@ -102,7 +102,7 @@ module Karafka
|
|
|
102
102
|
end
|
|
103
103
|
|
|
104
104
|
# Always publish info, that we completed all the work despite its result
|
|
105
|
-
monitor.instrument(
|
|
105
|
+
monitor.instrument("worker.completed", instrument_details)
|
|
106
106
|
end
|
|
107
107
|
end
|
|
108
108
|
end
|
data/lib/karafka/railtie.rb
CHANGED
|
@@ -12,17 +12,17 @@ end
|
|
|
12
12
|
|
|
13
13
|
if Karafka.rails?
|
|
14
14
|
# Load ActiveJob adapter
|
|
15
|
-
require
|
|
15
|
+
require "active_job/karafka"
|
|
16
16
|
|
|
17
17
|
# Setup env if configured (may be configured later by .net, etc)
|
|
18
|
-
ENV[
|
|
18
|
+
ENV["KARAFKA_ENV"] ||= ENV["RAILS_ENV"] if ENV.key?("RAILS_ENV")
|
|
19
19
|
|
|
20
20
|
module Karafka
|
|
21
21
|
# Railtie for setting up Rails integration
|
|
22
22
|
class Railtie < Rails::Railtie
|
|
23
23
|
railtie_name :karafka
|
|
24
24
|
|
|
25
|
-
initializer
|
|
25
|
+
initializer "karafka.active_job_integration" do
|
|
26
26
|
ActiveSupport.on_load(:active_job) do
|
|
27
27
|
# Extend ActiveJob with some Karafka specific ActiveJob magic
|
|
28
28
|
extend ::Karafka::ActiveJob::JobExtensions
|
|
@@ -32,12 +32,12 @@ if Karafka.rails?
|
|
|
32
32
|
# This lines will make Karafka print to stdout like puma or unicorn when we run karafka
|
|
33
33
|
# server + will support code reloading with each fetched loop. We do it only for karafka
|
|
34
34
|
# based commands as Rails processes and console will have it enabled already
|
|
35
|
-
initializer
|
|
35
|
+
initializer "karafka.configure_rails_logger" do
|
|
36
36
|
# Make Karafka uses Rails logger
|
|
37
37
|
::Karafka::App.config.logger = Rails.logger
|
|
38
38
|
|
|
39
39
|
next unless Rails.env.development?
|
|
40
|
-
next unless ENV.key?(
|
|
40
|
+
next unless ENV.key?("KARAFKA_CLI")
|
|
41
41
|
# If we are already publishing to STDOUT, no need to add it again.
|
|
42
42
|
# If added again, would print stuff twice
|
|
43
43
|
next if ActiveSupport::Logger.logger_outputs_to?(Rails.logger, $stdout)
|
|
@@ -46,7 +46,7 @@ if Karafka.rails?
|
|
|
46
46
|
# Inherit the logger level from Rails, otherwise would always run with the debug level
|
|
47
47
|
stdout_logger.level = Rails.logger.level
|
|
48
48
|
|
|
49
|
-
rails71plus = Rails.gem_version >= Gem::Version.new(
|
|
49
|
+
rails71plus = Rails.gem_version >= Gem::Version.new("7.1.0")
|
|
50
50
|
|
|
51
51
|
# Rails 7.1 replaced the broadcast module with a broadcast logger
|
|
52
52
|
# While 7.1 is EOL, we keep this for users who may still use it without official support
|
|
@@ -61,20 +61,20 @@ if Karafka.rails?
|
|
|
61
61
|
end
|
|
62
62
|
end
|
|
63
63
|
|
|
64
|
-
initializer
|
|
64
|
+
initializer "karafka.configure_rails_auto_load_paths" do |app|
|
|
65
65
|
# Consumers should autoload by default in the Rails app so they are visible
|
|
66
66
|
app.config.autoload_paths += %w[app/consumers]
|
|
67
67
|
end
|
|
68
68
|
|
|
69
|
-
initializer
|
|
70
|
-
rails6plus = Rails.gem_version >= Gem::Version.new(
|
|
69
|
+
initializer "karafka.require_karafka_boot_file" do |app|
|
|
70
|
+
rails6plus = Rails.gem_version >= Gem::Version.new("6.0.0")
|
|
71
71
|
|
|
72
72
|
# If the boot file location is set to "false", we should not raise an exception and we
|
|
73
73
|
# should just not load karafka stuff. Setting this explicitly to false indicates, that
|
|
74
74
|
# karafka is part of the supply chain but it is not a first class citizen of a given
|
|
75
75
|
# system (may be just a dependency of a dependency), thus railtie should not kick in to
|
|
76
76
|
# load the non-existing boot file
|
|
77
|
-
next if Karafka.boot_file.to_s ==
|
|
77
|
+
next if Karafka.boot_file.to_s == "false"
|
|
78
78
|
|
|
79
79
|
karafka_boot_file = Rails.root.join(Karafka.boot_file.to_s).to_s
|
|
80
80
|
|
|
@@ -96,7 +96,7 @@ if Karafka.rails?
|
|
|
96
96
|
end
|
|
97
97
|
end
|
|
98
98
|
|
|
99
|
-
initializer
|
|
99
|
+
initializer "karafka.configure_worker_external_executor" do |app|
|
|
100
100
|
app.config.after_initialize do
|
|
101
101
|
app_config = Karafka::App.config
|
|
102
102
|
|