karafka 2.3.1 → 2.3.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.rspec +2 -0
- data/CHANGELOG.md +12 -0
- data/Gemfile.lock +6 -6
- data/bin/integrations +2 -1
- data/bin/rspecs +6 -2
- data/config/locales/errors.yml +30 -8
- data/config/locales/pro_errors.yml +2 -0
- data/docker-compose.yml +1 -1
- data/lib/karafka/app.rb +14 -0
- data/lib/karafka/cli/base.rb +19 -0
- data/lib/karafka/cli/server.rb +62 -76
- data/lib/karafka/cli/swarm.rb +30 -0
- data/lib/karafka/constraints.rb +3 -3
- data/lib/karafka/contracts/config.rb +19 -0
- data/lib/karafka/errors.rb +12 -0
- data/lib/karafka/helpers/config_importer.rb +30 -0
- data/lib/karafka/instrumentation/logger_listener.rb +31 -0
- data/lib/karafka/instrumentation/notifications.rb +9 -0
- data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +2 -0
- data/lib/karafka/instrumentation/vendors/kubernetes/base_listener.rb +72 -0
- data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +11 -40
- data/lib/karafka/instrumentation/vendors/kubernetes/swarm_liveness_listener.rb +54 -0
- data/lib/karafka/pro/active_job/job_options_contract.rb +1 -1
- data/lib/karafka/pro/base_consumer.rb +16 -0
- data/lib/karafka/pro/connection/manager.rb +6 -1
- data/lib/karafka/pro/processing/coordinator.rb +13 -3
- data/lib/karafka/pro/processing/coordinators/errors_tracker.rb +74 -0
- data/lib/karafka/pro/processing/coordinators/filters_applier.rb +107 -0
- data/lib/karafka/pro/processing/coordinators/virtual_offset_manager.rb +180 -0
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb +5 -7
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb +5 -7
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom.rb +8 -10
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom_vp.rb +8 -16
- data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom.rb +5 -7
- data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom_vp.rb +5 -7
- data/lib/karafka/pro/processing/strategies/aj/dlq_mom.rb +8 -10
- data/lib/karafka/pro/processing/strategies/aj/dlq_mom_vp.rb +7 -9
- data/lib/karafka/pro/processing/strategies/dlq/default.rb +36 -10
- data/lib/karafka/pro/processing/strategies/dlq/ftr.rb +3 -7
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb +4 -8
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb +6 -9
- data/lib/karafka/pro/processing/strategies/dlq/ftr_mom.rb +5 -15
- data/lib/karafka/pro/processing/strategies/dlq/lrj.rb +4 -8
- data/lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb +6 -9
- data/lib/karafka/pro/processing/strategies/dlq/mom.rb +10 -20
- data/lib/karafka/pro/processing/strategies/vp/default.rb +7 -0
- data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +6 -0
- data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb +39 -0
- data/lib/karafka/pro/swarm/liveness_listener.rb +171 -0
- data/lib/karafka/process.rb +27 -1
- data/lib/karafka/routing/features/dead_letter_queue/config.rb +2 -0
- data/lib/karafka/routing/subscription_group.rb +31 -9
- data/lib/karafka/server.rb +11 -13
- data/lib/karafka/setup/config.rb +41 -2
- data/lib/karafka/status.rb +4 -2
- data/lib/karafka/swarm/liveness_listener.rb +55 -0
- data/lib/karafka/swarm/manager.rb +217 -0
- data/lib/karafka/swarm/node.rb +179 -0
- data/lib/karafka/swarm/pidfd.rb +131 -0
- data/lib/karafka/swarm/supervisor.rb +184 -0
- data/lib/karafka/swarm.rb +27 -0
- data/lib/karafka/version.rb +1 -1
- data/lib/karafka.rb +1 -1
- data.tar.gz.sig +0 -0
- metadata +17 -4
- metadata.gz.sig +0 -0
- data/lib/karafka/pro/processing/filters_applier.rb +0 -105
- data/lib/karafka/pro/processing/virtual_offset_manager.rb +0 -177
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: cb47082224d857f3029f9bb8e1b04a35e6b8ed2f7ae75bbe52bf1b778ff56226
|
4
|
+
data.tar.gz: 53d59fd7e140f5b3e9b89dd3e4af28469bc534074110e2e93fae24c59bf81b88
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 00e09a345122ad2facaf8adcbb52fae3ce87374083d9e6785a1f07a74c87e53c6b0b3dd32b82566e846d160569aafc55c614013d8c9f95664612150fb51d07b1
|
7
|
+
data.tar.gz: 283e50a6b3b25579419bdc9b947e5ba802e22a1cd6d0097ab8929c5394d3858461bda15c3e70a0d2d466a8705466ef1d5b24e1a8bfe5ac8e356c64790049c7b0
|
checksums.yaml.gz.sig
CHANGED
Binary file
|
data/.rspec
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,17 @@
|
|
1
1
|
# Karafka framework changelog
|
2
2
|
|
3
|
+
## 2.3.2 (2024-02-16)
|
4
|
+
- **[Feature]** Provide swarm capabilities to OSS and Pro.
|
5
|
+
- **[Feature]** Provide ability to use complex strategies in DLQ (Pro).
|
6
|
+
- [Enhancement] Support using `:partition` as the partition key for ActiveJob assignments.
|
7
|
+
- [Enhancement] Expand Logger listener with swarm notifications.
|
8
|
+
- [Enhancement] Introduce K8s swarm liveness listener.
|
9
|
+
- [Enhancement] Use `Process.warmup` in Ruby 3.3+ prior to forks (in swarm) and prior to app start.
|
10
|
+
- [Enhancement] Provide `app.before_warmup` event to allow hooking code loading tools prior to final warmup.
|
11
|
+
- [Enhancement] Provide `Consumer#errors_tracker` to be able to get errors that occurred while doing complex recovery.
|
12
|
+
- [Fix] Infinite consecutive error flow with VPs and without DLQ can cause endless offsets accumulation.
|
13
|
+
- [Fix] Quieting mode causes too early unsubscribe.
|
14
|
+
|
3
15
|
## 2.3.1 (2024-02-08)
|
4
16
|
- [Refactor] Ensure that `Karafka::Helpers::Async#async_call` can run from multiple threads.
|
5
17
|
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
karafka (2.3.
|
4
|
+
karafka (2.3.2)
|
5
5
|
karafka-core (>= 2.3.0, < 2.4.0)
|
6
6
|
waterdrop (>= 2.6.12, < 3.0.0)
|
7
7
|
zeitwerk (~> 2.3)
|
@@ -41,11 +41,11 @@ GEM
|
|
41
41
|
concurrent-ruby (~> 1.0)
|
42
42
|
karafka-core (2.3.0)
|
43
43
|
karafka-rdkafka (>= 0.14.8, < 0.15.0)
|
44
|
-
karafka-rdkafka (0.14.
|
44
|
+
karafka-rdkafka (0.14.10)
|
45
45
|
ffi (~> 1.15)
|
46
46
|
mini_portile2 (~> 2.6)
|
47
47
|
rake (> 12)
|
48
|
-
karafka-web (0.8.
|
48
|
+
karafka-web (0.8.2)
|
49
49
|
erubi (~> 1.4)
|
50
50
|
karafka (>= 2.3.0, < 2.4.0)
|
51
51
|
karafka-core (>= 2.3.0, < 2.4.0)
|
@@ -56,7 +56,7 @@ GEM
|
|
56
56
|
mutex_m (0.2.0)
|
57
57
|
rack (3.0.9)
|
58
58
|
rake (13.1.0)
|
59
|
-
roda (3.
|
59
|
+
roda (3.77.0)
|
60
60
|
rack
|
61
61
|
rspec (3.13.0)
|
62
62
|
rspec-core (~> 3.13.0)
|
@@ -81,10 +81,10 @@ GEM
|
|
81
81
|
tilt (2.3.0)
|
82
82
|
tzinfo (2.0.6)
|
83
83
|
concurrent-ruby (~> 1.0)
|
84
|
-
waterdrop (2.6.
|
84
|
+
waterdrop (2.6.14)
|
85
85
|
karafka-core (>= 2.2.3, < 3.0.0)
|
86
86
|
zeitwerk (~> 2.3)
|
87
|
-
zeitwerk (2.6.
|
87
|
+
zeitwerk (2.6.13)
|
88
88
|
|
89
89
|
PLATFORMS
|
90
90
|
ruby
|
data/bin/integrations
CHANGED
@@ -43,7 +43,8 @@ class Scenario
|
|
43
43
|
'consumption/worker_critical_error_behaviour_spec.rb' => [0, 2].freeze,
|
44
44
|
'shutdown/on_hanging_jobs_and_a_shutdown_spec.rb' => [2].freeze,
|
45
45
|
'shutdown/on_hanging_on_shutdown_job_and_a_shutdown_spec.rb' => [2].freeze,
|
46
|
-
'shutdown/on_hanging_listener_and_shutdown_spec.rb' => [2].freeze
|
46
|
+
'shutdown/on_hanging_listener_and_shutdown_spec.rb' => [2].freeze,
|
47
|
+
'swarm/forceful_shutdown_of_hanging_spec.rb' => [2].freeze
|
47
48
|
}.freeze
|
48
49
|
|
49
50
|
private_constant :MAX_RUN_TIME, :EXIT_CODES
|
data/bin/rspecs
CHANGED
@@ -2,5 +2,9 @@
|
|
2
2
|
|
3
3
|
set -e
|
4
4
|
|
5
|
-
|
6
|
-
SPECS_TYPE=
|
5
|
+
# Run only regular non-forking specs first
|
6
|
+
SPECS_TYPE=regular bundle exec rspec --tag ~type:pro --tag ~mode:fork
|
7
|
+
# Run forking specs, they need to run in isolation not to crash because of librdkafka
|
8
|
+
SPECS_TYPE=regular bundle exec rspec --tag mode:fork
|
9
|
+
# Run pro specs at the end
|
10
|
+
SPECS_TYPE=pro bundle exec rspec --tag type:pro --tag ~mode:fork
|
data/config/locales/errors.yml
CHANGED
@@ -1,11 +1,12 @@
|
|
1
1
|
en:
|
2
2
|
validations:
|
3
3
|
config:
|
4
|
-
missing: needs to be present
|
5
|
-
client_id_format: 'needs to be a string with a Kafka accepted format'
|
6
4
|
license.entity_format: needs to be a string
|
7
5
|
license.token_format: needs to be either false or a string
|
8
6
|
license.expires_on_format: needs to be a valid date
|
7
|
+
|
8
|
+
missing: needs to be present
|
9
|
+
client_id_format: 'needs to be a string with a Kafka accepted format'
|
9
10
|
concurrency_format: needs to be an integer bigger than 0
|
10
11
|
consumer_mapper_format: needs to be present
|
11
12
|
consumer_persistence_format: needs to be either true or false
|
@@ -15,6 +16,10 @@ en:
|
|
15
16
|
shutdown_timeout_format: needs to be an integer bigger than 0
|
16
17
|
max_wait_time_format: needs to be an integer bigger than 0
|
17
18
|
kafka_format: needs to be a filled hash
|
19
|
+
key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
|
20
|
+
max_timeout_vs_pause_max_timeout: pause_timeout must be less or equal to pause_max_timeout
|
21
|
+
shutdown_timeout_vs_max_wait_time: shutdown_timeout must be more than max_wait_time
|
22
|
+
|
18
23
|
internal.processing.jobs_builder_format: cannot be nil
|
19
24
|
internal.processing.jobs_queue_class_format: cannot be nil
|
20
25
|
internal.processing.scheduler_class_format: cannot be nil
|
@@ -22,14 +27,20 @@ en:
|
|
22
27
|
internal.processing.partitioner_class_format: cannot be nil
|
23
28
|
internal.processing.strategy_selector_format: cannot be nil
|
24
29
|
internal.processing.expansions_selector_format: cannot be nil
|
25
|
-
|
26
|
-
internal.active_job.
|
27
|
-
internal.active_job.
|
30
|
+
|
31
|
+
internal.active_job.dispatcher_format: cannot be nil
|
32
|
+
internal.active_job.job_options_contract_format: cannot be nil
|
33
|
+
internal.active_job.consumer_class_format: cannot be nil
|
34
|
+
|
28
35
|
internal.status_format: needs to be present
|
29
36
|
internal.process_format: needs to be present
|
30
37
|
internal.tick_interval_format: needs to be an integer bigger or equal to 1000
|
38
|
+
internal.supervision_sleep_format: needs to be an integer bigger than 0
|
39
|
+
internal.forceful_exit_code_format: needs to be an integer bigger or equal to 0
|
40
|
+
|
31
41
|
internal.routing.builder_format: needs to be present
|
32
42
|
internal.routing.subscription_groups_builder_format: needs to be present
|
43
|
+
|
33
44
|
internal.connection.manager_format: needs to be present
|
34
45
|
internal.connection.conductor_format: needs to be present
|
35
46
|
internal.connection.proxy.query_watermark_offsets.timeout_format: needs to be an integer bigger than 0
|
@@ -41,14 +52,25 @@ en:
|
|
41
52
|
internal.connection.proxy.committed.timeout_format: needs to be an integer bigger than 0
|
42
53
|
internal.connection.proxy.committed.max_attempts_format: needs to be an integer bigger than 0
|
43
54
|
internal.connection.proxy.committed.wait_time_format: needs to be an integer bigger than 0
|
44
|
-
|
45
|
-
|
46
|
-
|
55
|
+
|
56
|
+
internal.swarm.manager_format: cannot be nil
|
57
|
+
internal.swarm.orphaned_exit_code_format: needs to be an integer bigger or equal to 0
|
58
|
+
internal.swarm.pidfd_open_syscall_format: needs to be an integer bigger or equal to 0
|
59
|
+
internal.swarm.pidfd_signal_syscall_format: needs to be an integer bigger or equal to 0
|
60
|
+
internal.swarm.supervision_interval_format: needs to be an integer bigger or equal to 1000
|
61
|
+
internal.swarm.liveness_interval_format: needs to be an integer bigger or equal to 1000
|
62
|
+
internal.swarm.liveness_listener_format: cannot be nil
|
63
|
+
internal.swarm.node_report_timeout_format: needs to be an integer bigger or equal to 1000
|
64
|
+
internal.swarm.node_restart_timeout_format: needs to be an integer bigger or equal to 1000
|
65
|
+
|
47
66
|
admin.kafka_format: needs to be a hash
|
48
67
|
admin.group_id_format: 'needs to be a string with a Kafka accepted format'
|
49
68
|
admin.max_wait_time_format: 'needs to be an integer bigger than 0'
|
50
69
|
admin.max_attempts_format: 'needs to be an integer bigger than 0'
|
51
70
|
|
71
|
+
swarm.nodes_format: 'needs to be an integer bigger than 0'
|
72
|
+
swarm.node_format: needs to be false or node instance
|
73
|
+
|
52
74
|
server_cli_options:
|
53
75
|
missing: needs to be present
|
54
76
|
consumer_groups_inclusion: Unknown consumer group name
|
@@ -8,6 +8,8 @@ en:
|
|
8
8
|
long_running_job.active_format: needs to be either true or false
|
9
9
|
|
10
10
|
dead_letter_queue_with_virtual_partitions: when using Dead Letter Queue with Virtual Partitions, at least one retry is required.
|
11
|
+
dead_letter_queue.strategy_format: 'needs to respond to #call'
|
12
|
+
dead_letter_queue.strategy_missing: needs to be present
|
11
13
|
|
12
14
|
throttling.active_format: needs to be either true or false
|
13
15
|
throttling.limit_format: needs to be equal or more than 1
|
data/docker-compose.yml
CHANGED
data/lib/karafka/app.rb
CHANGED
@@ -6,6 +6,20 @@ module Karafka
|
|
6
6
|
extend Setup::Dsl
|
7
7
|
|
8
8
|
class << self
|
9
|
+
# Notifies the Ruby virtual machine that the boot sequence is finished, and that now is a
|
10
|
+
# good time to optimize the application. In case of older Ruby versions, runs compacting,
|
11
|
+
# which is part of the full warmup introduced in Ruby 3.3.
|
12
|
+
def warmup
|
13
|
+
# Per recommendation, this should not run in children nodes
|
14
|
+
return if Karafka::App.config.swarm.node
|
15
|
+
|
16
|
+
monitor.instrument('app.before_warmup', caller: self)
|
17
|
+
|
18
|
+
return GC.compact unless ::Process.respond_to?(:warmup)
|
19
|
+
|
20
|
+
::Process.warmup
|
21
|
+
end
|
22
|
+
|
9
23
|
# @return [Karafka::Routing::Builder] consumers builder instance alias
|
10
24
|
def consumer_groups
|
11
25
|
config
|
data/lib/karafka/cli/base.rb
CHANGED
@@ -5,6 +5,8 @@ module Karafka
|
|
5
5
|
# Base class for all the command that we want to define
|
6
6
|
# This base class provides an interface to easier separate single independent commands
|
7
7
|
class Base
|
8
|
+
include Helpers::Colorize
|
9
|
+
|
8
10
|
# @return [Hash] given command cli options
|
9
11
|
attr_reader :options
|
10
12
|
|
@@ -19,6 +21,23 @@ module Karafka
|
|
19
21
|
raise NotImplementedError, 'Implement this in a subclass'
|
20
22
|
end
|
21
23
|
|
24
|
+
private
|
25
|
+
|
26
|
+
# Prints marketing info
|
27
|
+
def print_marketing_info
|
28
|
+
Karafka.logger.info Info::BANNER
|
29
|
+
|
30
|
+
if Karafka.pro?
|
31
|
+
Karafka.logger.info(
|
32
|
+
green('Thank you for using Karafka Pro!')
|
33
|
+
)
|
34
|
+
else
|
35
|
+
Karafka.logger.info(
|
36
|
+
red('Upgrade to Karafka Pro for more features and support: https://karafka.io')
|
37
|
+
)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
22
41
|
class << self
|
23
42
|
# Loads proper environment with what is needed to run the CLI
|
24
43
|
def load
|
data/lib/karafka/cli/server.rb
CHANGED
@@ -5,8 +5,6 @@ module Karafka
|
|
5
5
|
class Cli
|
6
6
|
# Server Karafka Cli action
|
7
7
|
class Server < Base
|
8
|
-
include Helpers::Colorize
|
9
|
-
|
10
8
|
# Types of things we can include / exclude from the routing via the CLI options
|
11
9
|
SUPPORTED_TYPES = ::Karafka::Routing::ActivityManager::SUPPORTED_TYPES
|
12
10
|
|
@@ -16,63 +14,68 @@ module Karafka
|
|
16
14
|
|
17
15
|
aliases :s, :consumer
|
18
16
|
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
17
|
+
# Those options can also be used when in swarm mode, hence we re-use
|
18
|
+
OPTIONS_BUILDER = lambda do
|
19
|
+
option(
|
20
|
+
:consumer_groups,
|
21
|
+
'Runs server only with specified consumer groups',
|
22
|
+
Array,
|
23
|
+
%w[
|
24
|
+
-g
|
25
|
+
--consumer_groups
|
26
|
+
--include_consumer_groups
|
27
|
+
]
|
28
|
+
)
|
29
|
+
|
30
|
+
option(
|
31
|
+
:subscription_groups,
|
32
|
+
'Runs server only with specified subscription groups',
|
33
|
+
Array,
|
34
|
+
%w[
|
35
|
+
--subscription_groups
|
36
|
+
--include_subscription_groups
|
37
|
+
]
|
38
|
+
)
|
39
|
+
|
40
|
+
option(
|
41
|
+
:topics,
|
42
|
+
'Runs server only with specified topics',
|
43
|
+
Array,
|
44
|
+
%w[
|
45
|
+
--topics
|
46
|
+
--include_topics
|
47
|
+
]
|
48
|
+
)
|
49
|
+
|
50
|
+
option(
|
51
|
+
:exclude_consumer_groups,
|
52
|
+
'Runs server without specified consumer groups',
|
53
|
+
Array,
|
54
|
+
%w[
|
55
|
+
--exclude_consumer_groups
|
56
|
+
]
|
57
|
+
)
|
58
|
+
|
59
|
+
option(
|
60
|
+
:exclude_subscription_groups,
|
61
|
+
'Runs server without specified subscription groups',
|
62
|
+
Array,
|
63
|
+
%w[
|
64
|
+
--exclude_subscription_groups
|
65
|
+
]
|
66
|
+
)
|
67
|
+
|
68
|
+
option(
|
69
|
+
:exclude_topics,
|
70
|
+
'Runs server without specified topics',
|
71
|
+
Array,
|
72
|
+
%w[
|
73
|
+
--exclude_topics
|
74
|
+
]
|
75
|
+
)
|
76
|
+
end
|
77
|
+
|
78
|
+
instance_exec(&OPTIONS_BUILDER)
|
76
79
|
|
77
80
|
# Start the Karafka server
|
78
81
|
def call
|
@@ -85,8 +88,6 @@ module Karafka
|
|
85
88
|
Karafka::Server.run
|
86
89
|
end
|
87
90
|
|
88
|
-
private
|
89
|
-
|
90
91
|
# Registers things we want to include (if defined)
|
91
92
|
def register_inclusions
|
92
93
|
activities = ::Karafka::App.config.internal.routing.activity_manager
|
@@ -108,21 +109,6 @@ module Karafka
|
|
108
109
|
names.each { |name| activities.exclude(type, name) }
|
109
110
|
end
|
110
111
|
end
|
111
|
-
|
112
|
-
# Prints marketing info
|
113
|
-
def print_marketing_info
|
114
|
-
Karafka.logger.info Info::BANNER
|
115
|
-
|
116
|
-
if Karafka.pro?
|
117
|
-
Karafka.logger.info(
|
118
|
-
green('Thank you for using Karafka Pro!')
|
119
|
-
)
|
120
|
-
else
|
121
|
-
Karafka.logger.info(
|
122
|
-
red('Upgrade to Karafka Pro for more features and support: https://karafka.io')
|
123
|
-
)
|
124
|
-
end
|
125
|
-
end
|
126
112
|
end
|
127
113
|
end
|
128
114
|
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
# Karafka framework Cli
|
5
|
+
class Cli
|
6
|
+
# Starts swarm of consumers forked from the supervisor
|
7
|
+
class Swarm < Base
|
8
|
+
desc 'Starts swarm of Karafka consumers with a supervisor'
|
9
|
+
|
10
|
+
aliases :swarm
|
11
|
+
|
12
|
+
instance_exec(&Server::OPTIONS_BUILDER)
|
13
|
+
|
14
|
+
# Starts the swarm
|
15
|
+
def call
|
16
|
+
::Karafka::Swarm.ensure_supported!
|
17
|
+
|
18
|
+
# Print our banner and info in the dev mode
|
19
|
+
print_marketing_info if Karafka::App.env.development?
|
20
|
+
|
21
|
+
# This will register inclusions and exclusions in the routing, so all forks will use it
|
22
|
+
server = Server.new
|
23
|
+
server.register_inclusions
|
24
|
+
server.register_exclusions
|
25
|
+
|
26
|
+
Karafka::Swarm::Supervisor.new.run
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
data/lib/karafka/constraints.rb
CHANGED
@@ -15,13 +15,13 @@ module Karafka
|
|
15
15
|
# Skip verification if web is not used at all
|
16
16
|
return unless require_version('karafka/web')
|
17
17
|
|
18
|
-
# All good if version higher than 0.
|
19
|
-
return if version(Karafka::Web::VERSION) >= version('0.
|
18
|
+
# All good if version higher than 0.8.1 because we expect 0.8.2 or higher
|
19
|
+
return if version(Karafka::Web::VERSION) >= version('0.8.2')
|
20
20
|
|
21
21
|
# If older web-ui used, we cannot allow it
|
22
22
|
raise(
|
23
23
|
Errors::DependencyConstraintsError,
|
24
|
-
'karafka-web < 0.8.
|
24
|
+
'karafka-web < 0.8.2 is not compatible with this karafka version'
|
25
25
|
)
|
26
26
|
end
|
27
27
|
|
@@ -34,6 +34,11 @@ module Karafka
|
|
34
34
|
required(:max_wait_time) { |val| val.is_a?(Integer) && val.positive? }
|
35
35
|
required(:kafka) { |val| val.is_a?(Hash) && !val.empty? }
|
36
36
|
|
37
|
+
nested(:swarm) do
|
38
|
+
required(:nodes) { |val| val.is_a?(Integer) && val.positive? }
|
39
|
+
required(:node) { |val| val == false || val.is_a?(Karafka::Swarm::Node) }
|
40
|
+
end
|
41
|
+
|
37
42
|
nested(:admin) do
|
38
43
|
# Can be empty because inherits values from the root kafka
|
39
44
|
required(:kafka) { |val| val.is_a?(Hash) }
|
@@ -49,6 +54,20 @@ module Karafka
|
|
49
54
|
# In theory this could be less than a second, however this would impact the maximum time
|
50
55
|
# of a single consumer queue poll, hence we prevent it
|
51
56
|
required(:tick_interval) { |val| val.is_a?(Integer) && val >= 1_000 }
|
57
|
+
required(:supervision_sleep) { |val| val.is_a?(Numeric) && val.positive? }
|
58
|
+
required(:forceful_exit_code) { |val| val.is_a?(Integer) && val >= 0 }
|
59
|
+
|
60
|
+
nested(:swarm) do
|
61
|
+
required(:manager) { |val| !val.nil? }
|
62
|
+
required(:orphaned_exit_code) { |val| val.is_a?(Integer) && val >= 0 }
|
63
|
+
required(:pidfd_open_syscall) { |val| val.is_a?(Integer) && val >= 0 }
|
64
|
+
required(:pidfd_signal_syscall) { |val| val.is_a?(Integer) && val >= 0 }
|
65
|
+
required(:supervision_interval) { |val| val.is_a?(Integer) && val >= 1_000 }
|
66
|
+
required(:liveness_interval) { |val| val.is_a?(Integer) && val >= 1_000 }
|
67
|
+
required(:liveness_listener) { |val| !val.nil? }
|
68
|
+
required(:node_report_timeout) { |val| val.is_a?(Integer) && val >= 1_000 }
|
69
|
+
required(:node_restart_timeout) { |val| val.is_a?(Integer) && val >= 1_000 }
|
70
|
+
end
|
52
71
|
|
53
72
|
nested(:connection) do
|
54
73
|
required(:manager) { |val| !val.nil? }
|
data/lib/karafka/errors.rb
CHANGED
@@ -87,5 +87,17 @@ module Karafka
|
|
87
87
|
# Because we do not want to require web out of the box and we do not want to lock web with
|
88
88
|
# karafka 1:1, we do such a sanity check
|
89
89
|
DependencyConstraintsError = Class.new(BaseError)
|
90
|
+
|
91
|
+
# Raised when we were not able to open pidfd for given pid
|
92
|
+
# This should not happen. If you see it, please report.
|
93
|
+
PidfdOpenFailedError = Class.new(BaseError)
|
94
|
+
|
95
|
+
# Failed to send signal to a process via pidfd
|
96
|
+
# This should not happen. If you see it, please report.
|
97
|
+
PidfdSignalFailedError = Class.new(BaseError)
|
98
|
+
|
99
|
+
# Raised when given option/feature is not supported on a given platform or when given option
|
100
|
+
# is not supported in a given configuration
|
101
|
+
UnsupportedOptionError = Class.new(BaseError)
|
90
102
|
end
|
91
103
|
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Helpers
|
5
|
+
# Module allowing for configuration injections. By default injects whole app config
|
6
|
+
# Allows for granular config injection
|
7
|
+
class ConfigImporter < Module
|
8
|
+
# @param attributes [Hash<Symbol, Array<Symbol>>] map defining what we want to inject.
|
9
|
+
# The key is the name under which attribute will be visible and the value is the full
|
10
|
+
# path to the attribute
|
11
|
+
def initialize(attributes = { config: %i[itself] })
|
12
|
+
super()
|
13
|
+
@attributes = attributes
|
14
|
+
end
|
15
|
+
|
16
|
+
# @param model [Object] object to which we want to add the config fetcher
|
17
|
+
def included(model)
|
18
|
+
super
|
19
|
+
|
20
|
+
@attributes.each do |name, path|
|
21
|
+
model.class_eval <<~RUBY, __FILE__, __LINE__ + 1
|
22
|
+
def #{name}
|
23
|
+
@#{name} ||= ::Karafka::App.config.#{path.join('.')}
|
24
|
+
end
|
25
|
+
RUBY
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -226,6 +226,34 @@ module Karafka
|
|
226
226
|
MSG
|
227
227
|
end
|
228
228
|
|
229
|
+
# @param event [Karafka::Core::Monitoring::Event] event details including payload
|
230
|
+
def on_swarm_manager_stopping(event)
|
231
|
+
node = event[:node]
|
232
|
+
error "Swarm manager detected unhealthy node #{node.pid}. Sending TERM signal..."
|
233
|
+
end
|
234
|
+
|
235
|
+
# @param event [Karafka::Core::Monitoring::Event] event details including payload
|
236
|
+
def on_swarm_manager_terminating(event)
|
237
|
+
node = event[:node]
|
238
|
+
error "Swarm manager detected unresponsive node #{node.pid}. Sending KILL signal..."
|
239
|
+
end
|
240
|
+
|
241
|
+
# @param event [Karafka::Core::Monitoring::Event] event details including payload
|
242
|
+
def on_swarm_manager_before_fork(event)
|
243
|
+
debug "Swarm manager starting node with id: #{event[:node].id}"
|
244
|
+
end
|
245
|
+
|
246
|
+
# @param _event [Karafka::Core::Monitoring::Event] event details including payload
|
247
|
+
def on_swarm_node_after_fork(_event)
|
248
|
+
info "Swarm node #{::Process.pid} forked from #{::Process.ppid}"
|
249
|
+
end
|
250
|
+
|
251
|
+
# @param event [Karafka::Core::Monitoring::Event] event details including payload
|
252
|
+
def on_swarm_manager_control(event)
|
253
|
+
pids = event[:caller].nodes.map(&:pid).join(', ')
|
254
|
+
debug "Swarm manager checking nodes: #{pids}"
|
255
|
+
end
|
256
|
+
|
229
257
|
# There are many types of errors that can occur in many places, but we provide a single
|
230
258
|
# handler for all of them to simplify error instrumentation.
|
231
259
|
# @param event [Karafka::Core::Monitoring::Event] event details including payload
|
@@ -259,6 +287,9 @@ module Karafka
|
|
259
287
|
when 'connection.listener.fetch_loop.error'
|
260
288
|
error "Listener fetch loop error: #{error}"
|
261
289
|
error details
|
290
|
+
when 'swarm.supervisor.error'
|
291
|
+
fatal "Swarm supervisor crashed due to an error: #{error}"
|
292
|
+
fatal details
|
262
293
|
when 'runner.call.error'
|
263
294
|
fatal "Runner crashed due to an error: #{error}"
|
264
295
|
fatal details
|
@@ -22,6 +22,8 @@ module Karafka
|
|
22
22
|
|
23
23
|
app.initializing
|
24
24
|
app.initialized
|
25
|
+
app.before_warmup
|
26
|
+
app.supervising
|
25
27
|
app.running
|
26
28
|
app.quieting
|
27
29
|
app.quiet
|
@@ -73,6 +75,13 @@ module Karafka
|
|
73
75
|
|
74
76
|
statistics.emitted
|
75
77
|
|
78
|
+
swarm.node.after_fork
|
79
|
+
swarm.manager.before_fork
|
80
|
+
swarm.manager.after_fork
|
81
|
+
swarm.manager.control
|
82
|
+
swarm.manager.stopping
|
83
|
+
swarm.manager.terminating
|
84
|
+
|
76
85
|
worker.process
|
77
86
|
worker.processed
|
78
87
|
worker.completed
|
@@ -129,6 +129,8 @@ module Karafka
|
|
129
129
|
fatal "Runner crashed due to an error: #{error}"
|
130
130
|
when 'app.stopping.error'
|
131
131
|
error 'Forceful Karafka server stop'
|
132
|
+
when 'swarm.supervisor.error'
|
133
|
+
fatal "Swarm supervisor crashed due to an error: #{error}"
|
132
134
|
when 'librdkafka.error'
|
133
135
|
error "librdkafka internal error occurred: #{error}"
|
134
136
|
# Those will only occur when retries in the client fail and when they did not stop
|