karafka 1.4.13 → 2.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +3 -3
- data/.github/workflows/ci.yml +85 -30
- data/.ruby-version +1 -1
- data/CHANGELOG.md +268 -7
- data/CONTRIBUTING.md +10 -19
- data/Gemfile +6 -0
- data/Gemfile.lock +44 -87
- data/LICENSE +17 -0
- data/LICENSE-COMM +89 -0
- data/LICENSE-LGPL +165 -0
- data/README.md +44 -48
- data/bin/benchmarks +85 -0
- data/bin/create_token +22 -0
- data/bin/integrations +237 -0
- data/bin/karafka +4 -0
- data/bin/scenario +29 -0
- data/bin/stress_many +13 -0
- data/bin/stress_one +13 -0
- data/bin/wait_for_kafka +20 -0
- data/certs/karafka-pro.pem +11 -0
- data/config/errors.yml +55 -40
- data/docker-compose.yml +39 -3
- data/karafka.gemspec +11 -17
- data/lib/active_job/karafka.rb +21 -0
- data/lib/active_job/queue_adapters/karafka_adapter.rb +26 -0
- data/lib/karafka/active_job/consumer.rb +26 -0
- data/lib/karafka/active_job/dispatcher.rb +38 -0
- data/lib/karafka/active_job/job_extensions.rb +34 -0
- data/lib/karafka/active_job/job_options_contract.rb +21 -0
- data/lib/karafka/active_job/routing/extensions.rb +31 -0
- data/lib/karafka/app.rb +15 -20
- data/lib/karafka/base_consumer.rb +181 -31
- data/lib/karafka/cli/base.rb +4 -4
- data/lib/karafka/cli/info.rb +43 -9
- data/lib/karafka/cli/install.rb +19 -10
- data/lib/karafka/cli/server.rb +17 -42
- data/lib/karafka/cli.rb +4 -11
- data/lib/karafka/connection/client.rb +385 -90
- data/lib/karafka/connection/listener.rb +246 -38
- data/lib/karafka/connection/listeners_batch.rb +24 -0
- data/lib/karafka/connection/messages_buffer.rb +84 -0
- data/lib/karafka/connection/pauses_manager.rb +46 -0
- data/lib/karafka/connection/raw_messages_buffer.rb +101 -0
- data/lib/karafka/connection/rebalance_manager.rb +78 -0
- data/lib/karafka/contracts/base.rb +17 -0
- data/lib/karafka/contracts/config.rb +88 -11
- data/lib/karafka/contracts/consumer_group.rb +21 -189
- data/lib/karafka/contracts/consumer_group_topic.rb +34 -11
- data/lib/karafka/contracts/server_cli_options.rb +19 -18
- data/lib/karafka/contracts.rb +1 -1
- data/lib/karafka/env.rb +46 -0
- data/lib/karafka/errors.rb +21 -21
- data/lib/karafka/helpers/async.rb +33 -0
- data/lib/karafka/helpers/colorize.rb +20 -0
- data/lib/karafka/helpers/multi_delegator.rb +2 -2
- data/lib/karafka/instrumentation/callbacks/error.rb +40 -0
- data/lib/karafka/instrumentation/callbacks/statistics.rb +41 -0
- data/lib/karafka/instrumentation/logger_listener.rb +164 -0
- data/lib/karafka/instrumentation/monitor.rb +13 -61
- data/lib/karafka/instrumentation/notifications.rb +52 -0
- data/lib/karafka/instrumentation/proctitle_listener.rb +3 -3
- data/lib/karafka/instrumentation/vendors/datadog/dashboard.json +1 -0
- data/lib/karafka/instrumentation/vendors/datadog/listener.rb +232 -0
- data/lib/karafka/instrumentation.rb +21 -0
- data/lib/karafka/licenser.rb +75 -0
- data/lib/karafka/messages/batch_metadata.rb +45 -0
- data/lib/karafka/messages/builders/batch_metadata.rb +40 -0
- data/lib/karafka/messages/builders/message.rb +39 -0
- data/lib/karafka/messages/builders/messages.rb +32 -0
- data/lib/karafka/{params/params.rb → messages/message.rb} +7 -12
- data/lib/karafka/messages/messages.rb +64 -0
- data/lib/karafka/{params → messages}/metadata.rb +4 -6
- data/lib/karafka/messages/seek.rb +9 -0
- data/lib/karafka/patches/rdkafka/consumer.rb +22 -0
- data/lib/karafka/pro/active_job/consumer.rb +46 -0
- data/lib/karafka/pro/active_job/dispatcher.rb +61 -0
- data/lib/karafka/pro/active_job/job_options_contract.rb +32 -0
- data/lib/karafka/pro/base_consumer.rb +82 -0
- data/lib/karafka/pro/contracts/base.rb +21 -0
- data/lib/karafka/pro/contracts/consumer_group.rb +34 -0
- data/lib/karafka/pro/contracts/consumer_group_topic.rb +33 -0
- data/lib/karafka/pro/loader.rb +76 -0
- data/lib/karafka/pro/performance_tracker.rb +80 -0
- data/lib/karafka/pro/processing/coordinator.rb +72 -0
- data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb +37 -0
- data/lib/karafka/pro/processing/jobs_builder.rb +32 -0
- data/lib/karafka/pro/processing/partitioner.rb +60 -0
- data/lib/karafka/pro/processing/scheduler.rb +56 -0
- data/lib/karafka/pro/routing/builder_extensions.rb +30 -0
- data/lib/karafka/pro/routing/topic_extensions.rb +38 -0
- data/lib/karafka/pro.rb +13 -0
- data/lib/karafka/process.rb +1 -0
- data/lib/karafka/processing/coordinator.rb +88 -0
- data/lib/karafka/processing/coordinators_buffer.rb +54 -0
- data/lib/karafka/processing/executor.rb +118 -0
- data/lib/karafka/processing/executors_buffer.rb +88 -0
- data/lib/karafka/processing/jobs/base.rb +51 -0
- data/lib/karafka/processing/jobs/consume.rb +42 -0
- data/lib/karafka/processing/jobs/revoked.rb +22 -0
- data/lib/karafka/processing/jobs/shutdown.rb +23 -0
- data/lib/karafka/processing/jobs_builder.rb +29 -0
- data/lib/karafka/processing/jobs_queue.rb +144 -0
- data/lib/karafka/processing/partitioner.rb +22 -0
- data/lib/karafka/processing/result.rb +29 -0
- data/lib/karafka/processing/scheduler.rb +22 -0
- data/lib/karafka/processing/worker.rb +88 -0
- data/lib/karafka/processing/workers_batch.rb +27 -0
- data/lib/karafka/railtie.rb +113 -0
- data/lib/karafka/routing/builder.rb +15 -24
- data/lib/karafka/routing/consumer_group.rb +11 -19
- data/lib/karafka/routing/consumer_mapper.rb +1 -2
- data/lib/karafka/routing/router.rb +1 -1
- data/lib/karafka/routing/subscription_group.rb +53 -0
- data/lib/karafka/routing/subscription_groups_builder.rb +53 -0
- data/lib/karafka/routing/topic.rb +61 -24
- data/lib/karafka/routing/topics.rb +38 -0
- data/lib/karafka/runner.rb +51 -0
- data/lib/karafka/serialization/json/deserializer.rb +6 -15
- data/lib/karafka/server.rb +67 -26
- data/lib/karafka/setup/config.rb +147 -175
- data/lib/karafka/status.rb +14 -5
- data/lib/karafka/templates/example_consumer.rb.erb +16 -0
- data/lib/karafka/templates/karafka.rb.erb +15 -51
- data/lib/karafka/time_trackers/base.rb +19 -0
- data/lib/karafka/time_trackers/pause.rb +92 -0
- data/lib/karafka/time_trackers/poll.rb +65 -0
- data/lib/karafka/version.rb +1 -1
- data/lib/karafka.rb +38 -17
- data.tar.gz.sig +0 -0
- metadata +118 -120
- metadata.gz.sig +0 -0
- data/MIT-LICENCE +0 -18
- data/lib/karafka/assignment_strategies/round_robin.rb +0 -13
- data/lib/karafka/attributes_map.rb +0 -63
- data/lib/karafka/backends/inline.rb +0 -16
- data/lib/karafka/base_responder.rb +0 -226
- data/lib/karafka/cli/flow.rb +0 -48
- data/lib/karafka/cli/missingno.rb +0 -19
- data/lib/karafka/code_reloader.rb +0 -67
- data/lib/karafka/connection/api_adapter.rb +0 -158
- data/lib/karafka/connection/batch_delegator.rb +0 -55
- data/lib/karafka/connection/builder.rb +0 -23
- data/lib/karafka/connection/message_delegator.rb +0 -36
- data/lib/karafka/consumers/batch_metadata.rb +0 -10
- data/lib/karafka/consumers/callbacks.rb +0 -71
- data/lib/karafka/consumers/includer.rb +0 -64
- data/lib/karafka/consumers/responders.rb +0 -24
- data/lib/karafka/consumers/single_params.rb +0 -15
- data/lib/karafka/contracts/responder_usage.rb +0 -54
- data/lib/karafka/fetcher.rb +0 -42
- data/lib/karafka/helpers/class_matcher.rb +0 -88
- data/lib/karafka/helpers/config_retriever.rb +0 -46
- data/lib/karafka/helpers/inflector.rb +0 -26
- data/lib/karafka/instrumentation/stdout_listener.rb +0 -140
- data/lib/karafka/params/batch_metadata.rb +0 -26
- data/lib/karafka/params/builders/batch_metadata.rb +0 -30
- data/lib/karafka/params/builders/params.rb +0 -38
- data/lib/karafka/params/builders/params_batch.rb +0 -25
- data/lib/karafka/params/params_batch.rb +0 -60
- data/lib/karafka/patches/ruby_kafka.rb +0 -47
- data/lib/karafka/persistence/client.rb +0 -29
- data/lib/karafka/persistence/consumers.rb +0 -45
- data/lib/karafka/persistence/topics.rb +0 -48
- data/lib/karafka/responders/builder.rb +0 -36
- data/lib/karafka/responders/topic.rb +0 -55
- data/lib/karafka/routing/topic_mapper.rb +0 -53
- data/lib/karafka/serialization/json/serializer.rb +0 -31
- data/lib/karafka/setup/configurators/water_drop.rb +0 -36
- data/lib/karafka/templates/application_responder.rb.erb +0 -11
@@ -2,19 +2,96 @@
|
|
2
2
|
|
3
3
|
module Karafka
|
4
4
|
module Contracts
|
5
|
-
# Contract with validation rules for Karafka configuration details
|
5
|
+
# Contract with validation rules for Karafka configuration details.
|
6
|
+
#
|
6
7
|
# @note There are many more configuration options inside of the
|
7
|
-
# Karafka::Setup::Config model, but we don't validate them here as they are
|
8
|
+
# `Karafka::Setup::Config` model, but we don't validate them here as they are
|
8
9
|
# validated per each route (topic + consumer_group) because they can be overwritten,
|
9
|
-
# so we validate all of that once all the routes are defined and ready
|
10
|
-
class Config <
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
10
|
+
# so we validate all of that once all the routes are defined and ready.
|
11
|
+
class Config < Base
|
12
|
+
configure do |config|
|
13
|
+
config.error_messages = YAML.safe_load(
|
14
|
+
File.read(
|
15
|
+
File.join(Karafka.gem_root, 'config', 'errors.yml')
|
16
|
+
)
|
17
|
+
).fetch('en').fetch('validations').fetch('config')
|
18
|
+
end
|
19
|
+
|
20
|
+
# License validity happens in the licenser. Here we do only the simple consistency checks
|
21
|
+
nested(:license) do
|
22
|
+
required(:token) { |val| [true, false].include?(val) || val.is_a?(String) }
|
23
|
+
required(:entity) { |val| val.is_a?(String) }
|
24
|
+
end
|
25
|
+
|
26
|
+
required(:client_id) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
|
27
|
+
required(:concurrency) { |val| val.is_a?(Integer) && val.positive? }
|
28
|
+
required(:consumer_mapper) { |val| !val.nil? }
|
29
|
+
required(:consumer_persistence) { |val| [true, false].include?(val) }
|
30
|
+
required(:pause_timeout) { |val| val.is_a?(Integer) && val.positive? }
|
31
|
+
required(:pause_max_timeout) { |val| val.is_a?(Integer) && val.positive? }
|
32
|
+
required(:pause_with_exponential_backoff) { |val| [true, false].include?(val) }
|
33
|
+
required(:shutdown_timeout) { |val| val.is_a?(Integer) && val.positive? }
|
34
|
+
required(:max_wait_time) { |val| val.is_a?(Integer) && val.positive? }
|
35
|
+
required(:kafka) { |val| val.is_a?(Hash) && !val.empty? }
|
36
|
+
|
37
|
+
# We validate internals just to be sure, that they are present and working
|
38
|
+
nested(:internal) do
|
39
|
+
required(:status) { |val| !val.nil? }
|
40
|
+
required(:process) { |val| !val.nil? }
|
41
|
+
|
42
|
+
nested(:routing) do
|
43
|
+
required(:builder) { |val| !val.nil? }
|
44
|
+
required(:subscription_groups_builder) { |val| !val.nil? }
|
45
|
+
end
|
46
|
+
|
47
|
+
nested(:processing) do
|
48
|
+
required(:jobs_builder) { |val| !val.nil? }
|
49
|
+
required(:scheduler) { |val| !val.nil? }
|
50
|
+
required(:coordinator_class) { |val| !val.nil? }
|
51
|
+
required(:partitioner_class) { |val| !val.nil? }
|
52
|
+
end
|
53
|
+
|
54
|
+
nested(:active_job) do
|
55
|
+
required(:dispatcher) { |val| !val.nil? }
|
56
|
+
required(:job_options_contract) { |val| !val.nil? }
|
57
|
+
required(:consumer_class) { |val| !val.nil? }
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
virtual do |data, errors|
|
62
|
+
next unless errors.empty?
|
63
|
+
|
64
|
+
detected_errors = []
|
65
|
+
|
66
|
+
data.fetch(:kafka).each_key do |key|
|
67
|
+
next if key.is_a?(Symbol)
|
68
|
+
|
69
|
+
detected_errors << [[:kafka, key], :key_must_be_a_symbol]
|
70
|
+
end
|
71
|
+
|
72
|
+
detected_errors
|
73
|
+
end
|
74
|
+
|
75
|
+
virtual do |data, errors|
|
76
|
+
next unless errors.empty?
|
77
|
+
|
78
|
+
pause_timeout = data.fetch(:pause_timeout)
|
79
|
+
pause_max_timeout = data.fetch(:pause_max_timeout)
|
80
|
+
|
81
|
+
next if pause_timeout <= pause_max_timeout
|
82
|
+
|
83
|
+
[[%i[pause_timeout], :max_timeout_vs_pause_max_timeout]]
|
84
|
+
end
|
85
|
+
|
86
|
+
virtual do |data, errors|
|
87
|
+
next unless errors.empty?
|
88
|
+
|
89
|
+
shutdown_timeout = data.fetch(:shutdown_timeout)
|
90
|
+
max_wait_time = data.fetch(:max_wait_time)
|
91
|
+
|
92
|
+
next if max_wait_time < shutdown_timeout
|
93
|
+
|
94
|
+
[[%i[shutdown_timeout], :shutdown_timeout_vs_max_wait_time]]
|
18
95
|
end
|
19
96
|
end
|
20
97
|
end
|
@@ -3,208 +3,40 @@
|
|
3
3
|
module Karafka
|
4
4
|
module Contracts
|
5
5
|
# Contract for single full route (consumer group + topics) validation.
|
6
|
-
class ConsumerGroup <
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
# Available sasl scram mechanism of authentication (plus nil)
|
15
|
-
SASL_SCRAM_MECHANISMS ||= %w[sha256 sha512].freeze
|
16
|
-
|
17
|
-
# Internal contract for sub-validating topics schema
|
18
|
-
TOPIC_CONTRACT = ConsumerGroupTopic.new.freeze
|
19
|
-
|
20
|
-
private_constant :TOPIC_CONTRACT
|
21
|
-
|
22
|
-
params do
|
23
|
-
required(:id).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)
|
24
|
-
required(:topics).value(:array, :filled?)
|
25
|
-
required(:seed_brokers).value(:array, :filled?)
|
26
|
-
required(:session_timeout).filled { int? | float? }
|
27
|
-
required(:pause_timeout).maybe(%i[integer float]) { filled? > gteq?(0) }
|
28
|
-
required(:pause_max_timeout).maybe(%i[integer float]) { filled? > gteq?(0) }
|
29
|
-
required(:pause_exponential_backoff).filled(:bool?)
|
30
|
-
required(:offset_commit_interval) { int? | float? }
|
31
|
-
required(:offset_commit_threshold).filled(:int?)
|
32
|
-
required(:offset_retention_time).maybe(:integer)
|
33
|
-
required(:heartbeat_interval).filled { (int? | float?) & gteq?(0) }
|
34
|
-
required(:fetcher_max_queue_size).filled(:int?, gt?: 0)
|
35
|
-
required(:assignment_strategy).value(:any)
|
36
|
-
required(:connect_timeout).filled { (int? | float?) & gt?(0) }
|
37
|
-
required(:reconnect_timeout).filled { (int? | float?) & gteq?(0) }
|
38
|
-
required(:socket_timeout).filled { (int? | float?) & gt?(0) }
|
39
|
-
required(:min_bytes).filled(:int?, gt?: 0)
|
40
|
-
required(:max_bytes).filled(:int?, gt?: 0)
|
41
|
-
required(:max_wait_time).filled { (int? | float?) & gteq?(0) }
|
42
|
-
required(:batch_fetching).filled(:bool?)
|
43
|
-
|
44
|
-
%i[
|
45
|
-
ssl_ca_cert
|
46
|
-
ssl_ca_cert_file_path
|
47
|
-
ssl_client_cert
|
48
|
-
ssl_client_cert_key
|
49
|
-
ssl_client_cert_chain
|
50
|
-
ssl_client_cert_key_password
|
51
|
-
sasl_gssapi_principal
|
52
|
-
sasl_gssapi_keytab
|
53
|
-
sasl_plain_authzid
|
54
|
-
sasl_plain_username
|
55
|
-
sasl_plain_password
|
56
|
-
sasl_scram_username
|
57
|
-
sasl_scram_password
|
58
|
-
].each do |encryption_attribute|
|
59
|
-
optional(encryption_attribute).maybe(:str?)
|
60
|
-
end
|
61
|
-
|
62
|
-
optional(:ssl_verify_hostname).maybe(:bool?)
|
63
|
-
optional(:ssl_ca_certs_from_system).maybe(:bool?)
|
64
|
-
optional(:sasl_over_ssl).maybe(:bool?)
|
65
|
-
optional(:sasl_oauth_token_provider).value(:any)
|
66
|
-
|
67
|
-
# It's not with other encryptions as it has some more rules
|
68
|
-
optional(:sasl_scram_mechanism)
|
69
|
-
.maybe(:str?, included_in?: SASL_SCRAM_MECHANISMS)
|
70
|
-
end
|
71
|
-
|
72
|
-
# Uri rule to check if uri is in a Karafka acceptable format
|
73
|
-
rule(:seed_brokers) do
|
74
|
-
if value.is_a?(Array) && !value.all?(&method(:kafka_uri?))
|
75
|
-
key.failure(:invalid_broker_schema)
|
76
|
-
end
|
77
|
-
end
|
78
|
-
|
79
|
-
rule(:topics) do
|
80
|
-
if value.is_a?(Array)
|
81
|
-
names = value.map { |topic| topic[:name] }
|
82
|
-
|
83
|
-
key.failure(:topics_names_not_unique) if names.size != names.uniq.size
|
84
|
-
end
|
85
|
-
end
|
86
|
-
|
87
|
-
rule(:topics) do
|
88
|
-
if value.is_a?(Array)
|
89
|
-
value.each_with_index do |topic, index|
|
90
|
-
TOPIC_CONTRACT.call(topic).errors.each do |error|
|
91
|
-
key([:topics, index, error.path[0]]).failure(error.text)
|
92
|
-
end
|
93
|
-
end
|
94
|
-
end
|
95
|
-
end
|
96
|
-
|
97
|
-
rule(:assignment_strategy) do
|
98
|
-
key.failure(:does_not_respond_to_call) unless value.respond_to?(:call)
|
99
|
-
end
|
100
|
-
|
101
|
-
rule(:ssl_client_cert, :ssl_client_cert_key) do
|
102
|
-
if values[:ssl_client_cert] && !values[:ssl_client_cert_key]
|
103
|
-
key(:ssl_client_cert_key).failure(:ssl_client_cert_with_ssl_client_cert_key)
|
104
|
-
end
|
105
|
-
end
|
106
|
-
|
107
|
-
rule(:ssl_client_cert, :ssl_client_cert_key) do
|
108
|
-
if values[:ssl_client_cert_key] && !values[:ssl_client_cert]
|
109
|
-
key(:ssl_client_cert).failure(:ssl_client_cert_key_with_ssl_client_cert)
|
110
|
-
end
|
6
|
+
class ConsumerGroup < Base
|
7
|
+
configure do |config|
|
8
|
+
config.error_messages = YAML.safe_load(
|
9
|
+
File.read(
|
10
|
+
File.join(Karafka.gem_root, 'config', 'errors.yml')
|
11
|
+
)
|
12
|
+
).fetch('en').fetch('validations').fetch('consumer_group')
|
111
13
|
end
|
112
14
|
|
113
|
-
|
114
|
-
|
115
|
-
key(:ssl_client_cert).failure(:ssl_client_cert_chain_with_ssl_client_cert)
|
116
|
-
end
|
117
|
-
end
|
15
|
+
required(:id) { |id| id.is_a?(String) && Contracts::TOPIC_REGEXP.match?(id) }
|
16
|
+
required(:topics) { |topics| topics.is_a?(Array) && !topics.empty? }
|
118
17
|
|
119
|
-
|
120
|
-
|
121
|
-
key(:ssl_client_cert).failure(:ssl_client_cert_chain_with_ssl_client_cert_key)
|
122
|
-
end
|
123
|
-
end
|
124
|
-
|
125
|
-
rule(:ssl_client_cert_key_password, :ssl_client_cert_key) do
|
126
|
-
if values[:ssl_client_cert_key_password] && !values[:ssl_client_cert_key]
|
127
|
-
key(:ssl_client_cert_key).failure(:ssl_client_cert_key_password_with_ssl_client_cert_key)
|
128
|
-
end
|
129
|
-
end
|
18
|
+
virtual do |data, errors|
|
19
|
+
next unless errors.empty?
|
130
20
|
|
131
|
-
|
132
|
-
key.failure(:invalid_certificate) if value && !valid_certificate?(value)
|
133
|
-
end
|
21
|
+
names = data.fetch(:topics).map { |topic| topic[:name] }
|
134
22
|
|
135
|
-
|
136
|
-
key.failure(:invalid_certificate) if value && !valid_certificate?(value)
|
137
|
-
end
|
23
|
+
next if names.size == names.uniq.size
|
138
24
|
|
139
|
-
|
140
|
-
if value
|
141
|
-
if File.exist?(value)
|
142
|
-
key.failure(:invalid_certificate_from_path) unless valid_certificate?(File.read(value))
|
143
|
-
else
|
144
|
-
key.failure(:does_not_exist)
|
145
|
-
end
|
146
|
-
end
|
25
|
+
[[%i[topics], :names_not_unique]]
|
147
26
|
end
|
148
27
|
|
149
|
-
|
150
|
-
|
151
|
-
end
|
28
|
+
virtual do |data, errors|
|
29
|
+
next unless errors.empty?
|
152
30
|
|
153
|
-
|
154
|
-
key.failure(:invalid_certificate) if value && !valid_certificate?(value)
|
155
|
-
end
|
31
|
+
fetched_errors = []
|
156
32
|
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
rule(:max_wait_time, :socket_timeout) do
|
162
|
-
max_wait_time = values[:max_wait_time]
|
163
|
-
socket_timeout = values[:socket_timeout]
|
164
|
-
|
165
|
-
if socket_timeout.is_a?(Numeric) &&
|
166
|
-
max_wait_time.is_a?(Numeric) &&
|
167
|
-
max_wait_time > socket_timeout
|
168
|
-
|
169
|
-
key(:max_wait_time).failure(:max_wait_time_limit)
|
170
|
-
end
|
171
|
-
end
|
172
|
-
|
173
|
-
rule(:pause_timeout, :pause_max_timeout, :pause_exponential_backoff) do
|
174
|
-
if values[:pause_exponential_backoff]
|
175
|
-
if values[:pause_timeout].to_i > values[:pause_max_timeout].to_i
|
176
|
-
key(:pause_max_timeout).failure(:max_timeout_size_for_exponential)
|
33
|
+
data.fetch(:topics).each do |topic|
|
34
|
+
ConsumerGroupTopic.new.call(topic).errors.each do |key, value|
|
35
|
+
fetched_errors << [[topic, key].flatten, value]
|
177
36
|
end
|
178
37
|
end
|
179
|
-
end
|
180
|
-
|
181
|
-
private
|
182
|
-
|
183
|
-
# @param value [String] potential RSA key value
|
184
|
-
# @return [Boolean] is the given string a valid RSA key
|
185
|
-
def valid_private_key?(value)
|
186
|
-
OpenSSL::PKey.read(value)
|
187
|
-
true
|
188
|
-
rescue OpenSSL::PKey::PKeyError
|
189
|
-
false
|
190
|
-
end
|
191
|
-
|
192
|
-
# @param value [String] potential X509 cert value
|
193
|
-
# @return [Boolean] is the given string a valid X509 cert
|
194
|
-
def valid_certificate?(value)
|
195
|
-
OpenSSL::X509::Certificate.new(value)
|
196
|
-
true
|
197
|
-
rescue OpenSSL::X509::CertificateError
|
198
|
-
false
|
199
|
-
end
|
200
38
|
|
201
|
-
|
202
|
-
# @return [Boolean] true if it is a kafka uri, otherwise false
|
203
|
-
def kafka_uri?(value)
|
204
|
-
uri = URI.parse(value)
|
205
|
-
URI_SCHEMES.include?(uri.scheme) && uri.port
|
206
|
-
rescue URI::InvalidURIError
|
207
|
-
false
|
39
|
+
fetched_errors
|
208
40
|
end
|
209
41
|
end
|
210
42
|
end
|
@@ -2,17 +2,40 @@
|
|
2
2
|
|
3
3
|
module Karafka
|
4
4
|
module Contracts
|
5
|
-
# Consumer group topic validation rules
|
6
|
-
class ConsumerGroupTopic <
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
5
|
+
# Consumer group topic validation rules.
|
6
|
+
class ConsumerGroupTopic < Base
|
7
|
+
configure do |config|
|
8
|
+
config.error_messages = YAML.safe_load(
|
9
|
+
File.read(
|
10
|
+
File.join(Karafka.gem_root, 'config', 'errors.yml')
|
11
|
+
)
|
12
|
+
).fetch('en').fetch('validations').fetch('consumer_group_topic')
|
13
|
+
end
|
14
|
+
|
15
|
+
required(:consumer) { |consumer_group| !consumer_group.nil? }
|
16
|
+
required(:deserializer) { |deserializer| !deserializer.nil? }
|
17
|
+
required(:id) { |id| id.is_a?(String) && Contracts::TOPIC_REGEXP.match?(id) }
|
18
|
+
required(:kafka) { |kafka| kafka.is_a?(Hash) && !kafka.empty? }
|
19
|
+
required(:max_messages) { |mm| mm.is_a?(Integer) && mm >= 1 }
|
20
|
+
required(:initial_offset) { |io| %w[earliest latest].include?(io) }
|
21
|
+
required(:max_wait_time) { |mwt| mwt.is_a?(Integer) && mwt >= 10 }
|
22
|
+
required(:manual_offset_management) { |mmm| [true, false].include?(mmm) }
|
23
|
+
required(:name) { |name| name.is_a?(String) && Contracts::TOPIC_REGEXP.match?(name) }
|
24
|
+
|
25
|
+
virtual do |data, errors|
|
26
|
+
next unless errors.empty?
|
27
|
+
|
28
|
+
value = data.fetch(:kafka)
|
29
|
+
|
30
|
+
begin
|
31
|
+
# This will trigger rdkafka validations that we catch and re-map the info and use dry
|
32
|
+
# compatible format
|
33
|
+
Rdkafka::Config.new(value).send(:native_config)
|
34
|
+
|
35
|
+
nil
|
36
|
+
rescue Rdkafka::Config::ConfigError => e
|
37
|
+
[[%w[kafka], e.message]]
|
38
|
+
end
|
16
39
|
end
|
17
40
|
end
|
18
41
|
end
|
@@ -2,29 +2,30 @@
|
|
2
2
|
|
3
3
|
module Karafka
|
4
4
|
module Contracts
|
5
|
-
# Contract for validating correctness of the server cli command options
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
optional(:daemon).filled(:bool?)
|
14
|
-
optional(:consumer_groups).value(:array, :filled?)
|
5
|
+
# Contract for validating correctness of the server cli command options.
|
6
|
+
class ServerCliOptions < Base
|
7
|
+
configure do |config|
|
8
|
+
config.error_messages = YAML.safe_load(
|
9
|
+
File.read(
|
10
|
+
File.join(Karafka.gem_root, 'config', 'errors.yml')
|
11
|
+
)
|
12
|
+
).fetch('en').fetch('validations').fetch('server_cli_options')
|
15
13
|
end
|
16
14
|
|
17
|
-
|
18
|
-
|
19
|
-
|
15
|
+
optional(:consumer_groups) { |cg| cg.is_a?(Array) && !cg.empty? }
|
16
|
+
|
17
|
+
virtual do |data, errors|
|
18
|
+
next unless errors.empty?
|
19
|
+
next unless data.key?(:consumer_groups)
|
20
|
+
|
21
|
+
value = data.fetch(:consumer_groups)
|
20
22
|
|
21
|
-
rule(:consumer_groups) do
|
22
23
|
# If there were no consumer_groups declared in the server cli, it means that we will
|
23
24
|
# run all of them and no need to validate them here at all
|
24
|
-
if
|
25
|
-
|
26
|
-
|
27
|
-
|
25
|
+
next if value.nil?
|
26
|
+
next if (value - Karafka::App.config.internal.routing.builder.map(&:name)).empty?
|
27
|
+
|
28
|
+
[[%i[consumer_groups], :consumer_groups_inclusion]]
|
28
29
|
end
|
29
30
|
end
|
30
31
|
end
|
data/lib/karafka/contracts.rb
CHANGED
data/lib/karafka/env.rb
ADDED
@@ -0,0 +1,46 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
# Env management class to get and set environment for Karafka
|
5
|
+
class Env < String
|
6
|
+
# Keys where we look for environment details for Karafka
|
7
|
+
LOOKUP_ENV_KEYS = %w[
|
8
|
+
KARAFKA_ENV
|
9
|
+
RACK_ENV
|
10
|
+
RAILS_ENV
|
11
|
+
].freeze
|
12
|
+
|
13
|
+
# Default fallback env
|
14
|
+
DEFAULT_ENV = 'development'
|
15
|
+
|
16
|
+
private_constant :LOOKUP_ENV_KEYS, :DEFAULT_ENV
|
17
|
+
|
18
|
+
# @return [Karafka::Env] env object
|
19
|
+
# @note Will load appropriate environment automatically
|
20
|
+
def initialize
|
21
|
+
super('')
|
22
|
+
|
23
|
+
LOOKUP_ENV_KEYS
|
24
|
+
.map { |key| ENV[key] }
|
25
|
+
.compact
|
26
|
+
.first
|
27
|
+
.then { |env| env || DEFAULT_ENV }
|
28
|
+
.then { |env| replace(env) }
|
29
|
+
end
|
30
|
+
|
31
|
+
# @param method_name [String] method name
|
32
|
+
# @param include_private [Boolean] should we include private methods as well
|
33
|
+
# @return [Boolean] true if we respond to a given missing method, otherwise false
|
34
|
+
def respond_to_missing?(method_name, include_private = false)
|
35
|
+
(method_name[-1] == '?') || super
|
36
|
+
end
|
37
|
+
|
38
|
+
# Reacts to missing methods, from which some might be the env checks.
|
39
|
+
# If the method ends with '?' we assume, that it is an env check
|
40
|
+
# @param method_name [String] method name for missing or env name with question mark
|
41
|
+
# @param arguments [Array] any arguments that we pass to the method
|
42
|
+
def method_missing(method_name, *arguments)
|
43
|
+
method_name[-1] == '?' ? self == method_name[0..-2] : super
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
data/lib/karafka/errors.rb
CHANGED
@@ -6,12 +6,6 @@ module Karafka
|
|
6
6
|
# Base class for all the Karafka internal errors
|
7
7
|
BaseError = Class.new(StandardError)
|
8
8
|
|
9
|
-
# Should be raised when we have that that we cannot serialize
|
10
|
-
SerializationError = Class.new(BaseError)
|
11
|
-
|
12
|
-
# Should be raised when we tried to deserialize incoming data but we failed
|
13
|
-
DeserializationError = Class.new(BaseError)
|
14
|
-
|
15
9
|
# Raised when router receives topic name which does not correspond with any routes
|
16
10
|
# This can only happen in a case when:
|
17
11
|
# - you've received a message and we cannot match it with a consumer
|
@@ -24,28 +18,34 @@ module Karafka
|
|
24
18
|
# @see https://github.com/karafka/karafka/issues/135
|
25
19
|
NonMatchingRouteError = Class.new(BaseError)
|
26
20
|
|
27
|
-
# Raised when we don't use or use responder not in the way it expected to based on the
|
28
|
-
# topics usage definitions
|
29
|
-
InvalidResponderUsageError = Class.new(BaseError)
|
30
|
-
|
31
|
-
# Raised when options that we provide to the responder to respond aren't what the contract
|
32
|
-
# requires
|
33
|
-
InvalidResponderMessageOptionsError = Class.new(BaseError)
|
34
|
-
|
35
21
|
# Raised when configuration doesn't match with validation contract
|
36
22
|
InvalidConfigurationError = Class.new(BaseError)
|
37
23
|
|
38
24
|
# Raised when we try to use Karafka CLI commands (except install) without a boot file
|
39
25
|
MissingBootFileError = Class.new(BaseError)
|
40
26
|
|
41
|
-
# Raised when we want to read a persisted thread messages consumer but it is unavailable
|
42
|
-
# This should never happen and if it does, please contact us
|
43
|
-
MissingClientError = Class.new(BaseError)
|
44
|
-
|
45
|
-
# Raised when want to hook up to an event that is not registered and supported
|
46
|
-
UnregisteredMonitorEventError = Class.new(BaseError)
|
47
|
-
|
48
27
|
# Raised when we've waited enough for shutting down a non-responsive process
|
49
28
|
ForcefulShutdownError = Class.new(BaseError)
|
29
|
+
|
30
|
+
# Raised when the jobs queue receives a job that should not be received as it would cause
|
31
|
+
# the processing to go out of sync. We should never process in parallel data from the same
|
32
|
+
# topic partition (unless virtual partitions apply)
|
33
|
+
JobsQueueSynchronizationError = Class.new(BaseError)
|
34
|
+
|
35
|
+
# Raised when given topic is not found while expected
|
36
|
+
TopicNotFoundError = Class.new(BaseError)
|
37
|
+
|
38
|
+
# This should never happen. Please open an issue if it does.
|
39
|
+
UnsupportedCaseError = Class.new(BaseError)
|
40
|
+
|
41
|
+
# Raised when the license token is not valid
|
42
|
+
InvalidLicenseTokenError = Class.new(BaseError)
|
43
|
+
|
44
|
+
# Used to instrument this error into the error notifications
|
45
|
+
# We do not raise it so we won't crash deployed systems
|
46
|
+
ExpiredLicenseTokenError = Class.new(BaseError)
|
47
|
+
|
48
|
+
# This should never happen. Please open an issue if it does.
|
49
|
+
InvalidCoordinatorState = Class.new(BaseError)
|
50
50
|
end
|
51
51
|
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Helpers
|
5
|
+
# Allows a given class to run async in a separate thread. Provides also few methods we may
|
6
|
+
# want to use to control the underlying thread
|
7
|
+
#
|
8
|
+
# @note Thread running code needs to manage it's own exceptions. If they leak out, they will
|
9
|
+
# abort thread on exception.
|
10
|
+
module Async
|
11
|
+
class << self
|
12
|
+
# Adds forwardable to redirect thread-based control methods to the underlying thread that
|
13
|
+
# runs the async operations
|
14
|
+
#
|
15
|
+
# @param base [Class] class we're including this module in
|
16
|
+
def included(base)
|
17
|
+
base.extend ::Forwardable
|
18
|
+
|
19
|
+
base.def_delegators :@thread, :join, :terminate, :alive?
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
# Runs the `#call` method in a new thread
|
24
|
+
def async_call
|
25
|
+
@thread = Thread.new do
|
26
|
+
Thread.current.abort_on_exception = true
|
27
|
+
|
28
|
+
call
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Helpers
|
5
|
+
# Simple wrapper for adding colors to strings
|
6
|
+
module Colorize
|
7
|
+
# @param string [String] string we want to have in green
|
8
|
+
# @return [String] green string
|
9
|
+
def green(string)
|
10
|
+
"\033[0;32m#{string}\033[0m"
|
11
|
+
end
|
12
|
+
|
13
|
+
# @param string [String] string we want to have in red
|
14
|
+
# @return [String] red string
|
15
|
+
def red(string)
|
16
|
+
"\033[0;31m#{string}\033[0m"
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -1,10 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module Karafka
|
4
|
-
# Module containing classes and methods that provide some additional functionalities
|
4
|
+
# Module containing classes and methods that provide some additional helper functionalities.
|
5
5
|
module Helpers
|
6
6
|
# @note Taken from http://stackoverflow.com/questions/6407141
|
7
|
-
# Multidelegator is used to delegate calls to multiple targets
|
7
|
+
# Multidelegator is used to delegate calls to multiple targets.
|
8
8
|
class MultiDelegator
|
9
9
|
# @param targets to which we want to delegate methods
|
10
10
|
def initialize(*targets)
|
@@ -0,0 +1,40 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Instrumentation
|
5
|
+
# Callbacks used to transport things from rdkafka
|
6
|
+
module Callbacks
|
7
|
+
# Callback that kicks in when consumer error occurs and is published in a background thread
|
8
|
+
class Error
|
9
|
+
# @param subscription_group_id [String] id of the current subscription group instance
|
10
|
+
# @param consumer_group_id [String] id of the current consumer group
|
11
|
+
# @param client_name [String] rdkafka client name
|
12
|
+
# @param monitor [WaterDrop::Instrumentation::Monitor] monitor we are using
|
13
|
+
def initialize(subscription_group_id, consumer_group_id, client_name, monitor)
|
14
|
+
@subscription_group_id = subscription_group_id
|
15
|
+
@consumer_group_id = consumer_group_id
|
16
|
+
@client_name = client_name
|
17
|
+
@monitor = monitor
|
18
|
+
end
|
19
|
+
|
20
|
+
# Runs the instrumentation monitor with error
|
21
|
+
# @param client_name [String] rdkafka client name
|
22
|
+
# @param error [Rdkafka::Error] error that occurred
|
23
|
+
# @note It will only instrument on errors of the client of our consumer
|
24
|
+
def call(client_name, error)
|
25
|
+
# Emit only errors related to our client
|
26
|
+
# Same as with statistics (mor explanation there)
|
27
|
+
return unless @client_name == client_name
|
28
|
+
|
29
|
+
@monitor.instrument(
|
30
|
+
'error.occurred',
|
31
|
+
subscription_group_id: @subscription_group_id,
|
32
|
+
consumer_group_id: @consumer_group_id,
|
33
|
+
type: 'librdkafka.error',
|
34
|
+
error: error
|
35
|
+
)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|