karafka 1.4.13 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +3 -3
- data/.github/workflows/ci.yml +85 -30
- data/.ruby-version +1 -1
- data/CHANGELOG.md +268 -7
- data/CONTRIBUTING.md +10 -19
- data/Gemfile +6 -0
- data/Gemfile.lock +44 -87
- data/LICENSE +17 -0
- data/LICENSE-COMM +89 -0
- data/LICENSE-LGPL +165 -0
- data/README.md +44 -48
- data/bin/benchmarks +85 -0
- data/bin/create_token +22 -0
- data/bin/integrations +237 -0
- data/bin/karafka +4 -0
- data/bin/scenario +29 -0
- data/bin/stress_many +13 -0
- data/bin/stress_one +13 -0
- data/bin/wait_for_kafka +20 -0
- data/certs/karafka-pro.pem +11 -0
- data/config/errors.yml +55 -40
- data/docker-compose.yml +39 -3
- data/karafka.gemspec +11 -17
- data/lib/active_job/karafka.rb +21 -0
- data/lib/active_job/queue_adapters/karafka_adapter.rb +26 -0
- data/lib/karafka/active_job/consumer.rb +26 -0
- data/lib/karafka/active_job/dispatcher.rb +38 -0
- data/lib/karafka/active_job/job_extensions.rb +34 -0
- data/lib/karafka/active_job/job_options_contract.rb +21 -0
- data/lib/karafka/active_job/routing/extensions.rb +31 -0
- data/lib/karafka/app.rb +15 -20
- data/lib/karafka/base_consumer.rb +181 -31
- data/lib/karafka/cli/base.rb +4 -4
- data/lib/karafka/cli/info.rb +43 -9
- data/lib/karafka/cli/install.rb +19 -10
- data/lib/karafka/cli/server.rb +17 -42
- data/lib/karafka/cli.rb +4 -11
- data/lib/karafka/connection/client.rb +385 -90
- data/lib/karafka/connection/listener.rb +246 -38
- data/lib/karafka/connection/listeners_batch.rb +24 -0
- data/lib/karafka/connection/messages_buffer.rb +84 -0
- data/lib/karafka/connection/pauses_manager.rb +46 -0
- data/lib/karafka/connection/raw_messages_buffer.rb +101 -0
- data/lib/karafka/connection/rebalance_manager.rb +78 -0
- data/lib/karafka/contracts/base.rb +17 -0
- data/lib/karafka/contracts/config.rb +88 -11
- data/lib/karafka/contracts/consumer_group.rb +21 -189
- data/lib/karafka/contracts/consumer_group_topic.rb +34 -11
- data/lib/karafka/contracts/server_cli_options.rb +19 -18
- data/lib/karafka/contracts.rb +1 -1
- data/lib/karafka/env.rb +46 -0
- data/lib/karafka/errors.rb +21 -21
- data/lib/karafka/helpers/async.rb +33 -0
- data/lib/karafka/helpers/colorize.rb +20 -0
- data/lib/karafka/helpers/multi_delegator.rb +2 -2
- data/lib/karafka/instrumentation/callbacks/error.rb +40 -0
- data/lib/karafka/instrumentation/callbacks/statistics.rb +41 -0
- data/lib/karafka/instrumentation/logger_listener.rb +164 -0
- data/lib/karafka/instrumentation/monitor.rb +13 -61
- data/lib/karafka/instrumentation/notifications.rb +52 -0
- data/lib/karafka/instrumentation/proctitle_listener.rb +3 -3
- data/lib/karafka/instrumentation/vendors/datadog/dashboard.json +1 -0
- data/lib/karafka/instrumentation/vendors/datadog/listener.rb +232 -0
- data/lib/karafka/instrumentation.rb +21 -0
- data/lib/karafka/licenser.rb +75 -0
- data/lib/karafka/messages/batch_metadata.rb +45 -0
- data/lib/karafka/messages/builders/batch_metadata.rb +40 -0
- data/lib/karafka/messages/builders/message.rb +39 -0
- data/lib/karafka/messages/builders/messages.rb +32 -0
- data/lib/karafka/{params/params.rb → messages/message.rb} +7 -12
- data/lib/karafka/messages/messages.rb +64 -0
- data/lib/karafka/{params → messages}/metadata.rb +4 -6
- data/lib/karafka/messages/seek.rb +9 -0
- data/lib/karafka/patches/rdkafka/consumer.rb +22 -0
- data/lib/karafka/pro/active_job/consumer.rb +46 -0
- data/lib/karafka/pro/active_job/dispatcher.rb +61 -0
- data/lib/karafka/pro/active_job/job_options_contract.rb +32 -0
- data/lib/karafka/pro/base_consumer.rb +82 -0
- data/lib/karafka/pro/contracts/base.rb +21 -0
- data/lib/karafka/pro/contracts/consumer_group.rb +34 -0
- data/lib/karafka/pro/contracts/consumer_group_topic.rb +33 -0
- data/lib/karafka/pro/loader.rb +76 -0
- data/lib/karafka/pro/performance_tracker.rb +80 -0
- data/lib/karafka/pro/processing/coordinator.rb +72 -0
- data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb +37 -0
- data/lib/karafka/pro/processing/jobs_builder.rb +32 -0
- data/lib/karafka/pro/processing/partitioner.rb +60 -0
- data/lib/karafka/pro/processing/scheduler.rb +56 -0
- data/lib/karafka/pro/routing/builder_extensions.rb +30 -0
- data/lib/karafka/pro/routing/topic_extensions.rb +38 -0
- data/lib/karafka/pro.rb +13 -0
- data/lib/karafka/process.rb +1 -0
- data/lib/karafka/processing/coordinator.rb +88 -0
- data/lib/karafka/processing/coordinators_buffer.rb +54 -0
- data/lib/karafka/processing/executor.rb +118 -0
- data/lib/karafka/processing/executors_buffer.rb +88 -0
- data/lib/karafka/processing/jobs/base.rb +51 -0
- data/lib/karafka/processing/jobs/consume.rb +42 -0
- data/lib/karafka/processing/jobs/revoked.rb +22 -0
- data/lib/karafka/processing/jobs/shutdown.rb +23 -0
- data/lib/karafka/processing/jobs_builder.rb +29 -0
- data/lib/karafka/processing/jobs_queue.rb +144 -0
- data/lib/karafka/processing/partitioner.rb +22 -0
- data/lib/karafka/processing/result.rb +29 -0
- data/lib/karafka/processing/scheduler.rb +22 -0
- data/lib/karafka/processing/worker.rb +88 -0
- data/lib/karafka/processing/workers_batch.rb +27 -0
- data/lib/karafka/railtie.rb +113 -0
- data/lib/karafka/routing/builder.rb +15 -24
- data/lib/karafka/routing/consumer_group.rb +11 -19
- data/lib/karafka/routing/consumer_mapper.rb +1 -2
- data/lib/karafka/routing/router.rb +1 -1
- data/lib/karafka/routing/subscription_group.rb +53 -0
- data/lib/karafka/routing/subscription_groups_builder.rb +53 -0
- data/lib/karafka/routing/topic.rb +61 -24
- data/lib/karafka/routing/topics.rb +38 -0
- data/lib/karafka/runner.rb +51 -0
- data/lib/karafka/serialization/json/deserializer.rb +6 -15
- data/lib/karafka/server.rb +67 -26
- data/lib/karafka/setup/config.rb +147 -175
- data/lib/karafka/status.rb +14 -5
- data/lib/karafka/templates/example_consumer.rb.erb +16 -0
- data/lib/karafka/templates/karafka.rb.erb +15 -51
- data/lib/karafka/time_trackers/base.rb +19 -0
- data/lib/karafka/time_trackers/pause.rb +92 -0
- data/lib/karafka/time_trackers/poll.rb +65 -0
- data/lib/karafka/version.rb +1 -1
- data/lib/karafka.rb +38 -17
- data.tar.gz.sig +0 -0
- metadata +118 -120
- metadata.gz.sig +0 -0
- data/MIT-LICENCE +0 -18
- data/lib/karafka/assignment_strategies/round_robin.rb +0 -13
- data/lib/karafka/attributes_map.rb +0 -63
- data/lib/karafka/backends/inline.rb +0 -16
- data/lib/karafka/base_responder.rb +0 -226
- data/lib/karafka/cli/flow.rb +0 -48
- data/lib/karafka/cli/missingno.rb +0 -19
- data/lib/karafka/code_reloader.rb +0 -67
- data/lib/karafka/connection/api_adapter.rb +0 -158
- data/lib/karafka/connection/batch_delegator.rb +0 -55
- data/lib/karafka/connection/builder.rb +0 -23
- data/lib/karafka/connection/message_delegator.rb +0 -36
- data/lib/karafka/consumers/batch_metadata.rb +0 -10
- data/lib/karafka/consumers/callbacks.rb +0 -71
- data/lib/karafka/consumers/includer.rb +0 -64
- data/lib/karafka/consumers/responders.rb +0 -24
- data/lib/karafka/consumers/single_params.rb +0 -15
- data/lib/karafka/contracts/responder_usage.rb +0 -54
- data/lib/karafka/fetcher.rb +0 -42
- data/lib/karafka/helpers/class_matcher.rb +0 -88
- data/lib/karafka/helpers/config_retriever.rb +0 -46
- data/lib/karafka/helpers/inflector.rb +0 -26
- data/lib/karafka/instrumentation/stdout_listener.rb +0 -140
- data/lib/karafka/params/batch_metadata.rb +0 -26
- data/lib/karafka/params/builders/batch_metadata.rb +0 -30
- data/lib/karafka/params/builders/params.rb +0 -38
- data/lib/karafka/params/builders/params_batch.rb +0 -25
- data/lib/karafka/params/params_batch.rb +0 -60
- data/lib/karafka/patches/ruby_kafka.rb +0 -47
- data/lib/karafka/persistence/client.rb +0 -29
- data/lib/karafka/persistence/consumers.rb +0 -45
- data/lib/karafka/persistence/topics.rb +0 -48
- data/lib/karafka/responders/builder.rb +0 -36
- data/lib/karafka/responders/topic.rb +0 -55
- data/lib/karafka/routing/topic_mapper.rb +0 -53
- data/lib/karafka/serialization/json/serializer.rb +0 -31
- data/lib/karafka/setup/configurators/water_drop.rb +0 -36
- data/lib/karafka/templates/application_responder.rb.erb +0 -11
|
@@ -2,19 +2,96 @@
|
|
|
2
2
|
|
|
3
3
|
module Karafka
|
|
4
4
|
module Contracts
|
|
5
|
-
# Contract with validation rules for Karafka configuration details
|
|
5
|
+
# Contract with validation rules for Karafka configuration details.
|
|
6
|
+
#
|
|
6
7
|
# @note There are many more configuration options inside of the
|
|
7
|
-
# Karafka::Setup::Config model, but we don't validate them here as they are
|
|
8
|
+
# `Karafka::Setup::Config` model, but we don't validate them here as they are
|
|
8
9
|
# validated per each route (topic + consumer_group) because they can be overwritten,
|
|
9
|
-
# so we validate all of that once all the routes are defined and ready
|
|
10
|
-
class Config <
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
10
|
+
# so we validate all of that once all the routes are defined and ready.
|
|
11
|
+
class Config < Base
|
|
12
|
+
configure do |config|
|
|
13
|
+
config.error_messages = YAML.safe_load(
|
|
14
|
+
File.read(
|
|
15
|
+
File.join(Karafka.gem_root, 'config', 'errors.yml')
|
|
16
|
+
)
|
|
17
|
+
).fetch('en').fetch('validations').fetch('config')
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# License validity happens in the licenser. Here we do only the simple consistency checks
|
|
21
|
+
nested(:license) do
|
|
22
|
+
required(:token) { |val| [true, false].include?(val) || val.is_a?(String) }
|
|
23
|
+
required(:entity) { |val| val.is_a?(String) }
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
required(:client_id) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
|
|
27
|
+
required(:concurrency) { |val| val.is_a?(Integer) && val.positive? }
|
|
28
|
+
required(:consumer_mapper) { |val| !val.nil? }
|
|
29
|
+
required(:consumer_persistence) { |val| [true, false].include?(val) }
|
|
30
|
+
required(:pause_timeout) { |val| val.is_a?(Integer) && val.positive? }
|
|
31
|
+
required(:pause_max_timeout) { |val| val.is_a?(Integer) && val.positive? }
|
|
32
|
+
required(:pause_with_exponential_backoff) { |val| [true, false].include?(val) }
|
|
33
|
+
required(:shutdown_timeout) { |val| val.is_a?(Integer) && val.positive? }
|
|
34
|
+
required(:max_wait_time) { |val| val.is_a?(Integer) && val.positive? }
|
|
35
|
+
required(:kafka) { |val| val.is_a?(Hash) && !val.empty? }
|
|
36
|
+
|
|
37
|
+
# We validate internals just to be sure, that they are present and working
|
|
38
|
+
nested(:internal) do
|
|
39
|
+
required(:status) { |val| !val.nil? }
|
|
40
|
+
required(:process) { |val| !val.nil? }
|
|
41
|
+
|
|
42
|
+
nested(:routing) do
|
|
43
|
+
required(:builder) { |val| !val.nil? }
|
|
44
|
+
required(:subscription_groups_builder) { |val| !val.nil? }
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
nested(:processing) do
|
|
48
|
+
required(:jobs_builder) { |val| !val.nil? }
|
|
49
|
+
required(:scheduler) { |val| !val.nil? }
|
|
50
|
+
required(:coordinator_class) { |val| !val.nil? }
|
|
51
|
+
required(:partitioner_class) { |val| !val.nil? }
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
nested(:active_job) do
|
|
55
|
+
required(:dispatcher) { |val| !val.nil? }
|
|
56
|
+
required(:job_options_contract) { |val| !val.nil? }
|
|
57
|
+
required(:consumer_class) { |val| !val.nil? }
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
virtual do |data, errors|
|
|
62
|
+
next unless errors.empty?
|
|
63
|
+
|
|
64
|
+
detected_errors = []
|
|
65
|
+
|
|
66
|
+
data.fetch(:kafka).each_key do |key|
|
|
67
|
+
next if key.is_a?(Symbol)
|
|
68
|
+
|
|
69
|
+
detected_errors << [[:kafka, key], :key_must_be_a_symbol]
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
detected_errors
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
virtual do |data, errors|
|
|
76
|
+
next unless errors.empty?
|
|
77
|
+
|
|
78
|
+
pause_timeout = data.fetch(:pause_timeout)
|
|
79
|
+
pause_max_timeout = data.fetch(:pause_max_timeout)
|
|
80
|
+
|
|
81
|
+
next if pause_timeout <= pause_max_timeout
|
|
82
|
+
|
|
83
|
+
[[%i[pause_timeout], :max_timeout_vs_pause_max_timeout]]
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
virtual do |data, errors|
|
|
87
|
+
next unless errors.empty?
|
|
88
|
+
|
|
89
|
+
shutdown_timeout = data.fetch(:shutdown_timeout)
|
|
90
|
+
max_wait_time = data.fetch(:max_wait_time)
|
|
91
|
+
|
|
92
|
+
next if max_wait_time < shutdown_timeout
|
|
93
|
+
|
|
94
|
+
[[%i[shutdown_timeout], :shutdown_timeout_vs_max_wait_time]]
|
|
18
95
|
end
|
|
19
96
|
end
|
|
20
97
|
end
|
|
@@ -3,208 +3,40 @@
|
|
|
3
3
|
module Karafka
|
|
4
4
|
module Contracts
|
|
5
5
|
# Contract for single full route (consumer group + topics) validation.
|
|
6
|
-
class ConsumerGroup <
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
# Available sasl scram mechanism of authentication (plus nil)
|
|
15
|
-
SASL_SCRAM_MECHANISMS ||= %w[sha256 sha512].freeze
|
|
16
|
-
|
|
17
|
-
# Internal contract for sub-validating topics schema
|
|
18
|
-
TOPIC_CONTRACT = ConsumerGroupTopic.new.freeze
|
|
19
|
-
|
|
20
|
-
private_constant :TOPIC_CONTRACT
|
|
21
|
-
|
|
22
|
-
params do
|
|
23
|
-
required(:id).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)
|
|
24
|
-
required(:topics).value(:array, :filled?)
|
|
25
|
-
required(:seed_brokers).value(:array, :filled?)
|
|
26
|
-
required(:session_timeout).filled { int? | float? }
|
|
27
|
-
required(:pause_timeout).maybe(%i[integer float]) { filled? > gteq?(0) }
|
|
28
|
-
required(:pause_max_timeout).maybe(%i[integer float]) { filled? > gteq?(0) }
|
|
29
|
-
required(:pause_exponential_backoff).filled(:bool?)
|
|
30
|
-
required(:offset_commit_interval) { int? | float? }
|
|
31
|
-
required(:offset_commit_threshold).filled(:int?)
|
|
32
|
-
required(:offset_retention_time).maybe(:integer)
|
|
33
|
-
required(:heartbeat_interval).filled { (int? | float?) & gteq?(0) }
|
|
34
|
-
required(:fetcher_max_queue_size).filled(:int?, gt?: 0)
|
|
35
|
-
required(:assignment_strategy).value(:any)
|
|
36
|
-
required(:connect_timeout).filled { (int? | float?) & gt?(0) }
|
|
37
|
-
required(:reconnect_timeout).filled { (int? | float?) & gteq?(0) }
|
|
38
|
-
required(:socket_timeout).filled { (int? | float?) & gt?(0) }
|
|
39
|
-
required(:min_bytes).filled(:int?, gt?: 0)
|
|
40
|
-
required(:max_bytes).filled(:int?, gt?: 0)
|
|
41
|
-
required(:max_wait_time).filled { (int? | float?) & gteq?(0) }
|
|
42
|
-
required(:batch_fetching).filled(:bool?)
|
|
43
|
-
|
|
44
|
-
%i[
|
|
45
|
-
ssl_ca_cert
|
|
46
|
-
ssl_ca_cert_file_path
|
|
47
|
-
ssl_client_cert
|
|
48
|
-
ssl_client_cert_key
|
|
49
|
-
ssl_client_cert_chain
|
|
50
|
-
ssl_client_cert_key_password
|
|
51
|
-
sasl_gssapi_principal
|
|
52
|
-
sasl_gssapi_keytab
|
|
53
|
-
sasl_plain_authzid
|
|
54
|
-
sasl_plain_username
|
|
55
|
-
sasl_plain_password
|
|
56
|
-
sasl_scram_username
|
|
57
|
-
sasl_scram_password
|
|
58
|
-
].each do |encryption_attribute|
|
|
59
|
-
optional(encryption_attribute).maybe(:str?)
|
|
60
|
-
end
|
|
61
|
-
|
|
62
|
-
optional(:ssl_verify_hostname).maybe(:bool?)
|
|
63
|
-
optional(:ssl_ca_certs_from_system).maybe(:bool?)
|
|
64
|
-
optional(:sasl_over_ssl).maybe(:bool?)
|
|
65
|
-
optional(:sasl_oauth_token_provider).value(:any)
|
|
66
|
-
|
|
67
|
-
# It's not with other encryptions as it has some more rules
|
|
68
|
-
optional(:sasl_scram_mechanism)
|
|
69
|
-
.maybe(:str?, included_in?: SASL_SCRAM_MECHANISMS)
|
|
70
|
-
end
|
|
71
|
-
|
|
72
|
-
# Uri rule to check if uri is in a Karafka acceptable format
|
|
73
|
-
rule(:seed_brokers) do
|
|
74
|
-
if value.is_a?(Array) && !value.all?(&method(:kafka_uri?))
|
|
75
|
-
key.failure(:invalid_broker_schema)
|
|
76
|
-
end
|
|
77
|
-
end
|
|
78
|
-
|
|
79
|
-
rule(:topics) do
|
|
80
|
-
if value.is_a?(Array)
|
|
81
|
-
names = value.map { |topic| topic[:name] }
|
|
82
|
-
|
|
83
|
-
key.failure(:topics_names_not_unique) if names.size != names.uniq.size
|
|
84
|
-
end
|
|
85
|
-
end
|
|
86
|
-
|
|
87
|
-
rule(:topics) do
|
|
88
|
-
if value.is_a?(Array)
|
|
89
|
-
value.each_with_index do |topic, index|
|
|
90
|
-
TOPIC_CONTRACT.call(topic).errors.each do |error|
|
|
91
|
-
key([:topics, index, error.path[0]]).failure(error.text)
|
|
92
|
-
end
|
|
93
|
-
end
|
|
94
|
-
end
|
|
95
|
-
end
|
|
96
|
-
|
|
97
|
-
rule(:assignment_strategy) do
|
|
98
|
-
key.failure(:does_not_respond_to_call) unless value.respond_to?(:call)
|
|
99
|
-
end
|
|
100
|
-
|
|
101
|
-
rule(:ssl_client_cert, :ssl_client_cert_key) do
|
|
102
|
-
if values[:ssl_client_cert] && !values[:ssl_client_cert_key]
|
|
103
|
-
key(:ssl_client_cert_key).failure(:ssl_client_cert_with_ssl_client_cert_key)
|
|
104
|
-
end
|
|
105
|
-
end
|
|
106
|
-
|
|
107
|
-
rule(:ssl_client_cert, :ssl_client_cert_key) do
|
|
108
|
-
if values[:ssl_client_cert_key] && !values[:ssl_client_cert]
|
|
109
|
-
key(:ssl_client_cert).failure(:ssl_client_cert_key_with_ssl_client_cert)
|
|
110
|
-
end
|
|
6
|
+
class ConsumerGroup < Base
|
|
7
|
+
configure do |config|
|
|
8
|
+
config.error_messages = YAML.safe_load(
|
|
9
|
+
File.read(
|
|
10
|
+
File.join(Karafka.gem_root, 'config', 'errors.yml')
|
|
11
|
+
)
|
|
12
|
+
).fetch('en').fetch('validations').fetch('consumer_group')
|
|
111
13
|
end
|
|
112
14
|
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
key(:ssl_client_cert).failure(:ssl_client_cert_chain_with_ssl_client_cert)
|
|
116
|
-
end
|
|
117
|
-
end
|
|
15
|
+
required(:id) { |id| id.is_a?(String) && Contracts::TOPIC_REGEXP.match?(id) }
|
|
16
|
+
required(:topics) { |topics| topics.is_a?(Array) && !topics.empty? }
|
|
118
17
|
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
key(:ssl_client_cert).failure(:ssl_client_cert_chain_with_ssl_client_cert_key)
|
|
122
|
-
end
|
|
123
|
-
end
|
|
124
|
-
|
|
125
|
-
rule(:ssl_client_cert_key_password, :ssl_client_cert_key) do
|
|
126
|
-
if values[:ssl_client_cert_key_password] && !values[:ssl_client_cert_key]
|
|
127
|
-
key(:ssl_client_cert_key).failure(:ssl_client_cert_key_password_with_ssl_client_cert_key)
|
|
128
|
-
end
|
|
129
|
-
end
|
|
18
|
+
virtual do |data, errors|
|
|
19
|
+
next unless errors.empty?
|
|
130
20
|
|
|
131
|
-
|
|
132
|
-
key.failure(:invalid_certificate) if value && !valid_certificate?(value)
|
|
133
|
-
end
|
|
21
|
+
names = data.fetch(:topics).map { |topic| topic[:name] }
|
|
134
22
|
|
|
135
|
-
|
|
136
|
-
key.failure(:invalid_certificate) if value && !valid_certificate?(value)
|
|
137
|
-
end
|
|
23
|
+
next if names.size == names.uniq.size
|
|
138
24
|
|
|
139
|
-
|
|
140
|
-
if value
|
|
141
|
-
if File.exist?(value)
|
|
142
|
-
key.failure(:invalid_certificate_from_path) unless valid_certificate?(File.read(value))
|
|
143
|
-
else
|
|
144
|
-
key.failure(:does_not_exist)
|
|
145
|
-
end
|
|
146
|
-
end
|
|
25
|
+
[[%i[topics], :names_not_unique]]
|
|
147
26
|
end
|
|
148
27
|
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
end
|
|
28
|
+
virtual do |data, errors|
|
|
29
|
+
next unless errors.empty?
|
|
152
30
|
|
|
153
|
-
|
|
154
|
-
key.failure(:invalid_certificate) if value && !valid_certificate?(value)
|
|
155
|
-
end
|
|
31
|
+
fetched_errors = []
|
|
156
32
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
rule(:max_wait_time, :socket_timeout) do
|
|
162
|
-
max_wait_time = values[:max_wait_time]
|
|
163
|
-
socket_timeout = values[:socket_timeout]
|
|
164
|
-
|
|
165
|
-
if socket_timeout.is_a?(Numeric) &&
|
|
166
|
-
max_wait_time.is_a?(Numeric) &&
|
|
167
|
-
max_wait_time > socket_timeout
|
|
168
|
-
|
|
169
|
-
key(:max_wait_time).failure(:max_wait_time_limit)
|
|
170
|
-
end
|
|
171
|
-
end
|
|
172
|
-
|
|
173
|
-
rule(:pause_timeout, :pause_max_timeout, :pause_exponential_backoff) do
|
|
174
|
-
if values[:pause_exponential_backoff]
|
|
175
|
-
if values[:pause_timeout].to_i > values[:pause_max_timeout].to_i
|
|
176
|
-
key(:pause_max_timeout).failure(:max_timeout_size_for_exponential)
|
|
33
|
+
data.fetch(:topics).each do |topic|
|
|
34
|
+
ConsumerGroupTopic.new.call(topic).errors.each do |key, value|
|
|
35
|
+
fetched_errors << [[topic, key].flatten, value]
|
|
177
36
|
end
|
|
178
37
|
end
|
|
179
|
-
end
|
|
180
|
-
|
|
181
|
-
private
|
|
182
|
-
|
|
183
|
-
# @param value [String] potential RSA key value
|
|
184
|
-
# @return [Boolean] is the given string a valid RSA key
|
|
185
|
-
def valid_private_key?(value)
|
|
186
|
-
OpenSSL::PKey.read(value)
|
|
187
|
-
true
|
|
188
|
-
rescue OpenSSL::PKey::PKeyError
|
|
189
|
-
false
|
|
190
|
-
end
|
|
191
|
-
|
|
192
|
-
# @param value [String] potential X509 cert value
|
|
193
|
-
# @return [Boolean] is the given string a valid X509 cert
|
|
194
|
-
def valid_certificate?(value)
|
|
195
|
-
OpenSSL::X509::Certificate.new(value)
|
|
196
|
-
true
|
|
197
|
-
rescue OpenSSL::X509::CertificateError
|
|
198
|
-
false
|
|
199
|
-
end
|
|
200
38
|
|
|
201
|
-
|
|
202
|
-
# @return [Boolean] true if it is a kafka uri, otherwise false
|
|
203
|
-
def kafka_uri?(value)
|
|
204
|
-
uri = URI.parse(value)
|
|
205
|
-
URI_SCHEMES.include?(uri.scheme) && uri.port
|
|
206
|
-
rescue URI::InvalidURIError
|
|
207
|
-
false
|
|
39
|
+
fetched_errors
|
|
208
40
|
end
|
|
209
41
|
end
|
|
210
42
|
end
|
|
@@ -2,17 +2,40 @@
|
|
|
2
2
|
|
|
3
3
|
module Karafka
|
|
4
4
|
module Contracts
|
|
5
|
-
# Consumer group topic validation rules
|
|
6
|
-
class ConsumerGroupTopic <
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
5
|
+
# Consumer group topic validation rules.
|
|
6
|
+
class ConsumerGroupTopic < Base
|
|
7
|
+
configure do |config|
|
|
8
|
+
config.error_messages = YAML.safe_load(
|
|
9
|
+
File.read(
|
|
10
|
+
File.join(Karafka.gem_root, 'config', 'errors.yml')
|
|
11
|
+
)
|
|
12
|
+
).fetch('en').fetch('validations').fetch('consumer_group_topic')
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
required(:consumer) { |consumer_group| !consumer_group.nil? }
|
|
16
|
+
required(:deserializer) { |deserializer| !deserializer.nil? }
|
|
17
|
+
required(:id) { |id| id.is_a?(String) && Contracts::TOPIC_REGEXP.match?(id) }
|
|
18
|
+
required(:kafka) { |kafka| kafka.is_a?(Hash) && !kafka.empty? }
|
|
19
|
+
required(:max_messages) { |mm| mm.is_a?(Integer) && mm >= 1 }
|
|
20
|
+
required(:initial_offset) { |io| %w[earliest latest].include?(io) }
|
|
21
|
+
required(:max_wait_time) { |mwt| mwt.is_a?(Integer) && mwt >= 10 }
|
|
22
|
+
required(:manual_offset_management) { |mmm| [true, false].include?(mmm) }
|
|
23
|
+
required(:name) { |name| name.is_a?(String) && Contracts::TOPIC_REGEXP.match?(name) }
|
|
24
|
+
|
|
25
|
+
virtual do |data, errors|
|
|
26
|
+
next unless errors.empty?
|
|
27
|
+
|
|
28
|
+
value = data.fetch(:kafka)
|
|
29
|
+
|
|
30
|
+
begin
|
|
31
|
+
# This will trigger rdkafka validations that we catch and re-map the info and use dry
|
|
32
|
+
# compatible format
|
|
33
|
+
Rdkafka::Config.new(value).send(:native_config)
|
|
34
|
+
|
|
35
|
+
nil
|
|
36
|
+
rescue Rdkafka::Config::ConfigError => e
|
|
37
|
+
[[%w[kafka], e.message]]
|
|
38
|
+
end
|
|
16
39
|
end
|
|
17
40
|
end
|
|
18
41
|
end
|
|
@@ -2,29 +2,30 @@
|
|
|
2
2
|
|
|
3
3
|
module Karafka
|
|
4
4
|
module Contracts
|
|
5
|
-
# Contract for validating correctness of the server cli command options
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
optional(:daemon).filled(:bool?)
|
|
14
|
-
optional(:consumer_groups).value(:array, :filled?)
|
|
5
|
+
# Contract for validating correctness of the server cli command options.
|
|
6
|
+
class ServerCliOptions < Base
|
|
7
|
+
configure do |config|
|
|
8
|
+
config.error_messages = YAML.safe_load(
|
|
9
|
+
File.read(
|
|
10
|
+
File.join(Karafka.gem_root, 'config', 'errors.yml')
|
|
11
|
+
)
|
|
12
|
+
).fetch('en').fetch('validations').fetch('server_cli_options')
|
|
15
13
|
end
|
|
16
14
|
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
15
|
+
optional(:consumer_groups) { |cg| cg.is_a?(Array) && !cg.empty? }
|
|
16
|
+
|
|
17
|
+
virtual do |data, errors|
|
|
18
|
+
next unless errors.empty?
|
|
19
|
+
next unless data.key?(:consumer_groups)
|
|
20
|
+
|
|
21
|
+
value = data.fetch(:consumer_groups)
|
|
20
22
|
|
|
21
|
-
rule(:consumer_groups) do
|
|
22
23
|
# If there were no consumer_groups declared in the server cli, it means that we will
|
|
23
24
|
# run all of them and no need to validate them here at all
|
|
24
|
-
if
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
25
|
+
next if value.nil?
|
|
26
|
+
next if (value - Karafka::App.config.internal.routing.builder.map(&:name)).empty?
|
|
27
|
+
|
|
28
|
+
[[%i[consumer_groups], :consumer_groups_inclusion]]
|
|
28
29
|
end
|
|
29
30
|
end
|
|
30
31
|
end
|
data/lib/karafka/contracts.rb
CHANGED
data/lib/karafka/env.rb
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Karafka
|
|
4
|
+
# Env management class to get and set environment for Karafka
|
|
5
|
+
class Env < String
|
|
6
|
+
# Keys where we look for environment details for Karafka
|
|
7
|
+
LOOKUP_ENV_KEYS = %w[
|
|
8
|
+
KARAFKA_ENV
|
|
9
|
+
RACK_ENV
|
|
10
|
+
RAILS_ENV
|
|
11
|
+
].freeze
|
|
12
|
+
|
|
13
|
+
# Default fallback env
|
|
14
|
+
DEFAULT_ENV = 'development'
|
|
15
|
+
|
|
16
|
+
private_constant :LOOKUP_ENV_KEYS, :DEFAULT_ENV
|
|
17
|
+
|
|
18
|
+
# @return [Karafka::Env] env object
|
|
19
|
+
# @note Will load appropriate environment automatically
|
|
20
|
+
def initialize
|
|
21
|
+
super('')
|
|
22
|
+
|
|
23
|
+
LOOKUP_ENV_KEYS
|
|
24
|
+
.map { |key| ENV[key] }
|
|
25
|
+
.compact
|
|
26
|
+
.first
|
|
27
|
+
.then { |env| env || DEFAULT_ENV }
|
|
28
|
+
.then { |env| replace(env) }
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# @param method_name [String] method name
|
|
32
|
+
# @param include_private [Boolean] should we include private methods as well
|
|
33
|
+
# @return [Boolean] true if we respond to a given missing method, otherwise false
|
|
34
|
+
def respond_to_missing?(method_name, include_private = false)
|
|
35
|
+
(method_name[-1] == '?') || super
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Reacts to missing methods, from which some might be the env checks.
|
|
39
|
+
# If the method ends with '?' we assume, that it is an env check
|
|
40
|
+
# @param method_name [String] method name for missing or env name with question mark
|
|
41
|
+
# @param arguments [Array] any arguments that we pass to the method
|
|
42
|
+
def method_missing(method_name, *arguments)
|
|
43
|
+
method_name[-1] == '?' ? self == method_name[0..-2] : super
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
data/lib/karafka/errors.rb
CHANGED
|
@@ -6,12 +6,6 @@ module Karafka
|
|
|
6
6
|
# Base class for all the Karafka internal errors
|
|
7
7
|
BaseError = Class.new(StandardError)
|
|
8
8
|
|
|
9
|
-
# Should be raised when we have that that we cannot serialize
|
|
10
|
-
SerializationError = Class.new(BaseError)
|
|
11
|
-
|
|
12
|
-
# Should be raised when we tried to deserialize incoming data but we failed
|
|
13
|
-
DeserializationError = Class.new(BaseError)
|
|
14
|
-
|
|
15
9
|
# Raised when router receives topic name which does not correspond with any routes
|
|
16
10
|
# This can only happen in a case when:
|
|
17
11
|
# - you've received a message and we cannot match it with a consumer
|
|
@@ -24,28 +18,34 @@ module Karafka
|
|
|
24
18
|
# @see https://github.com/karafka/karafka/issues/135
|
|
25
19
|
NonMatchingRouteError = Class.new(BaseError)
|
|
26
20
|
|
|
27
|
-
# Raised when we don't use or use responder not in the way it expected to based on the
|
|
28
|
-
# topics usage definitions
|
|
29
|
-
InvalidResponderUsageError = Class.new(BaseError)
|
|
30
|
-
|
|
31
|
-
# Raised when options that we provide to the responder to respond aren't what the contract
|
|
32
|
-
# requires
|
|
33
|
-
InvalidResponderMessageOptionsError = Class.new(BaseError)
|
|
34
|
-
|
|
35
21
|
# Raised when configuration doesn't match with validation contract
|
|
36
22
|
InvalidConfigurationError = Class.new(BaseError)
|
|
37
23
|
|
|
38
24
|
# Raised when we try to use Karafka CLI commands (except install) without a boot file
|
|
39
25
|
MissingBootFileError = Class.new(BaseError)
|
|
40
26
|
|
|
41
|
-
# Raised when we want to read a persisted thread messages consumer but it is unavailable
|
|
42
|
-
# This should never happen and if it does, please contact us
|
|
43
|
-
MissingClientError = Class.new(BaseError)
|
|
44
|
-
|
|
45
|
-
# Raised when want to hook up to an event that is not registered and supported
|
|
46
|
-
UnregisteredMonitorEventError = Class.new(BaseError)
|
|
47
|
-
|
|
48
27
|
# Raised when we've waited enough for shutting down a non-responsive process
|
|
49
28
|
ForcefulShutdownError = Class.new(BaseError)
|
|
29
|
+
|
|
30
|
+
# Raised when the jobs queue receives a job that should not be received as it would cause
|
|
31
|
+
# the processing to go out of sync. We should never process in parallel data from the same
|
|
32
|
+
# topic partition (unless virtual partitions apply)
|
|
33
|
+
JobsQueueSynchronizationError = Class.new(BaseError)
|
|
34
|
+
|
|
35
|
+
# Raised when given topic is not found while expected
|
|
36
|
+
TopicNotFoundError = Class.new(BaseError)
|
|
37
|
+
|
|
38
|
+
# This should never happen. Please open an issue if it does.
|
|
39
|
+
UnsupportedCaseError = Class.new(BaseError)
|
|
40
|
+
|
|
41
|
+
# Raised when the license token is not valid
|
|
42
|
+
InvalidLicenseTokenError = Class.new(BaseError)
|
|
43
|
+
|
|
44
|
+
# Used to instrument this error into the error notifications
|
|
45
|
+
# We do not raise it so we won't crash deployed systems
|
|
46
|
+
ExpiredLicenseTokenError = Class.new(BaseError)
|
|
47
|
+
|
|
48
|
+
# This should never happen. Please open an issue if it does.
|
|
49
|
+
InvalidCoordinatorState = Class.new(BaseError)
|
|
50
50
|
end
|
|
51
51
|
end
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Karafka
|
|
4
|
+
module Helpers
|
|
5
|
+
# Allows a given class to run async in a separate thread. Provides also few methods we may
|
|
6
|
+
# want to use to control the underlying thread
|
|
7
|
+
#
|
|
8
|
+
# @note Thread running code needs to manage it's own exceptions. If they leak out, they will
|
|
9
|
+
# abort thread on exception.
|
|
10
|
+
module Async
|
|
11
|
+
class << self
|
|
12
|
+
# Adds forwardable to redirect thread-based control methods to the underlying thread that
|
|
13
|
+
# runs the async operations
|
|
14
|
+
#
|
|
15
|
+
# @param base [Class] class we're including this module in
|
|
16
|
+
def included(base)
|
|
17
|
+
base.extend ::Forwardable
|
|
18
|
+
|
|
19
|
+
base.def_delegators :@thread, :join, :terminate, :alive?
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# Runs the `#call` method in a new thread
|
|
24
|
+
def async_call
|
|
25
|
+
@thread = Thread.new do
|
|
26
|
+
Thread.current.abort_on_exception = true
|
|
27
|
+
|
|
28
|
+
call
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Karafka
|
|
4
|
+
module Helpers
|
|
5
|
+
# Simple wrapper for adding colors to strings
|
|
6
|
+
module Colorize
|
|
7
|
+
# @param string [String] string we want to have in green
|
|
8
|
+
# @return [String] green string
|
|
9
|
+
def green(string)
|
|
10
|
+
"\033[0;32m#{string}\033[0m"
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
# @param string [String] string we want to have in red
|
|
14
|
+
# @return [String] red string
|
|
15
|
+
def red(string)
|
|
16
|
+
"\033[0;31m#{string}\033[0m"
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Karafka
|
|
4
|
-
# Module containing classes and methods that provide some additional functionalities
|
|
4
|
+
# Module containing classes and methods that provide some additional helper functionalities.
|
|
5
5
|
module Helpers
|
|
6
6
|
# @note Taken from http://stackoverflow.com/questions/6407141
|
|
7
|
-
# Multidelegator is used to delegate calls to multiple targets
|
|
7
|
+
# Multidelegator is used to delegate calls to multiple targets.
|
|
8
8
|
class MultiDelegator
|
|
9
9
|
# @param targets to which we want to delegate methods
|
|
10
10
|
def initialize(*targets)
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Karafka
|
|
4
|
+
module Instrumentation
|
|
5
|
+
# Callbacks used to transport things from rdkafka
|
|
6
|
+
module Callbacks
|
|
7
|
+
# Callback that kicks in when consumer error occurs and is published in a background thread
|
|
8
|
+
class Error
|
|
9
|
+
# @param subscription_group_id [String] id of the current subscription group instance
|
|
10
|
+
# @param consumer_group_id [String] id of the current consumer group
|
|
11
|
+
# @param client_name [String] rdkafka client name
|
|
12
|
+
# @param monitor [WaterDrop::Instrumentation::Monitor] monitor we are using
|
|
13
|
+
def initialize(subscription_group_id, consumer_group_id, client_name, monitor)
|
|
14
|
+
@subscription_group_id = subscription_group_id
|
|
15
|
+
@consumer_group_id = consumer_group_id
|
|
16
|
+
@client_name = client_name
|
|
17
|
+
@monitor = monitor
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# Runs the instrumentation monitor with error
|
|
21
|
+
# @param client_name [String] rdkafka client name
|
|
22
|
+
# @param error [Rdkafka::Error] error that occurred
|
|
23
|
+
# @note It will only instrument on errors of the client of our consumer
|
|
24
|
+
def call(client_name, error)
|
|
25
|
+
# Emit only errors related to our client
|
|
26
|
+
# Same as with statistics (mor explanation there)
|
|
27
|
+
return unless @client_name == client_name
|
|
28
|
+
|
|
29
|
+
@monitor.instrument(
|
|
30
|
+
'error.occurred',
|
|
31
|
+
subscription_group_id: @subscription_group_id,
|
|
32
|
+
consumer_group_id: @consumer_group_id,
|
|
33
|
+
type: 'librdkafka.error',
|
|
34
|
+
error: error
|
|
35
|
+
)
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|