karafka 1.2.2 → 1.4.0.rc1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +2 -0
- data.tar.gz.sig +0 -0
- data/.coditsu/ci.yml +3 -0
- data/.console_irbrc +1 -3
- data/.diffend.yml +3 -0
- data/.github/FUNDING.yml +3 -0
- data/.github/ISSUE_TEMPLATE/bug_report.md +50 -0
- data/.github/ISSUE_TEMPLATE/feature_request.md +20 -0
- data/.github/workflows/ci.yml +52 -0
- data/.gitignore +1 -0
- data/.ruby-version +1 -1
- data/CHANGELOG.md +157 -13
- data/CODE_OF_CONDUCT.md +1 -1
- data/CONTRIBUTING.md +1 -1
- data/Gemfile +5 -2
- data/Gemfile.lock +95 -79
- data/README.md +15 -3
- data/bin/karafka +1 -1
- data/certs/mensfeld.pem +25 -0
- data/config/errors.yml +38 -5
- data/docker-compose.yml +17 -0
- data/karafka.gemspec +19 -13
- data/lib/karafka.rb +10 -16
- data/lib/karafka/app.rb +14 -6
- data/lib/karafka/attributes_map.rb +13 -18
- data/lib/karafka/base_consumer.rb +19 -30
- data/lib/karafka/base_responder.rb +51 -29
- data/lib/karafka/cli.rb +2 -2
- data/lib/karafka/cli/console.rb +11 -9
- data/lib/karafka/cli/flow.rb +9 -7
- data/lib/karafka/cli/info.rb +4 -2
- data/lib/karafka/cli/install.rb +30 -6
- data/lib/karafka/cli/server.rb +11 -6
- data/lib/karafka/code_reloader.rb +67 -0
- data/lib/karafka/connection/{config_adapter.rb → api_adapter.rb} +62 -21
- data/lib/karafka/connection/batch_delegator.rb +55 -0
- data/lib/karafka/connection/builder.rb +18 -0
- data/lib/karafka/connection/client.rb +40 -40
- data/lib/karafka/connection/listener.rb +26 -15
- data/lib/karafka/connection/message_delegator.rb +36 -0
- data/lib/karafka/consumers/batch_metadata.rb +10 -0
- data/lib/karafka/consumers/callbacks.rb +32 -15
- data/lib/karafka/consumers/includer.rb +31 -18
- data/lib/karafka/consumers/responders.rb +2 -2
- data/lib/karafka/contracts.rb +10 -0
- data/lib/karafka/contracts/config.rb +21 -0
- data/lib/karafka/contracts/consumer_group.rb +206 -0
- data/lib/karafka/contracts/consumer_group_topic.rb +19 -0
- data/lib/karafka/contracts/responder_usage.rb +54 -0
- data/lib/karafka/contracts/server_cli_options.rb +31 -0
- data/lib/karafka/errors.rb +17 -19
- data/lib/karafka/fetcher.rb +28 -30
- data/lib/karafka/helpers/class_matcher.rb +12 -2
- data/lib/karafka/helpers/config_retriever.rb +1 -1
- data/lib/karafka/helpers/inflector.rb +26 -0
- data/lib/karafka/helpers/multi_delegator.rb +0 -1
- data/lib/karafka/instrumentation/logger.rb +9 -6
- data/lib/karafka/instrumentation/monitor.rb +15 -9
- data/lib/karafka/instrumentation/proctitle_listener.rb +36 -0
- data/lib/karafka/instrumentation/stdout_listener.rb +140 -0
- data/lib/karafka/params/batch_metadata.rb +26 -0
- data/lib/karafka/params/builders/batch_metadata.rb +30 -0
- data/lib/karafka/params/builders/params.rb +38 -0
- data/lib/karafka/params/builders/params_batch.rb +25 -0
- data/lib/karafka/params/metadata.rb +20 -0
- data/lib/karafka/params/params.rb +50 -0
- data/lib/karafka/params/params_batch.rb +35 -21
- data/lib/karafka/patches/ruby_kafka.rb +21 -8
- data/lib/karafka/persistence/client.rb +15 -11
- data/lib/karafka/persistence/{consumer.rb → consumers.rb} +20 -13
- data/lib/karafka/persistence/topics.rb +48 -0
- data/lib/karafka/process.rb +0 -4
- data/lib/karafka/responders/builder.rb +1 -1
- data/lib/karafka/responders/topic.rb +6 -8
- data/lib/karafka/routing/builder.rb +36 -8
- data/lib/karafka/routing/consumer_group.rb +1 -1
- data/lib/karafka/routing/consumer_mapper.rb +9 -9
- data/lib/karafka/routing/proxy.rb +10 -1
- data/lib/karafka/routing/topic.rb +5 -3
- data/lib/karafka/routing/topic_mapper.rb +16 -18
- data/lib/karafka/serialization/json/deserializer.rb +27 -0
- data/lib/karafka/serialization/json/serializer.rb +31 -0
- data/lib/karafka/server.rb +34 -49
- data/lib/karafka/setup/config.rb +74 -40
- data/lib/karafka/setup/configurators/water_drop.rb +7 -3
- data/lib/karafka/setup/dsl.rb +0 -1
- data/lib/karafka/status.rb +7 -3
- data/lib/karafka/templates/{application_consumer.rb.example → application_consumer.rb.erb} +2 -1
- data/lib/karafka/templates/{application_responder.rb.example → application_responder.rb.erb} +0 -0
- data/lib/karafka/templates/karafka.rb.erb +92 -0
- data/lib/karafka/version.rb +1 -1
- metadata +97 -73
- metadata.gz.sig +4 -0
- data/.travis.yml +0 -13
- data/lib/karafka/callbacks.rb +0 -30
- data/lib/karafka/callbacks/config.rb +0 -22
- data/lib/karafka/callbacks/dsl.rb +0 -16
- data/lib/karafka/connection/delegator.rb +0 -46
- data/lib/karafka/instrumentation/listener.rb +0 -112
- data/lib/karafka/loader.rb +0 -28
- data/lib/karafka/params/dsl.rb +0 -156
- data/lib/karafka/parsers/json.rb +0 -38
- data/lib/karafka/patches/dry_configurable.rb +0 -35
- data/lib/karafka/persistence/topic.rb +0 -29
- data/lib/karafka/schemas/config.rb +0 -24
- data/lib/karafka/schemas/consumer_group.rb +0 -77
- data/lib/karafka/schemas/consumer_group_topic.rb +0 -18
- data/lib/karafka/schemas/responder_usage.rb +0 -39
- data/lib/karafka/schemas/server_cli_options.rb +0 -43
- data/lib/karafka/setup/configurators/base.rb +0 -29
- data/lib/karafka/setup/configurators/params.rb +0 -25
- data/lib/karafka/templates/karafka.rb.example +0 -54
@@ -29,7 +29,7 @@ module Karafka
|
|
29
29
|
|
30
30
|
# Builds a topic representation inside of a current consumer group route
|
31
31
|
# @param name [String, Symbol] name of topic to which we want to subscribe
|
32
|
-
# @
|
32
|
+
# @param block [Proc] block that we want to evaluate in the topic context
|
33
33
|
# @return [Karafka::Routing::Topic] newly built topic instance
|
34
34
|
def topic=(name, &block)
|
35
35
|
topic = Topic.new(name, self)
|
@@ -4,29 +4,29 @@ module Karafka
|
|
4
4
|
module Routing
|
5
5
|
# Default consumer mapper that builds consumer ids based on app id and consumer group name
|
6
6
|
# Different mapper can be used in case of preexisting consumer names or for applying
|
7
|
-
# other naming conventions not compatible
|
7
|
+
# other naming conventions not compatible with Karafka client_id + consumer name concept
|
8
8
|
#
|
9
9
|
# @example Mapper for using consumer groups without a client_id prefix
|
10
|
-
#
|
11
|
-
# def
|
10
|
+
# class MyMapper
|
11
|
+
# def call(raw_consumer_group_name)
|
12
12
|
# raw_consumer_group_name
|
13
13
|
# end
|
14
14
|
# end
|
15
15
|
#
|
16
16
|
# @example Mapper for replacing "_" with "." in topic names
|
17
|
-
#
|
18
|
-
# def
|
17
|
+
# class MyMapper
|
18
|
+
# def call(raw_consumer_group_name)
|
19
19
|
# [
|
20
|
-
#
|
20
|
+
# Karafka::Helpers::Inflector.map(Karafka::App.config.client_id.to_s),
|
21
21
|
# raw_consumer_group_name
|
22
22
|
# ].join('_').gsub('_', '.')
|
23
23
|
# end
|
24
24
|
# end
|
25
|
-
|
25
|
+
class ConsumerMapper
|
26
26
|
# @param raw_consumer_group_name [String, Symbol] string or symbolized consumer group name
|
27
27
|
# @return [String] remapped final consumer group name
|
28
|
-
def
|
29
|
-
client_name =
|
28
|
+
def call(raw_consumer_group_name)
|
29
|
+
client_name = Karafka::Helpers::Inflector.map(Karafka::App.config.client_id.to_s)
|
30
30
|
"#{client_name}_#{raw_consumer_group_name}"
|
31
31
|
end
|
32
32
|
end
|
@@ -14,22 +14,31 @@ module Karafka
|
|
14
14
|
!
|
15
15
|
].freeze
|
16
16
|
|
17
|
+
private_constant :IGNORED_POSTFIXES
|
18
|
+
|
17
19
|
# @param target [Object] target object to which we proxy any DSL call
|
18
|
-
# @
|
20
|
+
# @param block [Proc] block that we want to evaluate in the proxy context
|
19
21
|
def initialize(target, &block)
|
20
22
|
@target = target
|
21
23
|
instance_eval(&block)
|
22
24
|
end
|
23
25
|
|
24
26
|
# Translates the no "=" DSL of routing into elements assignments on target
|
27
|
+
# @param method_name [Symbol] name of the missing method
|
28
|
+
# @param arguments [Array] array with it's arguments
|
29
|
+
# @param block [Proc] block provided to the method
|
25
30
|
def method_missing(method_name, *arguments, &block)
|
26
31
|
return super unless respond_to_missing?(method_name)
|
32
|
+
|
27
33
|
@target.public_send(:"#{method_name}=", *arguments, &block)
|
28
34
|
end
|
29
35
|
|
30
36
|
# Tells whether or not a given element exists on the target
|
37
|
+
# @param method_name [Symbol] name of the missing method
|
38
|
+
# @param include_private [Boolean] should we include private in the check as well
|
31
39
|
def respond_to_missing?(method_name, include_private = false)
|
32
40
|
return false if IGNORED_POSTFIXES.any? { |postfix| method_name.to_s.end_with?(postfix) }
|
41
|
+
|
33
42
|
@target.respond_to?(:"#{method_name}=", include_private) || super
|
34
43
|
end
|
35
44
|
end
|
@@ -7,10 +7,13 @@ module Karafka
|
|
7
7
|
# It is a part of Karafka's DSL
|
8
8
|
class Topic
|
9
9
|
extend Helpers::ConfigRetriever
|
10
|
+
extend Forwardable
|
10
11
|
|
11
12
|
attr_reader :id, :consumer_group
|
12
13
|
attr_accessor :consumer
|
13
14
|
|
15
|
+
def_delegator :@consumer_group, :batch_fetching
|
16
|
+
|
14
17
|
# @param [String, Symbol] name of a topic on which we want to listen
|
15
18
|
# @param consumer_group [Karafka::Routing::ConsumerGroup] owning consumer group of this topic
|
16
19
|
def initialize(name, consumer_group)
|
@@ -19,7 +22,7 @@ module Karafka
|
|
19
22
|
@attributes = {}
|
20
23
|
# @note We use identifier related to the consumer group that owns a topic, because from
|
21
24
|
# Karafka 0.6 we can handle multiple Kafka instances with the same process and we can
|
22
|
-
# have same topic name across
|
25
|
+
# have same topic name across multiple Kafkas
|
23
26
|
@id = "#{consumer_group.id}_#{@name}"
|
24
27
|
end
|
25
28
|
|
@@ -29,12 +32,11 @@ module Karafka
|
|
29
32
|
# example for Sidekiq
|
30
33
|
def build
|
31
34
|
Karafka::AttributesMap.topic.each { |attr| send(attr) }
|
32
|
-
consumer&.topic = self
|
33
35
|
self
|
34
36
|
end
|
35
37
|
|
36
38
|
# @return [Class, nil] Class (not an instance) of a responder that should respond from
|
37
|
-
# consumer back to Kafka (
|
39
|
+
# consumer back to Kafka (useful for piping data flows)
|
38
40
|
def responder
|
39
41
|
@responder ||= Karafka::Responders::Builder.new(consumer).build
|
40
42
|
end
|
@@ -8,7 +8,7 @@ module Karafka
|
|
8
8
|
# routes and responders
|
9
9
|
#
|
10
10
|
# @example Mapper for mapping prefixed topics
|
11
|
-
#
|
11
|
+
# class MyMapper
|
12
12
|
# PREFIX = "my_user_name."
|
13
13
|
#
|
14
14
|
# def incoming(topic)
|
@@ -21,7 +21,7 @@ module Karafka
|
|
21
21
|
# end
|
22
22
|
#
|
23
23
|
# @example Mapper for replacing "." with "_" in topic names
|
24
|
-
#
|
24
|
+
# class MyMapper
|
25
25
|
# PREFIX = "my_user_name."
|
26
26
|
#
|
27
27
|
# def incoming(topic)
|
@@ -32,23 +32,21 @@ module Karafka
|
|
32
32
|
# topic.to_s.gsub('_', '.')
|
33
33
|
# end
|
34
34
|
# end
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
end
|
35
|
+
class TopicMapper
|
36
|
+
# @param topic [String, Symbol] topic
|
37
|
+
# @return [String, Symbol] same topic as on input
|
38
|
+
# @example
|
39
|
+
# incoming('topic_name') #=> 'topic_name'
|
40
|
+
def incoming(topic)
|
41
|
+
topic
|
42
|
+
end
|
44
43
|
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
end
|
44
|
+
# @param topic [String, Symbol] topic
|
45
|
+
# @return [String, Symbol] same topic as on input
|
46
|
+
# @example
|
47
|
+
# outgoing('topic_name') #=> 'topic_name'
|
48
|
+
def outgoing(topic)
|
49
|
+
topic
|
52
50
|
end
|
53
51
|
end
|
54
52
|
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
# Module for all supported by default serialization and deserialization ways
|
5
|
+
module Serialization
|
6
|
+
# Namespace for json ser/der
|
7
|
+
module Json
|
8
|
+
# Default Karafka Json deserializer for loading JSON data
|
9
|
+
class Deserializer
|
10
|
+
# @param params [Karafka::Params::Params] Full params object that we want to deserialize
|
11
|
+
# @return [Hash] hash with deserialized JSON data
|
12
|
+
# @example
|
13
|
+
# params = {
|
14
|
+
# 'payload' => "{\"a\":1}",
|
15
|
+
# 'topic' => 'my-topic',
|
16
|
+
# 'headers' => { 'message_type' => :test }
|
17
|
+
# }
|
18
|
+
# Deserializer.call(params) #=> { 'a' => 1 }
|
19
|
+
def call(params)
|
20
|
+
params.raw_payload.nil? ? nil : ::JSON.parse(params.raw_payload)
|
21
|
+
rescue ::JSON::ParserError => e
|
22
|
+
raise ::Karafka::Errors::DeserializationError, e
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
# Module for all supported by default serialization and deserialization ways
|
5
|
+
module Serialization
|
6
|
+
module Json
|
7
|
+
# Default Karafka Json serializer for serializing data
|
8
|
+
class Serializer
|
9
|
+
# @param content [Object] any object that we want to convert to a json string
|
10
|
+
# @return [String] Valid JSON string containing serialized data
|
11
|
+
# @raise [Karafka::Errors::SerializationError] raised when we don't have a way to
|
12
|
+
# serialize provided data to json
|
13
|
+
# @note When string is passed to this method, we assume that it is already a json
|
14
|
+
# string and we don't serialize it again. This allows us to serialize data before
|
15
|
+
# it is being forwarded to this serializer if we want to have a custom (not that simple)
|
16
|
+
# json serialization
|
17
|
+
#
|
18
|
+
# @example From an ActiveRecord object
|
19
|
+
# Serializer.call(Repository.first) #=> "{\"repository\":{\"id\":\"04b504e0\"}}"
|
20
|
+
# @example From a string (no changes)
|
21
|
+
# Serializer.call("{\"a\":1}") #=> "{\"a\":1}"
|
22
|
+
def call(content)
|
23
|
+
return content if content.is_a?(String)
|
24
|
+
return content.to_json if content.respond_to?(:to_json)
|
25
|
+
|
26
|
+
raise Karafka::Errors::SerializationError, content
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
data/lib/karafka/server.rb
CHANGED
@@ -6,9 +6,14 @@ module Karafka
|
|
6
6
|
@consumer_threads = Concurrent::Array.new
|
7
7
|
|
8
8
|
# How long should we sleep between checks on shutting down consumers
|
9
|
-
SUPERVISION_SLEEP = 1
|
9
|
+
SUPERVISION_SLEEP = 0.1
|
10
10
|
# What system exit code should we use when we terminated forcefully
|
11
11
|
FORCEFUL_EXIT_CODE = 2
|
12
|
+
# This factor allows us to calculate how many times we have to sleep before
|
13
|
+
# a forceful shutdown
|
14
|
+
SUPERVISION_CHECK_FACTOR = (1 / SUPERVISION_SLEEP)
|
15
|
+
|
16
|
+
private_constant :SUPERVISION_SLEEP, :FORCEFUL_EXIT_CODE, :SUPERVISION_CHECK_FACTOR
|
12
17
|
|
13
18
|
class << self
|
14
19
|
# Set of consuming threads. Each consumer thread contains a single consumer
|
@@ -19,10 +24,10 @@ module Karafka
|
|
19
24
|
|
20
25
|
# Method which runs app
|
21
26
|
def run
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
27
|
+
process.on_sigint { stop_supervised }
|
28
|
+
process.on_sigquit { stop_supervised }
|
29
|
+
process.on_sigterm { stop_supervised }
|
30
|
+
run_supervised
|
26
31
|
end
|
27
32
|
|
28
33
|
# @return [Array<String>] array with names of consumer groups that should be consumed in a
|
@@ -36,65 +41,45 @@ module Karafka
|
|
36
41
|
|
37
42
|
# @return [Karafka::Process] process wrapper instance used to catch system signal calls
|
38
43
|
def process
|
39
|
-
Karafka::
|
40
|
-
end
|
41
|
-
|
42
|
-
# What should happen when we decide to quit with sigint
|
43
|
-
def bind_on_sigint
|
44
|
-
process.on_sigint { stop_supervised }
|
45
|
-
end
|
46
|
-
|
47
|
-
# What should happen when we decide to quit with sigquit
|
48
|
-
def bind_on_sigquit
|
49
|
-
process.on_sigquit { stop_supervised }
|
50
|
-
end
|
51
|
-
|
52
|
-
# What should happen when we decide to quit with sigterm
|
53
|
-
def bind_on_sigterm
|
54
|
-
process.on_sigterm { stop_supervised }
|
44
|
+
Karafka::App.config.internal.process
|
55
45
|
end
|
56
46
|
|
57
47
|
# Starts Karafka with a supervision
|
58
48
|
# @note We don't need to sleep because Karafka::Fetcher is locking and waiting to
|
59
|
-
# finish loop (and it won't happen until we
|
60
|
-
def
|
61
|
-
process.supervise
|
62
|
-
|
63
|
-
|
64
|
-
end
|
49
|
+
# finish loop (and it won't happen until we explicitly want to stop)
|
50
|
+
def run_supervised
|
51
|
+
process.supervise
|
52
|
+
Karafka::App.run!
|
53
|
+
Karafka::App.config.internal.fetcher.call
|
65
54
|
end
|
66
55
|
|
67
56
|
# Stops Karafka with a supervision (as long as there is a shutdown timeout)
|
68
|
-
# If consumers won't stop in a given
|
57
|
+
# If consumers won't stop in a given time frame, it will force them to exit
|
69
58
|
def stop_supervised
|
70
|
-
# Because this is called in the trap context, there is a chance that instrumentation
|
71
|
-
# listeners contain things that aren't allowed from within a trap context.
|
72
|
-
# To bypass that (instead of telling users not to do things they need to)
|
73
|
-
# we spin up a thread to instrument server.stop and server.stop.error and wait until
|
74
|
-
# they're finished
|
75
|
-
Thread.new { Karafka.monitor.instrument('server.stop', {}) }.join
|
76
|
-
|
77
59
|
Karafka::App.stop!
|
78
|
-
|
79
|
-
#
|
80
|
-
|
81
|
-
|
82
|
-
#
|
83
|
-
#
|
84
|
-
|
85
|
-
|
86
|
-
|
60
|
+
|
61
|
+
# Temporary patch until https://github.com/dry-rb/dry-configurable/issues/93 is fixed
|
62
|
+
timeout = Thread.new { Karafka::App.config.shutdown_timeout }.join.value
|
63
|
+
|
64
|
+
# We check from time to time (for the timeout period) if all the threads finished
|
65
|
+
# their work and if so, we can just return and normal shutdown process will take place
|
66
|
+
(timeout * SUPERVISION_CHECK_FACTOR).to_i.times do
|
67
|
+
if consumer_threads.count(&:alive?).zero?
|
68
|
+
Thread.new { Karafka.monitor.instrument('app.stopped') }.join
|
69
|
+
return
|
70
|
+
end
|
71
|
+
|
87
72
|
sleep SUPERVISION_SLEEP
|
88
73
|
end
|
89
74
|
|
90
|
-
raise Errors::
|
91
|
-
rescue Errors::
|
92
|
-
Thread.new { Karafka.monitor.instrument('
|
75
|
+
raise Errors::ForcefulShutdownError
|
76
|
+
rescue Errors::ForcefulShutdownError => e
|
77
|
+
Thread.new { Karafka.monitor.instrument('app.stopping.error', error: e) }.join
|
93
78
|
# We're done waiting, lets kill them!
|
94
79
|
consumer_threads.each(&:terminate)
|
95
80
|
|
96
|
-
# exit is not within the instrumentation as it would not trigger due to exit
|
97
|
-
Kernel.exit FORCEFUL_EXIT_CODE
|
81
|
+
# exit! is not within the instrumentation as it would not trigger due to exit
|
82
|
+
Kernel.exit! FORCEFUL_EXIT_CODE
|
98
83
|
end
|
99
84
|
end
|
100
85
|
end
|
data/lib/karafka/setup/config.rb
CHANGED
@@ -8,12 +8,16 @@ module Karafka
|
|
8
8
|
# @note If you want to do some configurations after all of this is done, please add to
|
9
9
|
# karafka/config a proper file (needs to inherit from Karafka::Setup::Configurators::Base
|
10
10
|
# and implement setup method) after that everything will happen automatically
|
11
|
-
# @note This config object allows to create a 1 level
|
11
|
+
# @note This config object allows to create a 1 level nesting (nodes) only. This should be
|
12
12
|
# enough and will still keep the code simple
|
13
13
|
# @see Karafka::Setup::Configurators::Base for more details about configurators api
|
14
14
|
class Config
|
15
15
|
extend Dry::Configurable
|
16
|
-
|
16
|
+
|
17
|
+
# Contract for checking the config provided by the user
|
18
|
+
CONTRACT = Karafka::Contracts::Config.new.freeze
|
19
|
+
|
20
|
+
private_constant :CONTRACT
|
17
21
|
|
18
22
|
# Available settings
|
19
23
|
# option client_id [String] kafka client_id - used to provide
|
@@ -22,21 +26,23 @@ module Karafka
|
|
22
26
|
# What backend do we want to use to process messages
|
23
27
|
setting :backend, :inline
|
24
28
|
# option logger [Instance] logger that we want to use
|
25
|
-
setting :logger,
|
29
|
+
setting :logger, ::Karafka::Instrumentation::Logger.new
|
26
30
|
# option monitor [Instance] monitor that we will to use (defaults to Karafka::Monitor)
|
27
|
-
setting :monitor,
|
31
|
+
setting :monitor, ::Karafka::Instrumentation::Monitor.new
|
28
32
|
# Mapper used to remap consumer groups ids, so in case users migrate from other tools
|
29
33
|
# or they need to maintain their own internal consumer group naming conventions, they
|
30
34
|
# can easily do it, replacing the default client_id + consumer name pattern concept
|
31
|
-
setting :consumer_mapper,
|
32
|
-
# Mapper used to remap names of topics, so we can have a clean internal topic
|
35
|
+
setting :consumer_mapper, Routing::ConsumerMapper.new
|
36
|
+
# Mapper used to remap names of topics, so we can have a clean internal topic naming
|
33
37
|
# despite using any Kafka provider that uses namespacing, etc
|
34
38
|
# It needs to implement two methods:
|
35
39
|
# - #incoming - for remapping from the incoming message to our internal format
|
36
40
|
# - #outgoing - for remapping from internal topic name into outgoing message
|
37
|
-
setting :topic_mapper,
|
38
|
-
# Default
|
39
|
-
setting :
|
41
|
+
setting :topic_mapper, Routing::TopicMapper.new
|
42
|
+
# Default serializer for converting whatever we want to send to kafka to json
|
43
|
+
setting :serializer, Karafka::Serialization::Json::Serializer.new
|
44
|
+
# Default deserializer for converting incoming data into ruby objects
|
45
|
+
setting :deserializer, Karafka::Serialization::Json::Deserializer.new
|
40
46
|
# If batch_fetching is true, we will fetch kafka messages in batches instead of 1 by 1
|
41
47
|
# @note Fetching does not equal consuming, see batch_consuming description for details
|
42
48
|
setting :batch_fetching, true
|
@@ -44,37 +50,28 @@ module Karafka
|
|
44
50
|
# #params_batch will contain params received from Kafka (may be more than 1) so we can
|
45
51
|
# process them in batches
|
46
52
|
setting :batch_consuming, false
|
47
|
-
# Should we operate in a single consumer instance across multiple batches of messages,
|
48
|
-
# from the same partition or should we build a new one for each incoming batch.
|
49
|
-
# Disabling that can be useful when you want to create a new consumer instance for each
|
50
|
-
# incoming batch. It's disabled by default, not to create more objects that needed
|
51
|
-
# on each batch
|
52
|
-
setting :persistent, true
|
53
53
|
# option shutdown_timeout [Integer, nil] the number of seconds after which Karafka no
|
54
|
-
# longer wait for the consumers to stop gracefully but instead we force
|
55
|
-
#
|
56
|
-
# @note Keep in mind, that if your business logic
|
57
|
-
# @note If set to nil, it won't forcefully shutdown the process at all.
|
54
|
+
# longer wait for the consumers to stop gracefully but instead we force terminate
|
55
|
+
# everything.
|
58
56
|
setting :shutdown_timeout, 60
|
59
|
-
# option params_base_class [Class] base class for params class initialization
|
60
|
-
# This can be either a Hash or a HashWithIndifferentAccess depending on your
|
61
|
-
# requirements. Note, that by using HashWithIndifferentAccess, you remove some of the
|
62
|
-
# performance in favor of convenience. This can be useful especially if you already use
|
63
|
-
# it with Rails, etc
|
64
|
-
setting :params_base_class, Hash
|
65
57
|
|
66
58
|
# option kafka [Hash] - optional - kafka configuration options
|
67
59
|
setting :kafka do
|
68
60
|
# Array with at least one host
|
69
|
-
setting :seed_brokers
|
61
|
+
setting :seed_brokers, %w[kafka://127.0.0.1:9092]
|
70
62
|
# option session_timeout [Integer] the number of seconds after which, if a client
|
71
63
|
# hasn't contacted the Kafka cluster, it will be kicked out of the group.
|
72
64
|
setting :session_timeout, 30
|
73
65
|
# Time that a given partition will be paused from fetching messages, when message
|
74
66
|
# consumption fails. It allows us to process other partitions, while the error is being
|
75
67
|
# resolved and also "slows" things down, so it prevents from "eating" up all messages and
|
76
|
-
# consuming them with failed code
|
68
|
+
# consuming them with failed code. Use `nil` if you want to pause forever and never retry.
|
77
69
|
setting :pause_timeout, 10
|
70
|
+
# option pause_max_timeout [Integer, nil] the maximum number of seconds to pause for,
|
71
|
+
# or `nil` if no maximum should be enforced.
|
72
|
+
setting :pause_max_timeout, nil
|
73
|
+
# option pause_exponential_backoff [Boolean] whether to enable exponential backoff
|
74
|
+
setting :pause_exponential_backoff, false
|
78
75
|
# option offset_commit_interval [Integer] the interval between offset commits,
|
79
76
|
# in seconds.
|
80
77
|
setting :offset_commit_interval, 10
|
@@ -85,6 +82,13 @@ module Karafka
|
|
85
82
|
# option heartbeat_interval [Integer] the interval between heartbeats; must be less
|
86
83
|
# than the session window.
|
87
84
|
setting :heartbeat_interval, 10
|
85
|
+
# option offset_retention_time [Integer] The length of the retention window, known as
|
86
|
+
# offset retention time
|
87
|
+
setting :offset_retention_time, nil
|
88
|
+
# option fetcher_max_queue_size [Integer] max number of items in the fetch queue that
|
89
|
+
# are stored for further processing. Note, that each item in the queue represents a
|
90
|
+
# response from a single broker
|
91
|
+
setting :fetcher_max_queue_size, 10
|
88
92
|
# option max_bytes_per_partition [Integer] the maximum amount of data fetched
|
89
93
|
# from a single partition at a time.
|
90
94
|
setting :max_bytes_per_partition, 1_048_576
|
@@ -111,9 +115,6 @@ module Karafka
|
|
111
115
|
# option reconnect_timeout [Integer] How long should we wait before trying to reconnect to
|
112
116
|
# Kafka cluster that went down (in seconds)
|
113
117
|
setting :reconnect_timeout, 5
|
114
|
-
# option offset_retention_time [Integer] The length of the retention window, known as
|
115
|
-
# offset retention time
|
116
|
-
setting :offset_retention_time, nil
|
117
118
|
# option connect_timeout [Integer] Sets the number of seconds to wait while connecting to
|
118
119
|
# a broker for the first time. When ruby-kafka initializes, it needs to connect to at
|
119
120
|
# least one host.
|
@@ -123,6 +124,8 @@ module Karafka
|
|
123
124
|
# will be killed. Note that some Kafka operations are by definition long-running, such as
|
124
125
|
# waiting for new messages to arrive in a partition, so don't set this value too low
|
125
126
|
setting :socket_timeout, 30
|
127
|
+
# option partitioner [Object, nil] the partitioner that should be used by the client
|
128
|
+
setting :partitioner, nil
|
126
129
|
|
127
130
|
# SSL authentication related settings
|
128
131
|
# option ca_cert [String, nil] SSL CA certificate
|
@@ -132,6 +135,8 @@ module Karafka
|
|
132
135
|
# option ssl_ca_certs_from_system [Boolean] Use the CA certs from your system's default
|
133
136
|
# certificate store
|
134
137
|
setting :ssl_ca_certs_from_system, false
|
138
|
+
# option ssl_verify_hostname [Boolean] Verify the hostname for client certs
|
139
|
+
setting :ssl_verify_hostname, true
|
135
140
|
# option ssl_client_cert [String, nil] SSL client certificate
|
136
141
|
setting :ssl_client_cert, nil
|
137
142
|
# option ssl_client_cert_key [String, nil] SSL client certificate password
|
@@ -152,10 +157,39 @@ module Karafka
|
|
152
157
|
setting :sasl_scram_password, nil
|
153
158
|
# option sasl_scram_mechanism [String, nil] Scram mechanism, either 'sha256' or 'sha512'
|
154
159
|
setting :sasl_scram_mechanism, nil
|
160
|
+
# option sasl_over_ssl [Boolean] whether to enforce SSL with SASL
|
161
|
+
setting :sasl_over_ssl, true
|
162
|
+
# option ssl_client_cert_chain [String, nil] client cert chain or nil if not used
|
163
|
+
setting :ssl_client_cert_chain, nil
|
164
|
+
# option ssl_client_cert_key_password [String, nil] the password required to read
|
165
|
+
# the ssl_client_cert_key
|
166
|
+
setting :ssl_client_cert_key_password, nil
|
167
|
+
# @param sasl_oauth_token_provider [Object, nil] OAuthBearer Token Provider instance that
|
168
|
+
# implements method token.
|
169
|
+
setting :sasl_oauth_token_provider, nil
|
170
|
+
end
|
171
|
+
|
172
|
+
# Namespace for internal settings that should not be modified
|
173
|
+
# It's a temporary step to "declassify" several things internally before we move to a
|
174
|
+
# non global state
|
175
|
+
setting :internal do
|
176
|
+
# option routing_builder [Karafka::Routing::Builder] builder instance
|
177
|
+
setting :routing_builder, Routing::Builder.new
|
178
|
+
# option status [Karafka::Status] app status
|
179
|
+
setting :status, Status.new
|
180
|
+
# option process [Karafka::Process] process status
|
181
|
+
# @note In the future, we need to have a single process representation for all the karafka
|
182
|
+
# instances
|
183
|
+
setting :process, Process.new
|
184
|
+
# option fetcher [Karafka::Fetcher] fetcher instance
|
185
|
+
setting :fetcher, Fetcher.new
|
186
|
+
# option configurators [Array<Object>] all configurators that we want to run after
|
187
|
+
# the setup
|
188
|
+
setting :configurators, [Configurators::WaterDrop.new]
|
155
189
|
end
|
156
190
|
|
157
191
|
class << self
|
158
|
-
#
|
192
|
+
# Configuring method
|
159
193
|
# @yield Runs a block of code providing a config singleton instance to it
|
160
194
|
# @yieldparam [Karafka::Setup::Config] Karafka config instance
|
161
195
|
def setup
|
@@ -166,22 +200,22 @@ module Karafka
|
|
166
200
|
# Components are in karafka/config directory and are all loaded one by one
|
167
201
|
# If you want to configure a next component, please add a proper file to config dir
|
168
202
|
def setup_components
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
203
|
+
config
|
204
|
+
.internal
|
205
|
+
.configurators
|
206
|
+
.each { |configurator| configurator.call(config) }
|
173
207
|
end
|
174
208
|
|
175
|
-
# Validate config based on
|
209
|
+
# Validate config based on the config contract
|
176
210
|
# @return [Boolean] true if configuration is valid
|
177
|
-
# @raise [Karafka::Errors::
|
178
|
-
# doesn't match with
|
211
|
+
# @raise [Karafka::Errors::InvalidConfigurationError] raised when configuration
|
212
|
+
# doesn't match with the config contract
|
179
213
|
def validate!
|
180
|
-
validation_result =
|
214
|
+
validation_result = CONTRACT.call(config.to_h)
|
181
215
|
|
182
216
|
return true if validation_result.success?
|
183
217
|
|
184
|
-
raise Errors::
|
218
|
+
raise Errors::InvalidConfigurationError, validation_result.errors.to_h
|
185
219
|
end
|
186
220
|
end
|
187
221
|
end
|