karafka 1.4.0 → 2.0.10
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.github/workflows/ci.yml +89 -18
- data/.ruby-version +1 -1
- data/CHANGELOG.md +365 -1
- data/CONTRIBUTING.md +10 -19
- data/Gemfile +6 -0
- data/Gemfile.lock +56 -112
- data/LICENSE +17 -0
- data/LICENSE-COMM +89 -0
- data/LICENSE-LGPL +165 -0
- data/README.md +61 -68
- data/bin/benchmarks +85 -0
- data/bin/create_token +22 -0
- data/bin/integrations +272 -0
- data/bin/karafka +10 -0
- data/bin/scenario +29 -0
- data/bin/stress_many +13 -0
- data/bin/stress_one +13 -0
- data/certs/cert_chain.pem +26 -0
- data/certs/karafka-pro.pem +11 -0
- data/config/errors.yml +59 -38
- data/docker-compose.yml +10 -3
- data/karafka.gemspec +18 -21
- data/lib/active_job/karafka.rb +21 -0
- data/lib/active_job/queue_adapters/karafka_adapter.rb +26 -0
- data/lib/karafka/active_job/consumer.rb +26 -0
- data/lib/karafka/active_job/dispatcher.rb +38 -0
- data/lib/karafka/active_job/job_extensions.rb +34 -0
- data/lib/karafka/active_job/job_options_contract.rb +21 -0
- data/lib/karafka/active_job/routing/extensions.rb +33 -0
- data/lib/karafka/admin.rb +63 -0
- data/lib/karafka/app.rb +15 -20
- data/lib/karafka/base_consumer.rb +197 -31
- data/lib/karafka/cli/info.rb +44 -10
- data/lib/karafka/cli/install.rb +22 -12
- data/lib/karafka/cli/server.rb +17 -42
- data/lib/karafka/cli.rb +4 -3
- data/lib/karafka/connection/client.rb +379 -89
- data/lib/karafka/connection/listener.rb +250 -38
- data/lib/karafka/connection/listeners_batch.rb +24 -0
- data/lib/karafka/connection/messages_buffer.rb +84 -0
- data/lib/karafka/connection/pauses_manager.rb +46 -0
- data/lib/karafka/connection/raw_messages_buffer.rb +101 -0
- data/lib/karafka/connection/rebalance_manager.rb +78 -0
- data/lib/karafka/contracts/base.rb +17 -0
- data/lib/karafka/contracts/config.rb +88 -11
- data/lib/karafka/contracts/consumer_group.rb +21 -184
- data/lib/karafka/contracts/consumer_group_topic.rb +35 -11
- data/lib/karafka/contracts/server_cli_options.rb +19 -18
- data/lib/karafka/contracts.rb +1 -1
- data/lib/karafka/env.rb +46 -0
- data/lib/karafka/errors.rb +21 -21
- data/lib/karafka/helpers/async.rb +33 -0
- data/lib/karafka/helpers/colorize.rb +20 -0
- data/lib/karafka/helpers/multi_delegator.rb +2 -2
- data/lib/karafka/instrumentation/callbacks/error.rb +40 -0
- data/lib/karafka/instrumentation/callbacks/statistics.rb +41 -0
- data/lib/karafka/instrumentation/logger.rb +6 -10
- data/lib/karafka/instrumentation/logger_listener.rb +174 -0
- data/lib/karafka/instrumentation/monitor.rb +13 -61
- data/lib/karafka/instrumentation/notifications.rb +53 -0
- data/lib/karafka/instrumentation/proctitle_listener.rb +3 -3
- data/lib/karafka/instrumentation/vendors/datadog/dashboard.json +1 -0
- data/lib/karafka/instrumentation/vendors/datadog/listener.rb +232 -0
- data/lib/karafka/instrumentation.rb +21 -0
- data/lib/karafka/licenser.rb +75 -0
- data/lib/karafka/messages/batch_metadata.rb +45 -0
- data/lib/karafka/messages/builders/batch_metadata.rb +39 -0
- data/lib/karafka/messages/builders/message.rb +39 -0
- data/lib/karafka/messages/builders/messages.rb +34 -0
- data/lib/karafka/{params/params.rb → messages/message.rb} +7 -12
- data/lib/karafka/messages/messages.rb +64 -0
- data/lib/karafka/{params → messages}/metadata.rb +4 -6
- data/lib/karafka/messages/seek.rb +9 -0
- data/lib/karafka/patches/rdkafka/consumer.rb +22 -0
- data/lib/karafka/pro/active_job/consumer.rb +46 -0
- data/lib/karafka/pro/active_job/dispatcher.rb +61 -0
- data/lib/karafka/pro/active_job/job_options_contract.rb +32 -0
- data/lib/karafka/pro/base_consumer.rb +107 -0
- data/lib/karafka/pro/contracts/base.rb +21 -0
- data/lib/karafka/pro/contracts/consumer_group.rb +34 -0
- data/lib/karafka/pro/contracts/consumer_group_topic.rb +69 -0
- data/lib/karafka/pro/loader.rb +76 -0
- data/lib/karafka/pro/performance_tracker.rb +80 -0
- data/lib/karafka/pro/processing/coordinator.rb +85 -0
- data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb +38 -0
- data/lib/karafka/pro/processing/jobs_builder.rb +32 -0
- data/lib/karafka/pro/processing/partitioner.rb +58 -0
- data/lib/karafka/pro/processing/scheduler.rb +56 -0
- data/lib/karafka/pro/routing/builder_extensions.rb +30 -0
- data/lib/karafka/pro/routing/topic_extensions.rb +74 -0
- data/lib/karafka/pro.rb +13 -0
- data/lib/karafka/process.rb +1 -0
- data/lib/karafka/processing/coordinator.rb +103 -0
- data/lib/karafka/processing/coordinators_buffer.rb +54 -0
- data/lib/karafka/processing/executor.rb +126 -0
- data/lib/karafka/processing/executors_buffer.rb +88 -0
- data/lib/karafka/processing/jobs/base.rb +55 -0
- data/lib/karafka/processing/jobs/consume.rb +47 -0
- data/lib/karafka/processing/jobs/revoked.rb +22 -0
- data/lib/karafka/processing/jobs/shutdown.rb +23 -0
- data/lib/karafka/processing/jobs_builder.rb +29 -0
- data/lib/karafka/processing/jobs_queue.rb +144 -0
- data/lib/karafka/processing/partitioner.rb +22 -0
- data/lib/karafka/processing/result.rb +37 -0
- data/lib/karafka/processing/scheduler.rb +22 -0
- data/lib/karafka/processing/worker.rb +91 -0
- data/lib/karafka/processing/workers_batch.rb +27 -0
- data/lib/karafka/railtie.rb +127 -0
- data/lib/karafka/routing/builder.rb +26 -23
- data/lib/karafka/routing/consumer_group.rb +37 -17
- data/lib/karafka/routing/consumer_mapper.rb +1 -2
- data/lib/karafka/routing/proxy.rb +9 -16
- data/lib/karafka/routing/router.rb +1 -1
- data/lib/karafka/routing/subscription_group.rb +53 -0
- data/lib/karafka/routing/subscription_groups_builder.rb +54 -0
- data/lib/karafka/routing/topic.rb +65 -24
- data/lib/karafka/routing/topics.rb +38 -0
- data/lib/karafka/runner.rb +51 -0
- data/lib/karafka/serialization/json/deserializer.rb +6 -15
- data/lib/karafka/server.rb +67 -26
- data/lib/karafka/setup/config.rb +153 -175
- data/lib/karafka/status.rb +14 -5
- data/lib/karafka/templates/example_consumer.rb.erb +16 -0
- data/lib/karafka/templates/karafka.rb.erb +17 -55
- data/lib/karafka/time_trackers/base.rb +19 -0
- data/lib/karafka/time_trackers/pause.rb +92 -0
- data/lib/karafka/time_trackers/poll.rb +65 -0
- data/lib/karafka/version.rb +1 -1
- data/lib/karafka.rb +46 -16
- data.tar.gz.sig +0 -0
- metadata +145 -171
- metadata.gz.sig +0 -0
- data/.github/FUNDING.yml +0 -3
- data/MIT-LICENCE +0 -18
- data/certs/mensfeld.pem +0 -25
- data/lib/karafka/attributes_map.rb +0 -62
- data/lib/karafka/backends/inline.rb +0 -16
- data/lib/karafka/base_responder.rb +0 -226
- data/lib/karafka/cli/flow.rb +0 -48
- data/lib/karafka/code_reloader.rb +0 -67
- data/lib/karafka/connection/api_adapter.rb +0 -161
- data/lib/karafka/connection/batch_delegator.rb +0 -55
- data/lib/karafka/connection/builder.rb +0 -18
- data/lib/karafka/connection/message_delegator.rb +0 -36
- data/lib/karafka/consumers/batch_metadata.rb +0 -10
- data/lib/karafka/consumers/callbacks.rb +0 -71
- data/lib/karafka/consumers/includer.rb +0 -64
- data/lib/karafka/consumers/responders.rb +0 -24
- data/lib/karafka/consumers/single_params.rb +0 -15
- data/lib/karafka/contracts/responder_usage.rb +0 -54
- data/lib/karafka/fetcher.rb +0 -42
- data/lib/karafka/helpers/class_matcher.rb +0 -88
- data/lib/karafka/helpers/config_retriever.rb +0 -46
- data/lib/karafka/helpers/inflector.rb +0 -26
- data/lib/karafka/instrumentation/stdout_listener.rb +0 -140
- data/lib/karafka/params/batch_metadata.rb +0 -26
- data/lib/karafka/params/builders/batch_metadata.rb +0 -30
- data/lib/karafka/params/builders/params.rb +0 -38
- data/lib/karafka/params/builders/params_batch.rb +0 -25
- data/lib/karafka/params/params_batch.rb +0 -60
- data/lib/karafka/patches/ruby_kafka.rb +0 -47
- data/lib/karafka/persistence/client.rb +0 -29
- data/lib/karafka/persistence/consumers.rb +0 -45
- data/lib/karafka/persistence/topics.rb +0 -48
- data/lib/karafka/responders/builder.rb +0 -36
- data/lib/karafka/responders/topic.rb +0 -55
- data/lib/karafka/routing/topic_mapper.rb +0 -53
- data/lib/karafka/serialization/json/serializer.rb +0 -31
- data/lib/karafka/setup/configurators/water_drop.rb +0 -36
- data/lib/karafka/templates/application_responder.rb.erb +0 -11
@@ -5,12 +5,16 @@ module Karafka
|
|
5
5
|
# Object used to describe a single consumer group that is going to subscribe to
|
6
6
|
# given topics
|
7
7
|
# It is a part of Karafka's DSL
|
8
|
+
# @note A single consumer group represents Kafka consumer group, but it may not match 1:1 with
|
9
|
+
# subscription groups. There can be more subscription groups than consumer groups
|
8
10
|
class ConsumerGroup
|
9
|
-
|
11
|
+
attr_reader :id, :topics, :name
|
10
12
|
|
11
|
-
|
12
|
-
|
13
|
-
|
13
|
+
# This is a "virtual" attribute that is not building subscription groups.
|
14
|
+
# It allows us to store the "current" subscription group defined in the routing
|
15
|
+
# This subscription group id is then injected into topics, so we can compute the subscription
|
16
|
+
# groups
|
17
|
+
attr_accessor :current_subscription_group_name
|
14
18
|
|
15
19
|
# @param name [String, Symbol] raw name of this consumer group. Raw means, that it does not
|
16
20
|
# yet have an application client_id namespace, this will be added here by default.
|
@@ -19,7 +23,7 @@ module Karafka
|
|
19
23
|
def initialize(name)
|
20
24
|
@name = name
|
21
25
|
@id = Karafka::App.config.consumer_mapper.call(name)
|
22
|
-
@topics = []
|
26
|
+
@topics = Topics.new([])
|
23
27
|
end
|
24
28
|
|
25
29
|
# @return [Boolean] true if this consumer group should be active in our current process
|
@@ -33,28 +37,44 @@ module Karafka
|
|
33
37
|
# @return [Karafka::Routing::Topic] newly built topic instance
|
34
38
|
def topic=(name, &block)
|
35
39
|
topic = Topic.new(name, self)
|
36
|
-
@topics << Proxy.new(topic, &block).target
|
37
|
-
@topics.last
|
40
|
+
@topics << Proxy.new(topic, &block).target
|
41
|
+
built_topic = @topics.last
|
42
|
+
# We overwrite it conditionally in case it was not set by the user inline in the topic
|
43
|
+
# block definition
|
44
|
+
built_topic.subscription_group ||= current_subscription_group_name
|
45
|
+
built_topic
|
38
46
|
end
|
39
47
|
|
40
|
-
|
41
|
-
|
48
|
+
# Assigns the current subscription group id based on the defined one and allows for further
|
49
|
+
# topic definition
|
50
|
+
# @param name [String, Symbol]
|
51
|
+
# @param block [Proc] block that may include topics definitions
|
52
|
+
def subscription_group=(name, &block)
|
53
|
+
# We cast it here, so the routing supports symbol based but that's anyhow later on
|
54
|
+
# validated as a string
|
55
|
+
self.current_subscription_group_name = name.to_s
|
56
|
+
|
57
|
+
Proxy.new(self, &block)
|
58
|
+
|
59
|
+
# We need to reset the current subscription group after it is used, so it won't leak
|
60
|
+
# outside to other topics that would be defined without a defined subscription group
|
61
|
+
self.current_subscription_group_name = nil
|
62
|
+
end
|
63
|
+
|
64
|
+
# @return [Array<Routing::SubscriptionGroup>] all the subscription groups build based on
|
65
|
+
# the consumer group topics
|
66
|
+
def subscription_groups
|
67
|
+
App.config.internal.routing.subscription_groups_builder.call(topics)
|
42
68
|
end
|
43
69
|
|
44
70
|
# Hashed version of consumer group that can be used for validation purposes
|
45
71
|
# @return [Hash] hash with consumer group attributes including serialized to hash
|
46
72
|
# topics inside of it.
|
47
73
|
def to_h
|
48
|
-
|
74
|
+
{
|
49
75
|
topics: topics.map(&:to_h),
|
50
76
|
id: id
|
51
|
-
}
|
52
|
-
|
53
|
-
Karafka::AttributesMap.consumer_group.each do |attribute|
|
54
|
-
result[attribute] = public_send(attribute)
|
55
|
-
end
|
56
|
-
|
57
|
-
result
|
77
|
+
}.freeze
|
58
78
|
end
|
59
79
|
end
|
60
80
|
end
|
@@ -26,8 +26,7 @@ module Karafka
|
|
26
26
|
# @param raw_consumer_group_name [String, Symbol] string or symbolized consumer group name
|
27
27
|
# @return [String] remapped final consumer group name
|
28
28
|
def call(raw_consumer_group_name)
|
29
|
-
|
30
|
-
"#{client_name}_#{raw_consumer_group_name}"
|
29
|
+
"#{Karafka::App.config.client_id}_#{raw_consumer_group_name}"
|
31
30
|
end
|
32
31
|
end
|
33
32
|
end
|
@@ -7,15 +7,6 @@ module Karafka
|
|
7
7
|
class Proxy
|
8
8
|
attr_reader :target
|
9
9
|
|
10
|
-
# We should proxy only non ? and = methods as we want to have a regular dsl
|
11
|
-
IGNORED_POSTFIXES = %w[
|
12
|
-
?
|
13
|
-
=
|
14
|
-
!
|
15
|
-
].freeze
|
16
|
-
|
17
|
-
private_constant :IGNORED_POSTFIXES
|
18
|
-
|
19
10
|
# @param target [Object] target object to which we proxy any DSL call
|
20
11
|
# @param block [Proc] block that we want to evaluate in the proxy context
|
21
12
|
def initialize(target, &block)
|
@@ -25,21 +16,23 @@ module Karafka
|
|
25
16
|
|
26
17
|
# Translates the no "=" DSL of routing into elements assignments on target
|
27
18
|
# @param method_name [Symbol] name of the missing method
|
28
|
-
|
29
|
-
# @param block [Proc] block provided to the method
|
30
|
-
def method_missing(method_name, *arguments, &block)
|
19
|
+
def method_missing(method_name, ...)
|
31
20
|
return super unless respond_to_missing?(method_name)
|
32
21
|
|
33
|
-
@target.
|
22
|
+
if @target.respond_to?(:"#{method_name}=")
|
23
|
+
@target.public_send(:"#{method_name}=", ...)
|
24
|
+
else
|
25
|
+
@target.public_send(method_name, ...)
|
26
|
+
end
|
34
27
|
end
|
35
28
|
|
36
29
|
# Tells whether or not a given element exists on the target
|
37
30
|
# @param method_name [Symbol] name of the missing method
|
38
31
|
# @param include_private [Boolean] should we include private in the check as well
|
39
32
|
def respond_to_missing?(method_name, include_private = false)
|
40
|
-
|
41
|
-
|
42
|
-
|
33
|
+
@target.respond_to?(:"#{method_name}=", include_private) ||
|
34
|
+
@target.respond_to?(method_name, include_private) ||
|
35
|
+
super
|
43
36
|
end
|
44
37
|
end
|
45
38
|
end
|
@@ -10,7 +10,7 @@ module Karafka
|
|
10
10
|
# Find a proper topic based on full topic id
|
11
11
|
# @param topic_id [String] proper topic id (already mapped, etc) for which we want to find
|
12
12
|
# routing topic
|
13
|
-
# @return [Karafka::Routing::
|
13
|
+
# @return [Karafka::Routing::Topic] proper route details
|
14
14
|
# @raise [Karafka::Topic::NonMatchingTopicError] raised if topic name does not match
|
15
15
|
# any route defined by user using routes.draw
|
16
16
|
def find(topic_id)
|
@@ -0,0 +1,53 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Routing
|
5
|
+
# Object representing a set of single consumer group topics that can be subscribed together
|
6
|
+
# with one connection.
|
7
|
+
#
|
8
|
+
# @note One subscription group will always belong to one consumer group, but one consumer
|
9
|
+
# group can have multiple subscription groups.
|
10
|
+
class SubscriptionGroup
|
11
|
+
attr_reader :id, :topics
|
12
|
+
|
13
|
+
# @param topics [Karafka::Routing::Topics] all the topics that share the same key settings
|
14
|
+
# @return [SubscriptionGroup] built subscription group
|
15
|
+
def initialize(topics)
|
16
|
+
@id = SecureRandom.uuid
|
17
|
+
@topics = topics
|
18
|
+
freeze
|
19
|
+
end
|
20
|
+
|
21
|
+
# @return [String] consumer group id
|
22
|
+
def consumer_group_id
|
23
|
+
kafka[:'group.id']
|
24
|
+
end
|
25
|
+
|
26
|
+
# @return [Integer] max messages fetched in a single go
|
27
|
+
def max_messages
|
28
|
+
@topics.first.max_messages
|
29
|
+
end
|
30
|
+
|
31
|
+
# @return [Integer] max milliseconds we can wait for incoming messages
|
32
|
+
def max_wait_time
|
33
|
+
@topics.first.max_wait_time
|
34
|
+
end
|
35
|
+
|
36
|
+
# @return [Hash] kafka settings are a bit special. They are exactly the same for all of the
|
37
|
+
# topics but they lack the group.id (unless explicitly) provided. To make it compatible
|
38
|
+
# with our routing engine, we inject it before it will go to the consumer
|
39
|
+
def kafka
|
40
|
+
kafka = @topics.first.kafka.dup
|
41
|
+
|
42
|
+
kafka[:'client.id'] ||= Karafka::App.config.client_id
|
43
|
+
kafka[:'group.id'] ||= @topics.first.consumer_group.id
|
44
|
+
kafka[:'auto.offset.reset'] ||= @topics.first.initial_offset
|
45
|
+
# Karafka manages the offsets based on the processing state, thus we do not rely on the
|
46
|
+
# rdkafka offset auto-storing
|
47
|
+
kafka[:'enable.auto.offset.store'] = false
|
48
|
+
kafka.freeze
|
49
|
+
kafka
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Routing
|
5
|
+
# rdkafka allows us to group topics subscriptions when they have same settings.
|
6
|
+
# This builder groups topics from a single consumer group into subscription groups that can be
|
7
|
+
# subscribed with one rdkafka connection.
|
8
|
+
# This way we save resources as having several rdkafka consumers under the hood is not the
|
9
|
+
# cheapest thing in a bigger system.
|
10
|
+
#
|
11
|
+
# In general, if we can, we try to subscribe to as many topics with one rdkafka connection as
|
12
|
+
# possible, but if not possible, we divide.
|
13
|
+
class SubscriptionGroupsBuilder
|
14
|
+
# Keys used to build up a hash for subscription groups distribution.
|
15
|
+
# In order to be able to use the same rdkafka connection for several topics, those keys need
|
16
|
+
# to have same values.
|
17
|
+
DISTRIBUTION_KEYS = %i[
|
18
|
+
kafka
|
19
|
+
max_messages
|
20
|
+
max_wait_time
|
21
|
+
initial_offset
|
22
|
+
subscription_group
|
23
|
+
].freeze
|
24
|
+
|
25
|
+
private_constant :DISTRIBUTION_KEYS
|
26
|
+
|
27
|
+
# @param topics [Karafka::Routing::Topics] all the topics based on which we want to build
|
28
|
+
# subscription groups
|
29
|
+
# @return [Array<SubscriptionGroup>] all subscription groups we need in separate threads
|
30
|
+
def call(topics)
|
31
|
+
topics
|
32
|
+
.map { |topic| [checksum(topic), topic] }
|
33
|
+
.group_by(&:first)
|
34
|
+
.values
|
35
|
+
.map { |value| value.map(&:last) }
|
36
|
+
.map { |topics_array| Routing::Topics.new(topics_array) }
|
37
|
+
.map { |grouped_topics| SubscriptionGroup.new(grouped_topics) }
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
# @param topic [Karafka::Routing::Topic] topic for which we compute the grouping checksum
|
43
|
+
# @return [Integer] checksum that we can use to check if topics have the same set of
|
44
|
+
# settings based on which we group
|
45
|
+
def checksum(topic)
|
46
|
+
accu = {}
|
47
|
+
|
48
|
+
DISTRIBUTION_KEYS.each { |key| accu[key] = topic.public_send(key) }
|
49
|
+
|
50
|
+
accu.hash
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -2,17 +2,25 @@
|
|
2
2
|
|
3
3
|
module Karafka
|
4
4
|
module Routing
|
5
|
-
# Topic stores all the details on how we should interact with Kafka given topic
|
5
|
+
# Topic stores all the details on how we should interact with Kafka given topic.
|
6
6
|
# It belongs to a consumer group as from 0.6 all the topics can work in the same consumer group
|
7
|
-
# It is a part of Karafka's DSL
|
7
|
+
# It is a part of Karafka's DSL.
|
8
8
|
class Topic
|
9
|
-
|
10
|
-
|
9
|
+
attr_reader :id, :name, :consumer_group, :tags
|
10
|
+
attr_writer :consumer
|
11
|
+
attr_accessor :subscription_group
|
11
12
|
|
12
|
-
|
13
|
-
|
13
|
+
# Attributes we can inherit from the root unless they were defined on this level
|
14
|
+
INHERITABLE_ATTRIBUTES = %i[
|
15
|
+
kafka
|
16
|
+
deserializer
|
17
|
+
manual_offset_management
|
18
|
+
max_messages
|
19
|
+
max_wait_time
|
20
|
+
initial_offset
|
21
|
+
].freeze
|
14
22
|
|
15
|
-
|
23
|
+
private_constant :INHERITABLE_ATTRIBUTES
|
16
24
|
|
17
25
|
# @param [String, Symbol] name of a topic on which we want to listen
|
18
26
|
# @param consumer_group [Karafka::Routing::ConsumerGroup] owning consumer group of this topic
|
@@ -22,40 +30,73 @@ module Karafka
|
|
22
30
|
@attributes = {}
|
23
31
|
# @note We use identifier related to the consumer group that owns a topic, because from
|
24
32
|
# Karafka 0.6 we can handle multiple Kafka instances with the same process and we can
|
25
|
-
# have same topic name across multiple
|
33
|
+
# have same topic name across multiple consumer groups
|
26
34
|
@id = "#{consumer_group.id}_#{@name}"
|
35
|
+
@tags = []
|
27
36
|
end
|
28
37
|
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
38
|
+
INHERITABLE_ATTRIBUTES.each do |attribute|
|
39
|
+
attr_writer attribute
|
40
|
+
|
41
|
+
define_method attribute do
|
42
|
+
current_value = instance_variable_get(:"@#{attribute}")
|
43
|
+
|
44
|
+
return current_value unless current_value.nil?
|
45
|
+
|
46
|
+
value = Karafka::App.config.send(attribute)
|
47
|
+
|
48
|
+
instance_variable_set(:"@#{attribute}", value)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
# @return [Class] consumer class that we should use
|
53
|
+
def consumer
|
54
|
+
if Karafka::App.config.consumer_persistence
|
55
|
+
# When persistence of consumers is on, no need to reload them
|
56
|
+
@consumer
|
57
|
+
else
|
58
|
+
# In order to support code reload without having to change the topic api, we re-fetch the
|
59
|
+
# class of a consumer based on its class name. This will support all the cases where the
|
60
|
+
# consumer class is defined with a name. It won't support code reload for anonymous
|
61
|
+
# consumer classes, but this is an edge case
|
62
|
+
begin
|
63
|
+
::Object.const_get(@consumer.to_s)
|
64
|
+
rescue NameError
|
65
|
+
# It will only fail if the in case of anonymous classes
|
66
|
+
@consumer
|
67
|
+
end
|
68
|
+
end
|
36
69
|
end
|
37
70
|
|
38
|
-
# @return [Class
|
39
|
-
#
|
40
|
-
|
41
|
-
|
71
|
+
# @return [Class] consumer class that we should use
|
72
|
+
# @note This is just an alias to the `#consumer` method. We however want to use it internally
|
73
|
+
# instead of referencing the `#consumer`. We use this to indicate that this method returns
|
74
|
+
# class and not an instance. In the routing we want to keep the `#consumer Consumer`
|
75
|
+
# routing syntax, but for references outside, we should use this one.
|
76
|
+
def consumer_class
|
77
|
+
consumer
|
42
78
|
end
|
43
79
|
|
44
|
-
|
45
|
-
|
80
|
+
# @return [Boolean] true if this topic offset is handled by the end user
|
81
|
+
def manual_offset_management?
|
82
|
+
manual_offset_management
|
46
83
|
end
|
47
84
|
|
48
85
|
# @return [Hash] hash with all the topic attributes
|
49
86
|
# @note This is being used when we validate the consumer_group and its topics
|
50
87
|
def to_h
|
51
|
-
map =
|
88
|
+
map = INHERITABLE_ATTRIBUTES.map do |attribute|
|
52
89
|
[attribute, public_send(attribute)]
|
53
90
|
end
|
54
91
|
|
55
92
|
Hash[map].merge!(
|
56
93
|
id: id,
|
57
|
-
|
58
|
-
|
94
|
+
name: name,
|
95
|
+
consumer: consumer,
|
96
|
+
consumer_group_id: consumer_group.id,
|
97
|
+
subscription_group: subscription_group,
|
98
|
+
tags: tags
|
99
|
+
).freeze
|
59
100
|
end
|
60
101
|
end
|
61
102
|
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# frozen_string_literal: true
|
4
|
+
|
5
|
+
module Karafka
|
6
|
+
module Routing
|
7
|
+
# Abstraction layer on top of groups of topics
|
8
|
+
class Topics
|
9
|
+
include Enumerable
|
10
|
+
extend Forwardable
|
11
|
+
|
12
|
+
def_delegators :@accumulator, :[], :size, :empty?, :last, :<<
|
13
|
+
|
14
|
+
# @param topics_array [Array<Karafka::Routing::Topic>] array with topics
|
15
|
+
def initialize(topics_array)
|
16
|
+
@accumulator = topics_array.dup
|
17
|
+
end
|
18
|
+
|
19
|
+
# Yields each topic
|
20
|
+
#
|
21
|
+
# @param [Proc] block we want to yield with on each topic
|
22
|
+
def each(&block)
|
23
|
+
@accumulator.each(&block)
|
24
|
+
end
|
25
|
+
|
26
|
+
# Finds topic by its name
|
27
|
+
#
|
28
|
+
# @param topic_name [String] topic name
|
29
|
+
# @return [Karafka::Routing::Topic]
|
30
|
+
# @raise [Karafka::Errors::TopicNotFoundError] this should never happen. If you see it,
|
31
|
+
# please create an issue.
|
32
|
+
def find(topic_name)
|
33
|
+
@accumulator.find { |topic| topic.name == topic_name } ||
|
34
|
+
raise(Karafka::Errors::TopicNotFoundError, topic_name)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
# Class used to run the Karafka listeners in separate threads
|
5
|
+
class Runner
|
6
|
+
# Starts listening on all the listeners asynchronously and handles the jobs queue closing
|
7
|
+
# after listeners are done with their work.
|
8
|
+
def call
|
9
|
+
# Despite possibility of having several independent listeners, we aim to have one queue for
|
10
|
+
# jobs across and one workers poll for that
|
11
|
+
jobs_queue = Processing::JobsQueue.new
|
12
|
+
|
13
|
+
workers = Processing::WorkersBatch.new(jobs_queue)
|
14
|
+
listeners = Connection::ListenersBatch.new(jobs_queue)
|
15
|
+
|
16
|
+
workers.each(&:async_call)
|
17
|
+
listeners.each(&:async_call)
|
18
|
+
|
19
|
+
# We aggregate threads here for a supervised shutdown process
|
20
|
+
Karafka::Server.workers = workers
|
21
|
+
Karafka::Server.listeners = listeners
|
22
|
+
|
23
|
+
# All the listener threads need to finish
|
24
|
+
listeners.each(&:join)
|
25
|
+
|
26
|
+
# We close the jobs queue only when no listener threads are working.
|
27
|
+
# This ensures, that everything was closed prior to us not accepting anymore jobs and that
|
28
|
+
# no more jobs will be enqueued. Since each listener waits for jobs to finish, once those
|
29
|
+
# are done, we can close.
|
30
|
+
jobs_queue.close
|
31
|
+
|
32
|
+
# All the workers need to stop processing anything before we can stop the runner completely
|
33
|
+
# This ensures that even async long-running jobs have time to finish before we are done
|
34
|
+
# with everything. One thing worth keeping in mind though: It is the end user responsibility
|
35
|
+
# to handle the shutdown detection in their long-running processes. Otherwise if timeout
|
36
|
+
# is exceeded, there will be a forced shutdown.
|
37
|
+
workers.each(&:join)
|
38
|
+
# If anything crashes here, we need to raise the error and crush the runner because it means
|
39
|
+
# that something terrible happened
|
40
|
+
rescue StandardError => e
|
41
|
+
Karafka.monitor.instrument(
|
42
|
+
'error.occurred',
|
43
|
+
caller: self,
|
44
|
+
error: e,
|
45
|
+
type: 'runner.call.error'
|
46
|
+
)
|
47
|
+
Karafka::App.stop!
|
48
|
+
raise e
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -1,25 +1,16 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module Karafka
|
4
|
-
# Module for all supported by default serialization and deserialization ways
|
4
|
+
# Module for all supported by default serialization and deserialization ways.
|
5
5
|
module Serialization
|
6
|
-
# Namespace for json
|
6
|
+
# Namespace for json serializers and deserializers.
|
7
7
|
module Json
|
8
|
-
# Default Karafka Json deserializer for loading JSON data
|
8
|
+
# Default Karafka Json deserializer for loading JSON data.
|
9
9
|
class Deserializer
|
10
|
-
# @param
|
10
|
+
# @param message [Karafka::Messages::Message] Message object that we want to deserialize
|
11
11
|
# @return [Hash] hash with deserialized JSON data
|
12
|
-
|
13
|
-
|
14
|
-
# 'payload' => "{\"a\":1}",
|
15
|
-
# 'topic' => 'my-topic',
|
16
|
-
# 'headers' => { 'message_type' => :test }
|
17
|
-
# }
|
18
|
-
# Deserializer.call(params) #=> { 'a' => 1 }
|
19
|
-
def call(params)
|
20
|
-
params.raw_payload.nil? ? nil : ::JSON.parse(params.raw_payload)
|
21
|
-
rescue ::JSON::ParserError => e
|
22
|
-
raise ::Karafka::Errors::DeserializationError, e
|
12
|
+
def call(message)
|
13
|
+
message.raw_payload.nil? ? nil : ::JSON.parse(message.raw_payload)
|
23
14
|
end
|
24
15
|
end
|
25
16
|
end
|
data/lib/karafka/server.rb
CHANGED
@@ -3,8 +3,6 @@
|
|
3
3
|
module Karafka
|
4
4
|
# Karafka consuming server class
|
5
5
|
class Server
|
6
|
-
@consumer_threads = Concurrent::Array.new
|
7
|
-
|
8
6
|
# How long should we sleep between checks on shutting down consumers
|
9
7
|
SUPERVISION_SLEEP = 0.1
|
10
8
|
# What system exit code should we use when we terminated forcefully
|
@@ -17,55 +15,77 @@ module Karafka
|
|
17
15
|
|
18
16
|
class << self
|
19
17
|
# Set of consuming threads. Each consumer thread contains a single consumer
|
20
|
-
attr_accessor :
|
18
|
+
attr_accessor :listeners
|
19
|
+
|
20
|
+
# Set of workers
|
21
|
+
attr_accessor :workers
|
21
22
|
|
22
23
|
# Writer for list of consumer groups that we want to consume in our current process context
|
23
24
|
attr_writer :consumer_groups
|
24
25
|
|
25
26
|
# Method which runs app
|
26
27
|
def run
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
28
|
+
# Since we do a lot of threading and queuing, we don't want to stop from the trap context
|
29
|
+
# as some things may not work there as expected, that is why we spawn a separate thread to
|
30
|
+
# handle the stopping process
|
31
|
+
process.on_sigint { Thread.new { stop } }
|
32
|
+
process.on_sigquit { Thread.new { stop } }
|
33
|
+
process.on_sigterm { Thread.new { stop } }
|
34
|
+
|
35
|
+
# Start is blocking until stop is called and when we stop, it will wait until
|
36
|
+
# all of the things are ready to stop
|
37
|
+
start
|
38
|
+
|
39
|
+
# We always need to wait for Karafka to stop here since we should wait for the stop running
|
40
|
+
# in a separate thread (or trap context) to indicate everything is closed
|
41
|
+
# Since `#start` is blocking, we were get here only after the runner is done. This will
|
42
|
+
# not add any performance degradation because of that.
|
43
|
+
Thread.pass until Karafka::App.stopped?
|
44
|
+
# Try its best to shutdown underlying components before re-raising
|
45
|
+
# rubocop:disable Lint/RescueException
|
46
|
+
rescue Exception => e
|
47
|
+
# rubocop:enable Lint/RescueException
|
48
|
+
stop
|
49
|
+
|
50
|
+
raise e
|
31
51
|
end
|
32
52
|
|
33
53
|
# @return [Array<String>] array with names of consumer groups that should be consumed in a
|
34
54
|
# current server context
|
35
55
|
def consumer_groups
|
36
|
-
# If not specified, a server will
|
56
|
+
# If not specified, a server will listen on all the topics
|
37
57
|
@consumer_groups ||= Karafka::App.consumer_groups.map(&:name).freeze
|
38
58
|
end
|
39
59
|
|
40
|
-
private
|
41
|
-
|
42
|
-
# @return [Karafka::Process] process wrapper instance used to catch system signal calls
|
43
|
-
def process
|
44
|
-
Karafka::App.config.internal.process
|
45
|
-
end
|
46
|
-
|
47
60
|
# Starts Karafka with a supervision
|
48
61
|
# @note We don't need to sleep because Karafka::Fetcher is locking and waiting to
|
49
62
|
# finish loop (and it won't happen until we explicitly want to stop)
|
50
|
-
def
|
63
|
+
def start
|
51
64
|
process.supervise
|
52
65
|
Karafka::App.run!
|
53
|
-
Karafka::
|
66
|
+
Karafka::Runner.new.call
|
54
67
|
end
|
55
68
|
|
56
69
|
# Stops Karafka with a supervision (as long as there is a shutdown timeout)
|
57
|
-
# If consumers won't stop in a given time frame, it will force them to exit
|
58
|
-
|
70
|
+
# If consumers or workers won't stop in a given time frame, it will force them to exit
|
71
|
+
#
|
72
|
+
# @note This method is not async. It should not be executed from the workers as it will
|
73
|
+
# lock them forever. If you need to run Karafka shutdown from within workers threads,
|
74
|
+
# please start a separate thread to do so.
|
75
|
+
def stop
|
59
76
|
Karafka::App.stop!
|
60
77
|
|
61
|
-
|
62
|
-
timeout = Thread.new { Karafka::App.config.shutdown_timeout }.join.value
|
78
|
+
timeout = Karafka::App.config.shutdown_timeout
|
63
79
|
|
64
80
|
# We check from time to time (for the timeout period) if all the threads finished
|
65
81
|
# their work and if so, we can just return and normal shutdown process will take place
|
66
|
-
|
67
|
-
|
68
|
-
|
82
|
+
# We divide it by 1000 because we use time in ms.
|
83
|
+
((timeout / 1_000) * SUPERVISION_CHECK_FACTOR).to_i.times do
|
84
|
+
if listeners.count(&:alive?).zero? &&
|
85
|
+
workers.count(&:alive?).zero?
|
86
|
+
|
87
|
+
Karafka::App.producer.close
|
88
|
+
|
69
89
|
return
|
70
90
|
end
|
71
91
|
|
@@ -74,12 +94,33 @@ module Karafka
|
|
74
94
|
|
75
95
|
raise Errors::ForcefulShutdownError
|
76
96
|
rescue Errors::ForcefulShutdownError => e
|
77
|
-
|
97
|
+
Karafka.monitor.instrument(
|
98
|
+
'error.occurred',
|
99
|
+
caller: self,
|
100
|
+
error: e,
|
101
|
+
type: 'app.stopping.error'
|
102
|
+
)
|
103
|
+
|
78
104
|
# We're done waiting, lets kill them!
|
79
|
-
|
105
|
+
workers.each(&:terminate)
|
106
|
+
listeners.each(&:terminate)
|
107
|
+
# We always need to shutdown clients to make sure we do not force the GC to close consumer.
|
108
|
+
# This can cause memory leaks and crashes.
|
109
|
+
listeners.each(&:shutdown)
|
110
|
+
|
111
|
+
Karafka::App.producer.close
|
80
112
|
|
81
113
|
# exit! is not within the instrumentation as it would not trigger due to exit
|
82
114
|
Kernel.exit! FORCEFUL_EXIT_CODE
|
115
|
+
ensure
|
116
|
+
Karafka::App.stopped!
|
117
|
+
end
|
118
|
+
|
119
|
+
private
|
120
|
+
|
121
|
+
# @return [Karafka::Process] process wrapper instance used to catch system signal calls
|
122
|
+
def process
|
123
|
+
Karafka::App.config.internal.process
|
83
124
|
end
|
84
125
|
end
|
85
126
|
end
|