karafka 1.2.2 → 1.4.0.rc1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +2 -0
- data.tar.gz.sig +0 -0
- data/.coditsu/ci.yml +3 -0
- data/.console_irbrc +1 -3
- data/.diffend.yml +3 -0
- data/.github/FUNDING.yml +3 -0
- data/.github/ISSUE_TEMPLATE/bug_report.md +50 -0
- data/.github/ISSUE_TEMPLATE/feature_request.md +20 -0
- data/.github/workflows/ci.yml +52 -0
- data/.gitignore +1 -0
- data/.ruby-version +1 -1
- data/CHANGELOG.md +157 -13
- data/CODE_OF_CONDUCT.md +1 -1
- data/CONTRIBUTING.md +1 -1
- data/Gemfile +5 -2
- data/Gemfile.lock +95 -79
- data/README.md +15 -3
- data/bin/karafka +1 -1
- data/certs/mensfeld.pem +25 -0
- data/config/errors.yml +38 -5
- data/docker-compose.yml +17 -0
- data/karafka.gemspec +19 -13
- data/lib/karafka.rb +10 -16
- data/lib/karafka/app.rb +14 -6
- data/lib/karafka/attributes_map.rb +13 -18
- data/lib/karafka/base_consumer.rb +19 -30
- data/lib/karafka/base_responder.rb +51 -29
- data/lib/karafka/cli.rb +2 -2
- data/lib/karafka/cli/console.rb +11 -9
- data/lib/karafka/cli/flow.rb +9 -7
- data/lib/karafka/cli/info.rb +4 -2
- data/lib/karafka/cli/install.rb +30 -6
- data/lib/karafka/cli/server.rb +11 -6
- data/lib/karafka/code_reloader.rb +67 -0
- data/lib/karafka/connection/{config_adapter.rb → api_adapter.rb} +62 -21
- data/lib/karafka/connection/batch_delegator.rb +55 -0
- data/lib/karafka/connection/builder.rb +18 -0
- data/lib/karafka/connection/client.rb +40 -40
- data/lib/karafka/connection/listener.rb +26 -15
- data/lib/karafka/connection/message_delegator.rb +36 -0
- data/lib/karafka/consumers/batch_metadata.rb +10 -0
- data/lib/karafka/consumers/callbacks.rb +32 -15
- data/lib/karafka/consumers/includer.rb +31 -18
- data/lib/karafka/consumers/responders.rb +2 -2
- data/lib/karafka/contracts.rb +10 -0
- data/lib/karafka/contracts/config.rb +21 -0
- data/lib/karafka/contracts/consumer_group.rb +206 -0
- data/lib/karafka/contracts/consumer_group_topic.rb +19 -0
- data/lib/karafka/contracts/responder_usage.rb +54 -0
- data/lib/karafka/contracts/server_cli_options.rb +31 -0
- data/lib/karafka/errors.rb +17 -19
- data/lib/karafka/fetcher.rb +28 -30
- data/lib/karafka/helpers/class_matcher.rb +12 -2
- data/lib/karafka/helpers/config_retriever.rb +1 -1
- data/lib/karafka/helpers/inflector.rb +26 -0
- data/lib/karafka/helpers/multi_delegator.rb +0 -1
- data/lib/karafka/instrumentation/logger.rb +9 -6
- data/lib/karafka/instrumentation/monitor.rb +15 -9
- data/lib/karafka/instrumentation/proctitle_listener.rb +36 -0
- data/lib/karafka/instrumentation/stdout_listener.rb +140 -0
- data/lib/karafka/params/batch_metadata.rb +26 -0
- data/lib/karafka/params/builders/batch_metadata.rb +30 -0
- data/lib/karafka/params/builders/params.rb +38 -0
- data/lib/karafka/params/builders/params_batch.rb +25 -0
- data/lib/karafka/params/metadata.rb +20 -0
- data/lib/karafka/params/params.rb +50 -0
- data/lib/karafka/params/params_batch.rb +35 -21
- data/lib/karafka/patches/ruby_kafka.rb +21 -8
- data/lib/karafka/persistence/client.rb +15 -11
- data/lib/karafka/persistence/{consumer.rb → consumers.rb} +20 -13
- data/lib/karafka/persistence/topics.rb +48 -0
- data/lib/karafka/process.rb +0 -4
- data/lib/karafka/responders/builder.rb +1 -1
- data/lib/karafka/responders/topic.rb +6 -8
- data/lib/karafka/routing/builder.rb +36 -8
- data/lib/karafka/routing/consumer_group.rb +1 -1
- data/lib/karafka/routing/consumer_mapper.rb +9 -9
- data/lib/karafka/routing/proxy.rb +10 -1
- data/lib/karafka/routing/topic.rb +5 -3
- data/lib/karafka/routing/topic_mapper.rb +16 -18
- data/lib/karafka/serialization/json/deserializer.rb +27 -0
- data/lib/karafka/serialization/json/serializer.rb +31 -0
- data/lib/karafka/server.rb +34 -49
- data/lib/karafka/setup/config.rb +74 -40
- data/lib/karafka/setup/configurators/water_drop.rb +7 -3
- data/lib/karafka/setup/dsl.rb +0 -1
- data/lib/karafka/status.rb +7 -3
- data/lib/karafka/templates/{application_consumer.rb.example → application_consumer.rb.erb} +2 -1
- data/lib/karafka/templates/{application_responder.rb.example → application_responder.rb.erb} +0 -0
- data/lib/karafka/templates/karafka.rb.erb +92 -0
- data/lib/karafka/version.rb +1 -1
- metadata +97 -73
- metadata.gz.sig +4 -0
- data/.travis.yml +0 -13
- data/lib/karafka/callbacks.rb +0 -30
- data/lib/karafka/callbacks/config.rb +0 -22
- data/lib/karafka/callbacks/dsl.rb +0 -16
- data/lib/karafka/connection/delegator.rb +0 -46
- data/lib/karafka/instrumentation/listener.rb +0 -112
- data/lib/karafka/loader.rb +0 -28
- data/lib/karafka/params/dsl.rb +0 -156
- data/lib/karafka/parsers/json.rb +0 -38
- data/lib/karafka/patches/dry_configurable.rb +0 -35
- data/lib/karafka/persistence/topic.rb +0 -29
- data/lib/karafka/schemas/config.rb +0 -24
- data/lib/karafka/schemas/consumer_group.rb +0 -77
- data/lib/karafka/schemas/consumer_group_topic.rb +0 -18
- data/lib/karafka/schemas/responder_usage.rb +0 -39
- data/lib/karafka/schemas/server_cli_options.rb +0 -43
- data/lib/karafka/setup/configurators/base.rb +0 -29
- data/lib/karafka/setup/configurators/params.rb +0 -25
- data/lib/karafka/templates/karafka.rb.example +0 -54
@@ -0,0 +1,25 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Params
|
5
|
+
module Builders
|
6
|
+
# Builder for creating params batch instances
|
7
|
+
module ParamsBatch
|
8
|
+
class << self
|
9
|
+
# Creates params batch with params inside based on the incoming messages
|
10
|
+
# and the topic from which it comes
|
11
|
+
# @param kafka_messages [Array<Kafka::FetchedMessage>] raw fetched messages
|
12
|
+
# @param topic [Karafka::Routing::Topic] topic for which we're received messages
|
13
|
+
# @return [Karafka::Params::ParamsBatch<Karafka::Params::Params>] batch with params
|
14
|
+
def from_kafka_messages(kafka_messages, topic)
|
15
|
+
params_array = kafka_messages.map do |message|
|
16
|
+
Karafka::Params::Builders::Params.from_kafka_message(message, topic)
|
17
|
+
end
|
18
|
+
|
19
|
+
Karafka::Params::ParamsBatch.new(params_array).freeze
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Params
|
5
|
+
# Single message / params metadata details that can be accessed without the need for the
|
6
|
+
# payload deserialization
|
7
|
+
Metadata = Struct.new(
|
8
|
+
:create_time,
|
9
|
+
:headers,
|
10
|
+
:is_control_record,
|
11
|
+
:key,
|
12
|
+
:offset,
|
13
|
+
:deserializer,
|
14
|
+
:partition,
|
15
|
+
:receive_time,
|
16
|
+
:topic,
|
17
|
+
keyword_init: true
|
18
|
+
)
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
# Params namespace encapsulating all the logic that is directly related to params handling
|
5
|
+
module Params
|
6
|
+
# It provides lazy loading not only until the first usage, but also allows us to skip
|
7
|
+
# using deserializer until we execute our logic. That way we can operate with
|
8
|
+
# heavy-deserialization data without slowing down the whole application.
|
9
|
+
class Params
|
10
|
+
attr_reader :raw_payload, :metadata
|
11
|
+
|
12
|
+
# @param raw_payload [Object] incoming payload before deserialization
|
13
|
+
# @param metadata [Karafka::Params::Metadata] message metadata object
|
14
|
+
def initialize(raw_payload, metadata)
|
15
|
+
@raw_payload = raw_payload
|
16
|
+
@metadata = metadata
|
17
|
+
@deserialized = false
|
18
|
+
@payload = nil
|
19
|
+
end
|
20
|
+
|
21
|
+
# @return [Object] lazy-deserialized data (deserialized upon first request)
|
22
|
+
def payload
|
23
|
+
return @payload if deserialized?
|
24
|
+
|
25
|
+
@payload = deserialize
|
26
|
+
# We mark deserialization as successful after deserialization, as in case of an error
|
27
|
+
# this won't be falsely set to true
|
28
|
+
@deserialized = true
|
29
|
+
@payload
|
30
|
+
end
|
31
|
+
|
32
|
+
# @return [Boolean] did given params payload were deserialized already
|
33
|
+
def deserialized?
|
34
|
+
@deserialized
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
|
39
|
+
# @return [Object] tries de-serializes data
|
40
|
+
def deserialize
|
41
|
+
Karafka.monitor.instrument('params.params.deserialize', caller: self) do
|
42
|
+
metadata.deserializer.call(self)
|
43
|
+
end
|
44
|
+
rescue ::StandardError => e
|
45
|
+
Karafka.monitor.instrument('params.params.deserialize.error', caller: self, error: e)
|
46
|
+
raise e
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -3,43 +3,57 @@
|
|
3
3
|
module Karafka
|
4
4
|
module Params
|
5
5
|
# Params batch represents a set of messages received from Kafka.
|
6
|
-
# @note Params internally are lazy loaded before first use. That way we can skip
|
7
|
-
# process if we have after_fetch that rejects some incoming messages
|
8
|
-
# It can be also used when handling really heavy data
|
6
|
+
# @note Params internally are lazy loaded before first use. That way we can skip
|
7
|
+
# deserialization process if we have after_fetch that rejects some incoming messages
|
8
|
+
# without using params It can be also used when handling really heavy data.
|
9
9
|
class ParamsBatch
|
10
10
|
include Enumerable
|
11
11
|
|
12
|
-
#
|
13
|
-
# @
|
14
|
-
|
15
|
-
|
16
|
-
@params_batch = messages_batch.map! do |message|
|
17
|
-
Karafka::Params::Params.build(message, topic_parser)
|
18
|
-
end
|
12
|
+
# @param params_array [Array<Karafka::Params::Params>] array with karafka params
|
13
|
+
# @return [Karafka::Params::ParamsBatch] lazy evaluated params batch object
|
14
|
+
def initialize(params_array)
|
15
|
+
@params_array = params_array
|
19
16
|
end
|
20
17
|
|
21
|
-
# @yieldparam [Karafka::Params::Params] each
|
22
|
-
# @note Invocation of this method will cause loading and
|
23
|
-
#
|
18
|
+
# @yieldparam [Karafka::Params::Params] each params instance
|
19
|
+
# @note Invocation of this method will not cause loading and deserializing each param after
|
20
|
+
# another.
|
24
21
|
def each
|
25
|
-
@
|
22
|
+
@params_array.each { |param| yield(param) }
|
26
23
|
end
|
27
24
|
|
28
25
|
# @return [Array<Karafka::Params::Params>] returns all the params in a loaded state, so they
|
29
26
|
# can be used for batch insert, etc. Without invoking all, up until first use, they won't
|
30
|
-
# be
|
31
|
-
def
|
32
|
-
each(&:
|
27
|
+
# be deserialized
|
28
|
+
def deserialize!
|
29
|
+
each(&:payload)
|
33
30
|
end
|
34
31
|
|
35
|
-
# @return [
|
32
|
+
# @return [Array<Object>] array with deserialized payloads. This method can be useful when
|
33
|
+
# we don't care about metadata and just want to extract all the data payloads from the
|
34
|
+
# batch
|
35
|
+
def payloads
|
36
|
+
map(&:payload)
|
37
|
+
end
|
38
|
+
|
39
|
+
# @return [Karafka::Params::Params] first element
|
40
|
+
def first
|
41
|
+
@params_array.first
|
42
|
+
end
|
43
|
+
|
44
|
+
# @return [Karafka::Params::Params] last element
|
36
45
|
def last
|
37
|
-
@
|
46
|
+
@params_array.last
|
47
|
+
end
|
48
|
+
|
49
|
+
# @return [Integer] number of messages in the batch
|
50
|
+
def size
|
51
|
+
@params_array.size
|
38
52
|
end
|
39
53
|
|
40
|
-
# @return [Array<Karafka::Params::Params>] pure array with params
|
54
|
+
# @return [Array<Karafka::Params::Params>] pure array with params
|
41
55
|
def to_a
|
42
|
-
@
|
56
|
+
@params_array
|
43
57
|
end
|
44
58
|
end
|
45
59
|
end
|
@@ -1,34 +1,47 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module Karafka
|
4
|
+
# Namespace for various other libs patches
|
4
5
|
module Patches
|
5
6
|
# Patches for Ruby Kafka gem
|
6
7
|
module RubyKafka
|
7
8
|
# This patch allows us to inject business logic in between fetches and before the consumer
|
8
9
|
# stop, so we can perform stop commit or anything else that we need since
|
9
10
|
# ruby-kafka fetch loop does not allow that directly
|
10
|
-
# We don't
|
11
|
+
# We don't won't to use poll ruby-kafka api as it brings many more problems that we would
|
11
12
|
# have to take care of. That way, nothing like that ever happens but we get the control
|
12
13
|
# over the stopping process that we need (since we're the once that initiate it for each
|
13
14
|
# thread)
|
14
15
|
def consumer_loop
|
15
16
|
super do
|
16
|
-
consumers = Karafka::Persistence::
|
17
|
-
.
|
17
|
+
consumers = Karafka::Persistence::Consumers
|
18
|
+
.current
|
18
19
|
.values
|
19
20
|
.flat_map(&:values)
|
20
|
-
.select { |
|
21
|
+
.select { |consumer| consumer.class.respond_to?(:after_fetch) }
|
21
22
|
|
22
|
-
if Karafka::App.
|
23
|
-
consumers
|
23
|
+
if Karafka::App.stopping?
|
24
|
+
publish_event(consumers, 'before_stop')
|
24
25
|
Karafka::Persistence::Client.read.stop
|
25
26
|
else
|
26
|
-
consumers
|
27
|
+
publish_event(consumers, 'before_poll')
|
27
28
|
yield
|
28
|
-
consumers
|
29
|
+
publish_event(consumers, 'after_poll')
|
29
30
|
end
|
30
31
|
end
|
31
32
|
end
|
33
|
+
|
34
|
+
private
|
35
|
+
|
36
|
+
# Notifies consumers about particular events happening
|
37
|
+
# @param consumers [Array<Object>] all consumers that want to be notified about an event
|
38
|
+
# @param event_name [String] name of the event that happened
|
39
|
+
def publish_event(consumers, event_name)
|
40
|
+
consumers.each do |consumer|
|
41
|
+
key = "consumers.#{Helpers::Inflector.map(consumer.class.to_s)}.#{event_name}"
|
42
|
+
Karafka::App.monitor.instrument(key, context: consumer)
|
43
|
+
end
|
44
|
+
end
|
32
45
|
end
|
33
46
|
end
|
34
47
|
end
|
@@ -7,18 +7,22 @@ module Karafka
|
|
7
7
|
# Thread.current key under which we store current thread messages consumer client
|
8
8
|
PERSISTENCE_SCOPE = :client
|
9
9
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
10
|
+
private_constant :PERSISTENCE_SCOPE
|
11
|
+
|
12
|
+
class << self
|
13
|
+
# @param client [Karafka::Connection::Client] messages consumer client of
|
14
|
+
# a current thread
|
15
|
+
# @return [Karafka::Connection::Client] persisted messages consumer client
|
16
|
+
def write(client)
|
17
|
+
Thread.current[PERSISTENCE_SCOPE] = client
|
18
|
+
end
|
16
19
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
20
|
+
# @return [Karafka::Connection::Client] persisted messages consumer client
|
21
|
+
# @raise [Karafka::Errors::MissingClientError] raised when no thread messages consumer
|
22
|
+
# client but we try to use it anyway
|
23
|
+
def read
|
24
|
+
Thread.current[PERSISTENCE_SCOPE] || raise(Errors::MissingClientError)
|
25
|
+
end
|
22
26
|
end
|
23
27
|
end
|
24
28
|
end
|
@@ -7,30 +7,37 @@ module Karafka
|
|
7
7
|
# Module used to provide a persistent cache across batch requests for a given
|
8
8
|
# topic and partition to store some additional details when the persistent mode
|
9
9
|
# for a given topic is turned on
|
10
|
-
class
|
10
|
+
class Consumers
|
11
11
|
# Thread.current scope under which we store consumers data
|
12
12
|
PERSISTENCE_SCOPE = :consumers
|
13
13
|
|
14
|
+
private_constant :PERSISTENCE_SCOPE
|
15
|
+
|
14
16
|
class << self
|
15
|
-
# @return [Hash] current thread persistence scope hash with all the consumers
|
16
|
-
def
|
17
|
-
|
18
|
-
|
19
|
-
|
17
|
+
# @return [Hash] current thread's persistence scope hash with all the consumers
|
18
|
+
def current
|
19
|
+
Thread.current[PERSISTENCE_SCOPE] ||= Concurrent::Hash.new do |hash, key|
|
20
|
+
hash[key] = Concurrent::Hash.new
|
21
|
+
end
|
20
22
|
end
|
21
23
|
|
22
24
|
# Used to build (if block given) and/or fetch a current consumer instance that will be
|
23
25
|
# used to process messages from a given topic and partition
|
24
|
-
# @return [Karafka::BaseConsumer] base consumer descendant
|
25
26
|
# @param topic [Karafka::Routing::Topic] topic instance for which we might cache
|
26
27
|
# @param partition [Integer] number of partition for which we want to cache
|
28
|
+
# @return [Karafka::BaseConsumer] base consumer descendant
|
27
29
|
def fetch(topic, partition)
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
30
|
+
current[topic][partition] ||= topic.consumer.new(topic)
|
31
|
+
end
|
32
|
+
|
33
|
+
# Removes all persisted instances of consumers from the consumer cache
|
34
|
+
# @note This is used to reload consumers instances when code reloading in development mode
|
35
|
+
# is present. This should not be used in production.
|
36
|
+
def clear
|
37
|
+
Thread
|
38
|
+
.list
|
39
|
+
.select { |thread| thread[PERSISTENCE_SCOPE] }
|
40
|
+
.each { |thread| thread[PERSISTENCE_SCOPE].clear }
|
34
41
|
end
|
35
42
|
end
|
36
43
|
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Persistence
|
5
|
+
# Local cache for routing topics
|
6
|
+
# We use it in order not to build string instances and remap incoming topic upon each
|
7
|
+
# message / message batches received
|
8
|
+
class Topics
|
9
|
+
# Thread.current scope under which we store topics data
|
10
|
+
PERSISTENCE_SCOPE = :topics
|
11
|
+
|
12
|
+
private_constant :PERSISTENCE_SCOPE
|
13
|
+
|
14
|
+
class << self
|
15
|
+
# @return [Concurrent::Hash] hash with all the topics from given groups
|
16
|
+
def current
|
17
|
+
Thread.current[PERSISTENCE_SCOPE] ||= Concurrent::Hash.new do |hash, key|
|
18
|
+
hash[key] = Concurrent::Hash.new
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
# @param group_id [String] group id for which we fetch a topic representation
|
23
|
+
# @param raw_topic_name [String] raw topic name (before remapping) for which we fetch a
|
24
|
+
# topic representation
|
25
|
+
# @return [Karafka::Routing::Topics] remapped topic representation that can be used further
|
26
|
+
# on when working with given parameters
|
27
|
+
def fetch(group_id, raw_topic_name)
|
28
|
+
current[group_id][raw_topic_name] ||= begin
|
29
|
+
# We map from incoming topic name, as it might be namespaced, etc.
|
30
|
+
# @see topic_mapper internal docs
|
31
|
+
mapped_topic_name = Karafka::App.config.topic_mapper.incoming(raw_topic_name)
|
32
|
+
Routing::Router.find("#{group_id}_#{mapped_topic_name}")
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
# Clears the whole topics cache for all the threads
|
37
|
+
# This is used for in-development code reloading as we need to get rid of all the
|
38
|
+
# preloaded and cached instances of objects to make it work
|
39
|
+
def clear
|
40
|
+
Thread
|
41
|
+
.list
|
42
|
+
.select { |thread| thread[PERSISTENCE_SCOPE] }
|
43
|
+
.each { |thread| thread[PERSISTENCE_SCOPE].clear }
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
data/lib/karafka/process.rb
CHANGED
@@ -4,8 +4,6 @@ module Karafka
|
|
4
4
|
# Class used to catch signals from ruby Signal class in order to manage Karafka stop
|
5
5
|
# @note There might be only one process - this class is a singleton
|
6
6
|
class Process
|
7
|
-
include Singleton
|
8
|
-
|
9
7
|
# Signal types that we handle
|
10
8
|
HANDLED_SIGNALS = %i[
|
11
9
|
SIGINT
|
@@ -34,10 +32,8 @@ module Karafka
|
|
34
32
|
|
35
33
|
# Method catches all HANDLED_SIGNALS and performs appropriate callbacks (if defined)
|
36
34
|
# @note If there are no callbacks, this method will just ignore a given signal that was sent
|
37
|
-
# @yield [Block] block of code that we want to execute and supervise
|
38
35
|
def supervise
|
39
36
|
HANDLED_SIGNALS.each { |signal| trap_signal(signal) }
|
40
|
-
yield
|
41
37
|
end
|
42
38
|
|
43
39
|
private
|
@@ -3,7 +3,7 @@
|
|
3
3
|
module Karafka
|
4
4
|
# Responders namespace encapsulates all the internal responder implementation parts
|
5
5
|
module Responders
|
6
|
-
# Responders builder is used
|
6
|
+
# Responders builder is used for finding (based on the consumer class name) a responder
|
7
7
|
# that match the consumer. We use it when user does not provide a responder inside routing,
|
8
8
|
# but he still names responder with the same convention (and namespaces) as consumer
|
9
9
|
#
|
@@ -7,8 +7,6 @@ module Karafka
|
|
7
7
|
# Karafka::Responders::Topic.new(:topic_name, {}) #=> #<Karafka::Responders::Topic...
|
8
8
|
# @example Define optional topic
|
9
9
|
# Karafka::Responders::Topic.new(:topic_name, required: false)
|
10
|
-
# @example Define topic that on which we want to respond multiple times
|
11
|
-
# Karafka::Responders::Topic.new(:topic_name, multiple_usage: true)
|
12
10
|
class Topic
|
13
11
|
# Name of the topic on which we want to respond
|
14
12
|
attr_reader :name
|
@@ -26,16 +24,16 @@ module Karafka
|
|
26
24
|
@options.key?(:required) ? @options[:required] : true
|
27
25
|
end
|
28
26
|
|
29
|
-
# @return [Boolean] do we expect to use it multiple times in a single respond flow
|
30
|
-
def multiple_usage?
|
31
|
-
@options[:multiple_usage] || false
|
32
|
-
end
|
33
|
-
|
34
27
|
# @return [Boolean] was usage of this topic registered or not
|
35
28
|
def registered?
|
36
29
|
@options[:registered] == true
|
37
30
|
end
|
38
31
|
|
32
|
+
# @return [Class] Class to use to serialize messages for this topic
|
33
|
+
def serializer
|
34
|
+
@options[:serializer]
|
35
|
+
end
|
36
|
+
|
39
37
|
# @return [Boolean] do we want to use async producer. Defaults to false as the sync producer
|
40
38
|
# is safer and introduces less problems
|
41
39
|
def async?
|
@@ -46,9 +44,9 @@ module Karafka
|
|
46
44
|
def to_h
|
47
45
|
{
|
48
46
|
name: name,
|
49
|
-
multiple_usage: multiple_usage?,
|
50
47
|
required: required?,
|
51
48
|
registered: registered?,
|
49
|
+
serializer: serializer,
|
52
50
|
async: async?
|
53
51
|
}
|
54
52
|
end
|
@@ -9,26 +9,39 @@ module Karafka
|
|
9
9
|
# consumer NewVideosConsumer
|
10
10
|
# end
|
11
11
|
# end
|
12
|
-
class Builder < Array
|
13
|
-
|
12
|
+
class Builder < Concurrent::Array
|
13
|
+
# Consumer group consistency checking contract
|
14
|
+
CONTRACT = Karafka::Contracts::ConsumerGroup.new.freeze
|
15
|
+
|
16
|
+
private_constant :CONTRACT
|
17
|
+
|
18
|
+
def initialize
|
19
|
+
@draws = Concurrent::Array.new
|
20
|
+
end
|
14
21
|
|
15
22
|
# Used to draw routes for Karafka
|
23
|
+
# @param block [Proc] block we will evaluate within the builder context
|
24
|
+
# @yield Evaluates provided block in a builder context so we can describe routes
|
25
|
+
# @raise [Karafka::Errors::InvalidConfigurationError] raised when configuration
|
26
|
+
# doesn't match with the config contract
|
16
27
|
# @note After it is done drawing it will store and validate all the routes to make sure that
|
17
28
|
# they are correct and that there are no topic/group duplications (this is forbidden)
|
18
|
-
# @yield Evaluates provided block in a builder context so we can describe routes
|
19
29
|
# @example
|
20
30
|
# draw do
|
21
31
|
# topic :xyz do
|
22
32
|
# end
|
23
33
|
# end
|
24
34
|
def draw(&block)
|
35
|
+
@draws << block
|
36
|
+
|
25
37
|
instance_eval(&block)
|
26
38
|
|
27
39
|
each do |consumer_group|
|
28
40
|
hashed_group = consumer_group.to_h
|
29
|
-
validation_result =
|
30
|
-
|
31
|
-
|
41
|
+
validation_result = CONTRACT.call(hashed_group)
|
42
|
+
next if validation_result.success?
|
43
|
+
|
44
|
+
raise Errors::InvalidConfigurationError, validation_result.errors.to_h
|
32
45
|
end
|
33
46
|
end
|
34
47
|
|
@@ -39,18 +52,33 @@ module Karafka
|
|
39
52
|
select(&:active?)
|
40
53
|
end
|
41
54
|
|
55
|
+
# Clears the builder and the draws memory
|
56
|
+
def clear
|
57
|
+
@draws.clear
|
58
|
+
super
|
59
|
+
end
|
60
|
+
|
61
|
+
# Redraws all the routes for the in-process code reloading.
|
62
|
+
# @note This won't allow registration of new topics without process restart but will trigger
|
63
|
+
# cache invalidation so all the classes, etc are re-fetched after code reload
|
64
|
+
def reload
|
65
|
+
draws = @draws.dup
|
66
|
+
clear
|
67
|
+
draws.each { |block| draw(&block) }
|
68
|
+
end
|
69
|
+
|
42
70
|
private
|
43
71
|
|
44
72
|
# Builds and saves given consumer group
|
45
73
|
# @param group_id [String, Symbol] name for consumer group
|
46
|
-
# @
|
74
|
+
# @param block [Proc] proc that should be executed in the proxy context
|
47
75
|
def consumer_group(group_id, &block)
|
48
76
|
consumer_group = ConsumerGroup.new(group_id.to_s)
|
49
77
|
self << Proxy.new(consumer_group, &block).target
|
50
78
|
end
|
51
79
|
|
52
80
|
# @param topic_name [String, Symbol] name of a topic from which we want to consumer
|
53
|
-
# @
|
81
|
+
# @param block [Proc] proc we want to evaluate in the topic context
|
54
82
|
def topic(topic_name, &block)
|
55
83
|
consumer_group(topic_name) do
|
56
84
|
topic(topic_name, &block).tap(&:build)
|