karafka 1.1.0 → 1.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- checksums.yaml.gz.sig +2 -0
- data.tar.gz.sig +0 -0
- data/.coditsu/ci.yml +3 -0
- data/.console_irbrc +1 -3
- data/.github/FUNDING.yml +3 -0
- data/.github/ISSUE_TEMPLATE/bug_report.md +50 -0
- data/.github/ISSUE_TEMPLATE/feature_request.md +20 -0
- data/.gitignore +1 -0
- data/.ruby-version +1 -1
- data/.travis.yml +35 -16
- data/CHANGELOG.md +151 -2
- data/CONTRIBUTING.md +6 -7
- data/Gemfile +3 -3
- data/Gemfile.lock +96 -70
- data/README.md +29 -23
- data/bin/karafka +1 -1
- data/certs/mensfeld.pem +25 -0
- data/config/errors.yml +38 -5
- data/karafka.gemspec +19 -10
- data/lib/karafka.rb +15 -12
- data/lib/karafka/app.rb +19 -18
- data/lib/karafka/attributes_map.rb +15 -14
- data/lib/karafka/backends/inline.rb +1 -2
- data/lib/karafka/base_consumer.rb +57 -0
- data/lib/karafka/base_responder.rb +72 -31
- data/lib/karafka/cli.rb +1 -1
- data/lib/karafka/cli/console.rb +11 -9
- data/lib/karafka/cli/flow.rb +0 -1
- data/lib/karafka/cli/info.rb +3 -1
- data/lib/karafka/cli/install.rb +29 -8
- data/lib/karafka/cli/server.rb +11 -7
- data/lib/karafka/code_reloader.rb +67 -0
- data/lib/karafka/connection/{config_adapter.rb → api_adapter.rb} +67 -24
- data/lib/karafka/connection/batch_delegator.rb +51 -0
- data/lib/karafka/connection/builder.rb +16 -0
- data/lib/karafka/connection/client.rb +117 -0
- data/lib/karafka/connection/listener.rb +37 -17
- data/lib/karafka/connection/message_delegator.rb +36 -0
- data/lib/karafka/consumers/callbacks.rb +71 -0
- data/lib/karafka/consumers/includer.rb +63 -0
- data/lib/karafka/consumers/metadata.rb +10 -0
- data/lib/karafka/consumers/responders.rb +24 -0
- data/lib/karafka/{controllers → consumers}/single_params.rb +3 -3
- data/lib/karafka/contracts.rb +10 -0
- data/lib/karafka/contracts/config.rb +21 -0
- data/lib/karafka/contracts/consumer_group.rb +206 -0
- data/lib/karafka/contracts/consumer_group_topic.rb +19 -0
- data/lib/karafka/contracts/responder_usage.rb +54 -0
- data/lib/karafka/contracts/server_cli_options.rb +29 -0
- data/lib/karafka/errors.rb +23 -15
- data/lib/karafka/fetcher.rb +6 -12
- data/lib/karafka/helpers/class_matcher.rb +19 -9
- data/lib/karafka/helpers/config_retriever.rb +3 -3
- data/lib/karafka/helpers/inflector.rb +26 -0
- data/lib/karafka/helpers/multi_delegator.rb +0 -1
- data/lib/karafka/instrumentation/logger.rb +57 -0
- data/lib/karafka/instrumentation/monitor.rb +70 -0
- data/lib/karafka/instrumentation/proctitle_listener.rb +36 -0
- data/lib/karafka/instrumentation/stdout_listener.rb +138 -0
- data/lib/karafka/params/builders/metadata.rb +33 -0
- data/lib/karafka/params/builders/params.rb +36 -0
- data/lib/karafka/params/builders/params_batch.rb +25 -0
- data/lib/karafka/params/metadata.rb +35 -0
- data/lib/karafka/params/params.rb +35 -95
- data/lib/karafka/params/params_batch.rb +38 -18
- data/lib/karafka/patches/ruby_kafka.rb +25 -12
- data/lib/karafka/persistence/client.rb +29 -0
- data/lib/karafka/persistence/consumers.rb +45 -0
- data/lib/karafka/persistence/topics.rb +48 -0
- data/lib/karafka/process.rb +5 -8
- data/lib/karafka/responders/builder.rb +15 -14
- data/lib/karafka/responders/topic.rb +6 -8
- data/lib/karafka/routing/builder.rb +37 -9
- data/lib/karafka/routing/consumer_group.rb +1 -1
- data/lib/karafka/routing/consumer_mapper.rb +10 -9
- data/lib/karafka/routing/proxy.rb +10 -1
- data/lib/karafka/routing/router.rb +1 -1
- data/lib/karafka/routing/topic.rb +8 -12
- data/lib/karafka/routing/topic_mapper.rb +16 -18
- data/lib/karafka/serialization/json/deserializer.rb +27 -0
- data/lib/karafka/serialization/json/serializer.rb +31 -0
- data/lib/karafka/server.rb +45 -24
- data/lib/karafka/setup/config.rb +95 -37
- data/lib/karafka/setup/configurators/water_drop.rb +12 -5
- data/lib/karafka/setup/dsl.rb +21 -0
- data/lib/karafka/status.rb +7 -3
- data/lib/karafka/templates/{application_controller.rb.example → application_consumer.rb.erb} +2 -2
- data/lib/karafka/templates/{application_responder.rb.example → application_responder.rb.erb} +0 -0
- data/lib/karafka/templates/karafka.rb.erb +92 -0
- data/lib/karafka/version.rb +1 -1
- metadata +126 -57
- metadata.gz.sig +0 -0
- data/.github/ISSUE_TEMPLATE.md +0 -2
- data/lib/karafka/base_controller.rb +0 -60
- data/lib/karafka/connection/consumer.rb +0 -121
- data/lib/karafka/connection/processor.rb +0 -61
- data/lib/karafka/controllers/callbacks.rb +0 -54
- data/lib/karafka/controllers/includer.rb +0 -51
- data/lib/karafka/controllers/responders.rb +0 -19
- data/lib/karafka/loader.rb +0 -29
- data/lib/karafka/logger.rb +0 -53
- data/lib/karafka/monitor.rb +0 -98
- data/lib/karafka/parsers/json.rb +0 -38
- data/lib/karafka/patches/dry_configurable.rb +0 -31
- data/lib/karafka/persistence/consumer.rb +0 -25
- data/lib/karafka/persistence/controller.rb +0 -38
- data/lib/karafka/schemas/config.rb +0 -21
- data/lib/karafka/schemas/consumer_group.rb +0 -65
- data/lib/karafka/schemas/consumer_group_topic.rb +0 -18
- data/lib/karafka/schemas/responder_usage.rb +0 -39
- data/lib/karafka/schemas/server_cli_options.rb +0 -43
- data/lib/karafka/setup/configurators/base.rb +0 -35
- data/lib/karafka/templates/karafka.rb.example +0 -41
@@ -1,34 +1,47 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module Karafka
|
4
|
+
# Namespace for various other libs patches
|
4
5
|
module Patches
|
5
|
-
#
|
6
|
+
# Patches for Ruby Kafka gem
|
6
7
|
module RubyKafka
|
7
8
|
# This patch allows us to inject business logic in between fetches and before the consumer
|
8
9
|
# stop, so we can perform stop commit or anything else that we need since
|
9
10
|
# ruby-kafka fetch loop does not allow that directly
|
10
|
-
# We don't
|
11
|
+
# We don't won't to use poll ruby-kafka api as it brings many more problems that we would
|
11
12
|
# have to take care of. That way, nothing like that ever happens but we get the control
|
12
13
|
# over the stopping process that we need (since we're the once that initiate it for each
|
13
14
|
# thread)
|
14
15
|
def consumer_loop
|
15
16
|
super do
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
17
|
+
consumers = Karafka::Persistence::Consumers
|
18
|
+
.current
|
19
|
+
.values
|
20
|
+
.flat_map(&:values)
|
21
|
+
.select { |consumer| consumer.class.respond_to?(:after_fetch) }
|
21
22
|
|
22
|
-
if Karafka::App.
|
23
|
-
|
24
|
-
Karafka::Persistence::
|
23
|
+
if Karafka::App.stopping?
|
24
|
+
publish_event(consumers, 'before_stop')
|
25
|
+
Karafka::Persistence::Client.read.stop
|
25
26
|
else
|
26
|
-
|
27
|
+
publish_event(consumers, 'before_poll')
|
27
28
|
yield
|
28
|
-
|
29
|
+
publish_event(consumers, 'after_poll')
|
29
30
|
end
|
30
31
|
end
|
31
32
|
end
|
33
|
+
|
34
|
+
private
|
35
|
+
|
36
|
+
# Notifies consumers about particular events happening
|
37
|
+
# @param consumers [Array<Object>] all consumers that want to be notified about an event
|
38
|
+
# @param event_name [String] name of the event that happened
|
39
|
+
def publish_event(consumers, event_name)
|
40
|
+
consumers.each do |consumer|
|
41
|
+
key = "consumers.#{Helpers::Inflector.map(consumer.class.to_s)}.#{event_name}"
|
42
|
+
Karafka::App.monitor.instrument(key, context: consumer)
|
43
|
+
end
|
44
|
+
end
|
32
45
|
end
|
33
46
|
end
|
34
47
|
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Persistence
|
5
|
+
# Persistence layer to store current thread messages consumer client for further use
|
6
|
+
class Client
|
7
|
+
# Thread.current key under which we store current thread messages consumer client
|
8
|
+
PERSISTENCE_SCOPE = :client
|
9
|
+
|
10
|
+
private_constant :PERSISTENCE_SCOPE
|
11
|
+
|
12
|
+
class << self
|
13
|
+
# @param client [Karafka::Connection::Client] messages consumer client of
|
14
|
+
# a current thread
|
15
|
+
# @return [Karafka::Connection::Client] persisted messages consumer client
|
16
|
+
def write(client)
|
17
|
+
Thread.current[PERSISTENCE_SCOPE] = client
|
18
|
+
end
|
19
|
+
|
20
|
+
# @return [Karafka::Connection::Client] persisted messages consumer client
|
21
|
+
# @raise [Karafka::Errors::MissingClientError] raised when no thread messages consumer
|
22
|
+
# client but we try to use it anyway
|
23
|
+
def read
|
24
|
+
Thread.current[PERSISTENCE_SCOPE] || raise(Errors::MissingClientError)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
# Module used to provide a persistent cache layer for Karafka components that need to be
|
5
|
+
# shared inside of a same thread
|
6
|
+
module Persistence
|
7
|
+
# Module used to provide a persistent cache across batch requests for a given
|
8
|
+
# topic and partition to store some additional details when the persistent mode
|
9
|
+
# for a given topic is turned on
|
10
|
+
class Consumers
|
11
|
+
# Thread.current scope under which we store consumers data
|
12
|
+
PERSISTENCE_SCOPE = :consumers
|
13
|
+
|
14
|
+
private_constant :PERSISTENCE_SCOPE
|
15
|
+
|
16
|
+
class << self
|
17
|
+
# @return [Hash] current thread's persistence scope hash with all the consumers
|
18
|
+
def current
|
19
|
+
Thread.current[PERSISTENCE_SCOPE] ||= Concurrent::Hash.new do |hash, key|
|
20
|
+
hash[key] = Concurrent::Hash.new
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
# Used to build (if block given) and/or fetch a current consumer instance that will be
|
25
|
+
# used to process messages from a given topic and partition
|
26
|
+
# @param topic [Karafka::Routing::Topic] topic instance for which we might cache
|
27
|
+
# @param partition [Integer] number of partition for which we want to cache
|
28
|
+
# @return [Karafka::BaseConsumer] base consumer descendant
|
29
|
+
def fetch(topic, partition)
|
30
|
+
current[topic][partition] ||= topic.consumer.new(topic)
|
31
|
+
end
|
32
|
+
|
33
|
+
# Removes all persisted instances of consumers from the consumer cache
|
34
|
+
# @note This is used to reload consumers instances when code reloading in development mode
|
35
|
+
# is present. This should not be used in production.
|
36
|
+
def clear
|
37
|
+
Thread
|
38
|
+
.list
|
39
|
+
.select { |thread| thread[PERSISTENCE_SCOPE] }
|
40
|
+
.each { |thread| thread[PERSISTENCE_SCOPE].clear }
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Persistence
|
5
|
+
# Local cache for routing topics
|
6
|
+
# We use it in order not to build string instances and remap incoming topic upon each
|
7
|
+
# message / message batches received
|
8
|
+
class Topics
|
9
|
+
# Thread.current scope under which we store topics data
|
10
|
+
PERSISTENCE_SCOPE = :topics
|
11
|
+
|
12
|
+
private_constant :PERSISTENCE_SCOPE
|
13
|
+
|
14
|
+
class << self
|
15
|
+
# @return [Concurrent::Hash] hash with all the topics from given groups
|
16
|
+
def current
|
17
|
+
Thread.current[PERSISTENCE_SCOPE] ||= Concurrent::Hash.new do |hash, key|
|
18
|
+
hash[key] = Concurrent::Hash.new
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
# @param group_id [String] group id for which we fetch a topic representation
|
23
|
+
# @param raw_topic_name [String] raw topic name (before remapping) for which we fetch a
|
24
|
+
# topic representation
|
25
|
+
# @return [Karafka::Routing::Topics] remapped topic representation that can be used further
|
26
|
+
# on when working with given parameters
|
27
|
+
def fetch(group_id, raw_topic_name)
|
28
|
+
current[group_id][raw_topic_name] ||= begin
|
29
|
+
# We map from incoming topic name, as it might be namespaced, etc.
|
30
|
+
# @see topic_mapper internal docs
|
31
|
+
mapped_topic_name = Karafka::App.config.topic_mapper.incoming(raw_topic_name)
|
32
|
+
Routing::Router.find("#{group_id}_#{mapped_topic_name}")
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
# Clears the whole topics cache for all the threads
|
37
|
+
# This is used for in-development code reloading as we need to get rid of all the
|
38
|
+
# preloaded and cached instances of objects to make it work
|
39
|
+
def clear
|
40
|
+
Thread
|
41
|
+
.list
|
42
|
+
.select { |thread| thread[PERSISTENCE_SCOPE] }
|
43
|
+
.each { |thread| thread[PERSISTENCE_SCOPE].clear }
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
data/lib/karafka/process.rb
CHANGED
@@ -4,11 +4,11 @@ module Karafka
|
|
4
4
|
# Class used to catch signals from ruby Signal class in order to manage Karafka stop
|
5
5
|
# @note There might be only one process - this class is a singleton
|
6
6
|
class Process
|
7
|
-
include Singleton
|
8
|
-
|
9
7
|
# Signal types that we handle
|
10
8
|
HANDLED_SIGNALS = %i[
|
11
|
-
SIGINT
|
9
|
+
SIGINT
|
10
|
+
SIGQUIT
|
11
|
+
SIGTERM
|
12
12
|
].freeze
|
13
13
|
|
14
14
|
HANDLED_SIGNALS.each do |signal|
|
@@ -27,16 +27,13 @@ module Karafka
|
|
27
27
|
|
28
28
|
# Creates an instance of process and creates empty hash for callbacks
|
29
29
|
def initialize
|
30
|
-
@callbacks = {}
|
31
|
-
HANDLED_SIGNALS.each { |signal| @callbacks[signal] = [] }
|
30
|
+
@callbacks = Hash.new { |hsh, key| hsh[key] = [] }
|
32
31
|
end
|
33
32
|
|
34
33
|
# Method catches all HANDLED_SIGNALS and performs appropriate callbacks (if defined)
|
35
34
|
# @note If there are no callbacks, this method will just ignore a given signal that was sent
|
36
|
-
# @yield [Block] block of code that we want to execute and supervise
|
37
35
|
def supervise
|
38
36
|
HANDLED_SIGNALS.each { |signal| trap_signal(signal) }
|
39
|
-
yield
|
40
37
|
end
|
41
38
|
|
42
39
|
private
|
@@ -56,7 +53,7 @@ module Karafka
|
|
56
53
|
# we have to spin up a new thread to do this
|
57
54
|
def notice_signal(signal)
|
58
55
|
Thread.new do
|
59
|
-
Karafka.monitor.
|
56
|
+
Karafka.monitor.instrument('process.notice_signal', caller: self, signal: signal)
|
60
57
|
end
|
61
58
|
end
|
62
59
|
end
|
@@ -3,30 +3,31 @@
|
|
3
3
|
module Karafka
|
4
4
|
# Responders namespace encapsulates all the internal responder implementation parts
|
5
5
|
module Responders
|
6
|
-
# Responders builder is used
|
7
|
-
# match the
|
8
|
-
# but he still names responder with the same convention (and namespaces) as
|
6
|
+
# Responders builder is used for finding (based on the consumer class name) a responder
|
7
|
+
# that match the consumer. We use it when user does not provide a responder inside routing,
|
8
|
+
# but he still names responder with the same convention (and namespaces) as consumer
|
9
|
+
#
|
9
10
|
# @example Matching responder exists
|
10
|
-
# Karafka::Responder::Builder(
|
11
|
+
# Karafka::Responder::Builder(NewEventsConsumer).build #=> NewEventsResponder
|
11
12
|
# @example Matching responder does not exist
|
12
|
-
# Karafka::Responder::Builder(
|
13
|
+
# Karafka::Responder::Builder(NewBuildsConsumer).build #=> nil
|
13
14
|
class Builder
|
14
|
-
# @param
|
15
|
-
# Karafka::
|
16
|
-
# @example Tries to find a responder that matches a given
|
17
|
-
# will return nil (nil is accepted, because it means that a given
|
15
|
+
# @param consumer_class [Karafka::BaseConsumer, nil] descendant of
|
16
|
+
# Karafka::BaseConsumer
|
17
|
+
# @example Tries to find a responder that matches a given consumer. If nothing found,
|
18
|
+
# will return nil (nil is accepted, because it means that a given consumer don't
|
18
19
|
# pipe stuff further on)
|
19
|
-
def initialize(
|
20
|
-
@
|
20
|
+
def initialize(consumer_class)
|
21
|
+
@consumer_class = consumer_class
|
21
22
|
end
|
22
23
|
|
23
|
-
# Tries to figure out a responder based on a
|
24
|
+
# Tries to figure out a responder based on a consumer class name
|
24
25
|
# @return [Class] Responder class (not an instance)
|
25
26
|
# @return [nil] or nil if there's no matching responding class
|
26
27
|
def build
|
27
28
|
Helpers::ClassMatcher.new(
|
28
|
-
@
|
29
|
-
from: '
|
29
|
+
@consumer_class,
|
30
|
+
from: 'Consumer',
|
30
31
|
to: 'Responder'
|
31
32
|
).match
|
32
33
|
end
|
@@ -7,8 +7,6 @@ module Karafka
|
|
7
7
|
# Karafka::Responders::Topic.new(:topic_name, {}) #=> #<Karafka::Responders::Topic...
|
8
8
|
# @example Define optional topic
|
9
9
|
# Karafka::Responders::Topic.new(:topic_name, required: false)
|
10
|
-
# @example Define topic that on which we want to respond multiple times
|
11
|
-
# Karafka::Responders::Topic.new(:topic_name, multiple_usage: true)
|
12
10
|
class Topic
|
13
11
|
# Name of the topic on which we want to respond
|
14
12
|
attr_reader :name
|
@@ -26,16 +24,16 @@ module Karafka
|
|
26
24
|
@options.key?(:required) ? @options[:required] : true
|
27
25
|
end
|
28
26
|
|
29
|
-
# @return [Boolean] do we expect to use it multiple times in a single respond flow
|
30
|
-
def multiple_usage?
|
31
|
-
@options[:multiple_usage] || false
|
32
|
-
end
|
33
|
-
|
34
27
|
# @return [Boolean] was usage of this topic registered or not
|
35
28
|
def registered?
|
36
29
|
@options[:registered] == true
|
37
30
|
end
|
38
31
|
|
32
|
+
# @return [Class] Class to use to serialize messages for this topic
|
33
|
+
def serializer
|
34
|
+
@options[:serializer]
|
35
|
+
end
|
36
|
+
|
39
37
|
# @return [Boolean] do we want to use async producer. Defaults to false as the sync producer
|
40
38
|
# is safer and introduces less problems
|
41
39
|
def async?
|
@@ -46,9 +44,9 @@ module Karafka
|
|
46
44
|
def to_h
|
47
45
|
{
|
48
46
|
name: name,
|
49
|
-
multiple_usage: multiple_usage?,
|
50
47
|
required: required?,
|
51
48
|
registered: registered?,
|
49
|
+
serializer: serializer,
|
52
50
|
async: async?
|
53
51
|
}
|
54
52
|
end
|
@@ -6,29 +6,42 @@ module Karafka
|
|
6
6
|
# @example Build a simple (most common) route
|
7
7
|
# consumers do
|
8
8
|
# topic :new_videos do
|
9
|
-
#
|
9
|
+
# consumer NewVideosConsumer
|
10
10
|
# end
|
11
11
|
# end
|
12
|
-
class Builder < Array
|
13
|
-
|
12
|
+
class Builder < Concurrent::Array
|
13
|
+
# Consumer group consistency checking contract
|
14
|
+
CONTRACT = Karafka::Contracts::ConsumerGroup.new.freeze
|
15
|
+
|
16
|
+
private_constant :CONTRACT
|
17
|
+
|
18
|
+
def initialize
|
19
|
+
@draws = Concurrent::Array.new
|
20
|
+
end
|
14
21
|
|
15
22
|
# Used to draw routes for Karafka
|
23
|
+
# @param block [Proc] block we will evaluate within the builder context
|
24
|
+
# @yield Evaluates provided block in a builder context so we can describe routes
|
25
|
+
# @raise [Karafka::Errors::InvalidConfigurationError] raised when configuration
|
26
|
+
# doesn't match with the config contract
|
16
27
|
# @note After it is done drawing it will store and validate all the routes to make sure that
|
17
28
|
# they are correct and that there are no topic/group duplications (this is forbidden)
|
18
|
-
# @yield Evaluates provided block in a builder context so we can describe routes
|
19
29
|
# @example
|
20
30
|
# draw do
|
21
31
|
# topic :xyz do
|
22
32
|
# end
|
23
33
|
# end
|
24
34
|
def draw(&block)
|
35
|
+
@draws << block
|
36
|
+
|
25
37
|
instance_eval(&block)
|
26
38
|
|
27
39
|
each do |consumer_group|
|
28
40
|
hashed_group = consumer_group.to_h
|
29
|
-
validation_result =
|
30
|
-
|
31
|
-
|
41
|
+
validation_result = CONTRACT.call(hashed_group)
|
42
|
+
next if validation_result.success?
|
43
|
+
|
44
|
+
raise Errors::InvalidConfigurationError, validation_result.errors.to_h
|
32
45
|
end
|
33
46
|
end
|
34
47
|
|
@@ -39,18 +52,33 @@ module Karafka
|
|
39
52
|
select(&:active?)
|
40
53
|
end
|
41
54
|
|
55
|
+
# Clears the builder and the draws memory
|
56
|
+
def clear
|
57
|
+
@draws.clear
|
58
|
+
super
|
59
|
+
end
|
60
|
+
|
61
|
+
# Redraws all the routes for the in-process code reloading.
|
62
|
+
# @note This won't allow registration of new topics without process restart but will trigger
|
63
|
+
# cache invalidation so all the classes, etc are re-fetched after code reload
|
64
|
+
def reload
|
65
|
+
draws = @draws.dup
|
66
|
+
clear
|
67
|
+
draws.each { |block| draw(&block) }
|
68
|
+
end
|
69
|
+
|
42
70
|
private
|
43
71
|
|
44
72
|
# Builds and saves given consumer group
|
45
73
|
# @param group_id [String, Symbol] name for consumer group
|
46
|
-
# @
|
74
|
+
# @param block [Proc] proc that should be executed in the proxy context
|
47
75
|
def consumer_group(group_id, &block)
|
48
76
|
consumer_group = ConsumerGroup.new(group_id.to_s)
|
49
77
|
self << Proxy.new(consumer_group, &block).target
|
50
78
|
end
|
51
79
|
|
52
80
|
# @param topic_name [String, Symbol] name of a topic from which we want to consumer
|
53
|
-
# @
|
81
|
+
# @param block [Proc] proc we want to evaluate in the topic context
|
54
82
|
def topic(topic_name, &block)
|
55
83
|
consumer_group(topic_name) do
|
56
84
|
topic(topic_name, &block).tap(&:build)
|
@@ -29,7 +29,7 @@ module Karafka
|
|
29
29
|
|
30
30
|
# Builds a topic representation inside of a current consumer group route
|
31
31
|
# @param name [String, Symbol] name of topic to which we want to subscribe
|
32
|
-
# @
|
32
|
+
# @param block [Proc] block that we want to evaluate in the topic context
|
33
33
|
# @return [Karafka::Routing::Topic] newly built topic instance
|
34
34
|
def topic=(name, &block)
|
35
35
|
topic = Topic.new(name, self)
|
@@ -4,29 +4,30 @@ module Karafka
|
|
4
4
|
module Routing
|
5
5
|
# Default consumer mapper that builds consumer ids based on app id and consumer group name
|
6
6
|
# Different mapper can be used in case of preexisting consumer names or for applying
|
7
|
-
# other naming conventions not compatible
|
7
|
+
# other naming conventions not compatible with Karafka client_id + consumer name concept
|
8
8
|
#
|
9
9
|
# @example Mapper for using consumer groups without a client_id prefix
|
10
|
-
#
|
11
|
-
# def
|
10
|
+
# class MyMapper
|
11
|
+
# def call(raw_consumer_group_name)
|
12
12
|
# raw_consumer_group_name
|
13
13
|
# end
|
14
14
|
# end
|
15
15
|
#
|
16
16
|
# @example Mapper for replacing "_" with "." in topic names
|
17
|
-
#
|
18
|
-
# def
|
17
|
+
# class MyMapper
|
18
|
+
# def call(raw_consumer_group_name)
|
19
19
|
# [
|
20
|
-
# Karafka::App.config.client_id.to_s
|
20
|
+
# Karafka::Helpers::Inflector.map(Karafka::App.config.client_id.to_s),
|
21
21
|
# raw_consumer_group_name
|
22
22
|
# ].join('_').gsub('_', '.')
|
23
23
|
# end
|
24
24
|
# end
|
25
|
-
|
25
|
+
class ConsumerMapper
|
26
26
|
# @param raw_consumer_group_name [String, Symbol] string or symbolized consumer group name
|
27
27
|
# @return [String] remapped final consumer group name
|
28
|
-
def
|
29
|
-
|
28
|
+
def call(raw_consumer_group_name)
|
29
|
+
client_name = Karafka::Helpers::Inflector.map(Karafka::App.config.client_id.to_s)
|
30
|
+
"#{client_name}_#{raw_consumer_group_name}"
|
30
31
|
end
|
31
32
|
end
|
32
33
|
end
|