karafka 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.console_irbrc +13 -0
- data/.gitignore +68 -0
- data/.rspec +1 -0
- data/.ruby-gemset +1 -0
- data/.ruby-version +1 -0
- data/.travis.yml +18 -0
- data/CHANGELOG.md +415 -0
- data/CODE_OF_CONDUCT.md +46 -0
- data/CONTRIBUTING.md +41 -0
- data/Gemfile +11 -0
- data/Gemfile.lock +123 -0
- data/MIT-LICENCE +18 -0
- data/README.md +89 -0
- data/bin/karafka +19 -0
- data/config/errors.yml +6 -0
- data/karafka.gemspec +37 -0
- data/lib/karafka.rb +78 -0
- data/lib/karafka/app.rb +45 -0
- data/lib/karafka/attributes_map.rb +67 -0
- data/lib/karafka/backends/inline.rb +16 -0
- data/lib/karafka/base_consumer.rb +68 -0
- data/lib/karafka/base_responder.rb +204 -0
- data/lib/karafka/callbacks.rb +30 -0
- data/lib/karafka/callbacks/config.rb +22 -0
- data/lib/karafka/callbacks/dsl.rb +16 -0
- data/lib/karafka/cli.rb +54 -0
- data/lib/karafka/cli/base.rb +78 -0
- data/lib/karafka/cli/console.rb +29 -0
- data/lib/karafka/cli/flow.rb +46 -0
- data/lib/karafka/cli/info.rb +29 -0
- data/lib/karafka/cli/install.rb +42 -0
- data/lib/karafka/cli/server.rb +66 -0
- data/lib/karafka/connection/client.rb +117 -0
- data/lib/karafka/connection/config_adapter.rb +120 -0
- data/lib/karafka/connection/delegator.rb +46 -0
- data/lib/karafka/connection/listener.rb +60 -0
- data/lib/karafka/consumers/callbacks.rb +54 -0
- data/lib/karafka/consumers/includer.rb +51 -0
- data/lib/karafka/consumers/responders.rb +24 -0
- data/lib/karafka/consumers/single_params.rb +15 -0
- data/lib/karafka/errors.rb +50 -0
- data/lib/karafka/fetcher.rb +44 -0
- data/lib/karafka/helpers/class_matcher.rb +78 -0
- data/lib/karafka/helpers/config_retriever.rb +46 -0
- data/lib/karafka/helpers/multi_delegator.rb +33 -0
- data/lib/karafka/instrumentation/listener.rb +112 -0
- data/lib/karafka/instrumentation/logger.rb +55 -0
- data/lib/karafka/instrumentation/monitor.rb +64 -0
- data/lib/karafka/loader.rb +28 -0
- data/lib/karafka/params/dsl.rb +156 -0
- data/lib/karafka/params/params_batch.rb +46 -0
- data/lib/karafka/parsers/json.rb +38 -0
- data/lib/karafka/patches/dry_configurable.rb +35 -0
- data/lib/karafka/patches/ruby_kafka.rb +34 -0
- data/lib/karafka/persistence/client.rb +25 -0
- data/lib/karafka/persistence/consumer.rb +38 -0
- data/lib/karafka/persistence/topic.rb +29 -0
- data/lib/karafka/process.rb +64 -0
- data/lib/karafka/responders/builder.rb +36 -0
- data/lib/karafka/responders/topic.rb +57 -0
- data/lib/karafka/routing/builder.rb +61 -0
- data/lib/karafka/routing/consumer_group.rb +61 -0
- data/lib/karafka/routing/consumer_mapper.rb +34 -0
- data/lib/karafka/routing/proxy.rb +37 -0
- data/lib/karafka/routing/router.rb +29 -0
- data/lib/karafka/routing/topic.rb +60 -0
- data/lib/karafka/routing/topic_mapper.rb +55 -0
- data/lib/karafka/schemas/config.rb +24 -0
- data/lib/karafka/schemas/consumer_group.rb +77 -0
- data/lib/karafka/schemas/consumer_group_topic.rb +18 -0
- data/lib/karafka/schemas/responder_usage.rb +39 -0
- data/lib/karafka/schemas/server_cli_options.rb +43 -0
- data/lib/karafka/server.rb +94 -0
- data/lib/karafka/setup/config.rb +189 -0
- data/lib/karafka/setup/configurators/base.rb +29 -0
- data/lib/karafka/setup/configurators/params.rb +25 -0
- data/lib/karafka/setup/configurators/water_drop.rb +32 -0
- data/lib/karafka/setup/dsl.rb +22 -0
- data/lib/karafka/status.rb +25 -0
- data/lib/karafka/templates/application_consumer.rb.example +6 -0
- data/lib/karafka/templates/application_responder.rb.example +11 -0
- data/lib/karafka/templates/karafka.rb.example +54 -0
- data/lib/karafka/version.rb +7 -0
- data/log/.gitkeep +0 -0
- metadata +301 -0
@@ -0,0 +1,16 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Callbacks
|
5
|
+
# App level dsl to define callbacks
|
6
|
+
module Dsl
|
7
|
+
Callbacks::TYPES.each do |callback_type|
|
8
|
+
# Allows us to define a block, that will be executed for a given moment
|
9
|
+
# @param [Block] block that should be executed after the initialization process
|
10
|
+
define_method callback_type do |&block|
|
11
|
+
config.callbacks.send(callback_type).push block
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
data/lib/karafka/cli.rb
ADDED
@@ -0,0 +1,54 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
# Karafka framework Cli
|
5
|
+
# If you want to add/modify command that belongs to CLI, please review all commands
|
6
|
+
# available in cli/ directory inside Karafka source code.
|
7
|
+
#
|
8
|
+
# @note Whole Cli is built using Thor
|
9
|
+
# @see https://github.com/erikhuda/thor
|
10
|
+
class Cli < Thor
|
11
|
+
package_name 'Karafka'
|
12
|
+
|
13
|
+
class << self
|
14
|
+
# Loads all Cli commands into Thor framework
|
15
|
+
# This method should be executed before we run Karafka::Cli.start, otherwise we won't
|
16
|
+
# have any Cli commands available
|
17
|
+
def prepare
|
18
|
+
cli_commands.each do |action|
|
19
|
+
action.bind_to(self)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
# @return [Array<Class>] Array with Cli action classes that can be used as commands
|
26
|
+
def cli_commands
|
27
|
+
constants
|
28
|
+
.map! { |object| const_get(object) }
|
29
|
+
.keep_if do |object|
|
30
|
+
object.instance_of?(Class) && (object < Cli::Base)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
# This is kinda trick - since we don't have a autoload and other magic stuff
|
38
|
+
# like Rails does, so instead this method allows us to replace currently running
|
39
|
+
# console with a new one via Kernel.exec. It will start console with new code loaded
|
40
|
+
# Yes we know that it is not turbofast, however it is turbo convinient and small
|
41
|
+
#
|
42
|
+
# Also - the KARAFKA_CONSOLE is used to detect that we're executing the irb session
|
43
|
+
# so this method is only available when the Karafka console is running
|
44
|
+
#
|
45
|
+
# We skip this because this should exist and be only valid in the console
|
46
|
+
# :nocov:
|
47
|
+
if ENV['KARAFKA_CONSOLE']
|
48
|
+
# Reloads Karafka irb console session
|
49
|
+
def reload!
|
50
|
+
puts "Reloading...\n"
|
51
|
+
Kernel.exec Karafka::Cli::Console.command
|
52
|
+
end
|
53
|
+
end
|
54
|
+
# :nocov:
|
@@ -0,0 +1,78 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
class Cli < Thor
|
5
|
+
# Base class for all the command that we want to define
|
6
|
+
# This base class provides a nicer interface to Thor and allows to easier separate single
|
7
|
+
# independent commands
|
8
|
+
# In order to define a new command you need to:
|
9
|
+
# - specify its desc
|
10
|
+
# - implement call method
|
11
|
+
#
|
12
|
+
# @example Create a dummy command
|
13
|
+
# class Dummy < Base
|
14
|
+
# self.desc = 'Dummy command'
|
15
|
+
#
|
16
|
+
# def call
|
17
|
+
# puts 'I'm doing nothing!
|
18
|
+
# end
|
19
|
+
# end
|
20
|
+
class Base
|
21
|
+
include Thor::Shell
|
22
|
+
|
23
|
+
# We can use it to call other cli methods via this object
|
24
|
+
attr_reader :cli
|
25
|
+
|
26
|
+
# @param cli [Karafka::Cli] current Karafka Cli instance
|
27
|
+
def initialize(cli)
|
28
|
+
@cli = cli
|
29
|
+
end
|
30
|
+
|
31
|
+
# This method should implement proper cli action
|
32
|
+
def call
|
33
|
+
raise NotImplementedError, 'Implement this in a subclass'
|
34
|
+
end
|
35
|
+
|
36
|
+
class << self
|
37
|
+
# Allows to set options for Thor cli
|
38
|
+
# @see https://github.com/erikhuda/thor
|
39
|
+
# @param option Single option details
|
40
|
+
def option(*option)
|
41
|
+
@options ||= []
|
42
|
+
@options << option
|
43
|
+
end
|
44
|
+
|
45
|
+
# Allows to set description of a given cli command
|
46
|
+
# @param desc [String] Description of a given cli command
|
47
|
+
def desc(desc)
|
48
|
+
@desc ||= desc
|
49
|
+
end
|
50
|
+
|
51
|
+
# This method will bind a given Cli command into Karafka Cli
|
52
|
+
# This method is a wrapper to way Thor defines its commands
|
53
|
+
# @param cli_class [Karafka::Cli] Karafka cli_class
|
54
|
+
def bind_to(cli_class)
|
55
|
+
cli_class.desc name, @desc
|
56
|
+
|
57
|
+
(@options || []).each { |option| cli_class.option(*option) }
|
58
|
+
|
59
|
+
context = self
|
60
|
+
|
61
|
+
cli_class.send :define_method, name do |*args|
|
62
|
+
context.new(self).call(*args)
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
private
|
67
|
+
|
68
|
+
# @return [String] downcased current class name that we use to define name for
|
69
|
+
# given Cli command
|
70
|
+
# @example for Karafka::Cli::Install
|
71
|
+
# name #=> 'install'
|
72
|
+
def name
|
73
|
+
to_s.split('::').last.downcase
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
# Karafka framework Cli
|
5
|
+
class Cli < Thor
|
6
|
+
# Console Karafka Cli action
|
7
|
+
class Console < Base
|
8
|
+
desc 'Start the Karafka console (short-cut alias: "c")'
|
9
|
+
option aliases: 'c'
|
10
|
+
|
11
|
+
# @return [String] Console executing command
|
12
|
+
# @example
|
13
|
+
# Karafka::Cli::Console.command #=> 'KARAFKA_CONSOLE=true bundle exec irb...'
|
14
|
+
def self.command
|
15
|
+
envs = [
|
16
|
+
"IRBRC='#{Karafka.gem_root}/.console_irbrc'",
|
17
|
+
'KARAFKA_CONSOLE=true'
|
18
|
+
]
|
19
|
+
"#{envs.join(' ')} bundle exec irb"
|
20
|
+
end
|
21
|
+
|
22
|
+
# Start the Karafka console
|
23
|
+
def call
|
24
|
+
cli.info
|
25
|
+
exec self.class.command
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
# Karafka framework Cli
|
5
|
+
class Cli < Thor
|
6
|
+
# Description of topics flow (incoming/outgoing)
|
7
|
+
class Flow < Base
|
8
|
+
desc 'Print application data flow (incoming => outgoing)'
|
9
|
+
|
10
|
+
# Print out all defined routes in alphabetical order
|
11
|
+
def call
|
12
|
+
topics.each do |topic|
|
13
|
+
any_topics = !topic.responder&.topics.nil?
|
14
|
+
|
15
|
+
if any_topics
|
16
|
+
puts "#{topic.name} =>"
|
17
|
+
|
18
|
+
topic.responder.topics.each_value do |responder_topic|
|
19
|
+
features = []
|
20
|
+
features << (responder_topic.required? ? 'always' : 'conditionally')
|
21
|
+
features << (responder_topic.multiple_usage? ? 'one or more' : 'exactly once')
|
22
|
+
|
23
|
+
print responder_topic.name, "(#{features.join(', ')})"
|
24
|
+
end
|
25
|
+
else
|
26
|
+
puts "#{topic.name} => (nothing)"
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
# @return [Array<Karafka::Routing::Topic>] all topics sorted in alphabetical order
|
34
|
+
def topics
|
35
|
+
Karafka::App.consumer_groups.map(&:topics).flatten.sort_by(&:name)
|
36
|
+
end
|
37
|
+
|
38
|
+
# Prints a given value with label in a nice way
|
39
|
+
# @param label [String] label describing value
|
40
|
+
# @param value [String] value that should be printed
|
41
|
+
def print(label, value)
|
42
|
+
printf "%-25s %s\n", " - #{label}:", value
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
# Karafka framework Cli
|
5
|
+
class Cli < Thor
|
6
|
+
# Info Karafka Cli action
|
7
|
+
class Info < Base
|
8
|
+
desc 'Print configuration details and other options of your application'
|
9
|
+
|
10
|
+
# Print configuration details and other options of your application
|
11
|
+
def call
|
12
|
+
config = Karafka::App.config
|
13
|
+
|
14
|
+
info = [
|
15
|
+
"Karafka framework version: #{Karafka::VERSION}",
|
16
|
+
"Application client id: #{config.client_id}",
|
17
|
+
"Backend: #{config.backend}",
|
18
|
+
"Batch fetching: #{config.batch_fetching}",
|
19
|
+
"Batch consuming: #{config.batch_consuming}",
|
20
|
+
"Boot file: #{Karafka.boot_file}",
|
21
|
+
"Environment: #{Karafka.env}",
|
22
|
+
"Kafka seed brokers: #{config.kafka.seed_brokers}"
|
23
|
+
]
|
24
|
+
|
25
|
+
puts(info.join("\n"))
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
# Karafka framework Cli
|
5
|
+
class Cli < Thor
|
6
|
+
# Install Karafka Cli action
|
7
|
+
class Install < Base
|
8
|
+
desc 'Install all required things for Karafka application in current directory'
|
9
|
+
|
10
|
+
# Directories created by default
|
11
|
+
INSTALL_DIRS = %w[
|
12
|
+
app/consumers
|
13
|
+
app/responders
|
14
|
+
config
|
15
|
+
log
|
16
|
+
tmp/pids
|
17
|
+
].freeze
|
18
|
+
|
19
|
+
# Where should we map proper files from templates
|
20
|
+
INSTALL_FILES_MAP = {
|
21
|
+
'karafka.rb.example' => Karafka.boot_file.basename,
|
22
|
+
'application_consumer.rb.example' => 'app/consumers/application_consumer.rb',
|
23
|
+
'application_responder.rb.example' => 'app/responders/application_responder.rb'
|
24
|
+
}.freeze
|
25
|
+
|
26
|
+
# Install all required things for Karafka application in current directory
|
27
|
+
def call
|
28
|
+
INSTALL_DIRS.each do |dir|
|
29
|
+
FileUtils.mkdir_p Karafka.root.join(dir)
|
30
|
+
end
|
31
|
+
|
32
|
+
INSTALL_FILES_MAP.each do |source, target|
|
33
|
+
target = Karafka.root.join(target)
|
34
|
+
next if File.exist?(target)
|
35
|
+
|
36
|
+
source = Karafka.core_root.join("templates/#{source}")
|
37
|
+
FileUtils.cp_r(source, target)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,66 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
# Karafka framework Cli
|
5
|
+
class Cli < Thor
|
6
|
+
# Server Karafka Cli action
|
7
|
+
class Server < Base
|
8
|
+
desc 'Start the Karafka server (short-cut alias: "s")'
|
9
|
+
option aliases: 's'
|
10
|
+
option :daemon, default: false, type: :boolean, aliases: :d
|
11
|
+
option :pid, default: 'tmp/pids/karafka', type: :string, aliases: :p
|
12
|
+
option :consumer_groups, type: :array, default: nil, aliases: :g
|
13
|
+
|
14
|
+
# Start the Karafka server
|
15
|
+
def call
|
16
|
+
validate!
|
17
|
+
|
18
|
+
puts 'Starting Karafka server'
|
19
|
+
cli.info
|
20
|
+
|
21
|
+
if cli.options[:daemon]
|
22
|
+
FileUtils.mkdir_p File.dirname(cli.options[:pid])
|
23
|
+
daemonize
|
24
|
+
end
|
25
|
+
|
26
|
+
# We assign active topics on a server level, as only server is expected to listen on
|
27
|
+
# part of the topics
|
28
|
+
Karafka::Server.consumer_groups = cli.options[:consumer_groups]
|
29
|
+
|
30
|
+
# Remove pidfile on stop, just before the server instance is going to be GCed
|
31
|
+
# We want to delay the moment in which the pidfile is removed as much as we can,
|
32
|
+
# so instead of removing it after the server stops running, we rely on the gc moment
|
33
|
+
# when this object gets removed (it is a bit later), so it is closer to the actual
|
34
|
+
# system process end. We do that, so monitoring and deployment tools that rely on pids
|
35
|
+
# won't alarm or start new system process up until the current one is finished
|
36
|
+
ObjectSpace.define_finalizer(self, proc { send(:clean) })
|
37
|
+
|
38
|
+
Karafka::Server.run
|
39
|
+
end
|
40
|
+
|
41
|
+
private
|
42
|
+
|
43
|
+
# Checks the server cli configuration
|
44
|
+
# options validations in terms of app setup (topics, pid existence, etc)
|
45
|
+
def validate!
|
46
|
+
result = Schemas::ServerCliOptions.call(cli.options)
|
47
|
+
return if result.success?
|
48
|
+
raise Errors::InvalidConfiguration, result.errors
|
49
|
+
end
|
50
|
+
|
51
|
+
# Detaches current process into background and writes its pidfile
|
52
|
+
def daemonize
|
53
|
+
::Process.daemon(true)
|
54
|
+
File.open(
|
55
|
+
cli.options[:pid],
|
56
|
+
'w'
|
57
|
+
) { |file| file.write(::Process.pid) }
|
58
|
+
end
|
59
|
+
|
60
|
+
# Removes a pidfile (if exist)
|
61
|
+
def clean
|
62
|
+
FileUtils.rm_f(cli.options[:pid]) if cli.options[:pid]
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
@@ -0,0 +1,117 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Connection
|
5
|
+
# Class used as a wrapper around Ruby-Kafka client to simplify additional
|
6
|
+
# features that we provide/might provide in future and to hide the internal implementation
|
7
|
+
class Client
|
8
|
+
extend Forwardable
|
9
|
+
|
10
|
+
def_delegator :kafka_consumer, :seek
|
11
|
+
|
12
|
+
# Creates a queue consumer client that will pull the data from Kafka
|
13
|
+
# @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group for which
|
14
|
+
# we create a client
|
15
|
+
# @return [Karafka::Connection::Client] group consumer that can subscribe to
|
16
|
+
# multiple topics
|
17
|
+
def initialize(consumer_group)
|
18
|
+
@consumer_group = consumer_group
|
19
|
+
Persistence::Client.write(self)
|
20
|
+
end
|
21
|
+
|
22
|
+
# Opens connection, gets messages and calls a block for each of the incoming messages
|
23
|
+
# @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
|
24
|
+
# @note This will yield with raw messages - no preprocessing or reformatting.
|
25
|
+
def fetch_loop
|
26
|
+
settings = ConfigAdapter.consuming(consumer_group)
|
27
|
+
|
28
|
+
if consumer_group.batch_fetching
|
29
|
+
kafka_consumer.each_batch(*settings) { |batch| yield(batch.messages) }
|
30
|
+
else
|
31
|
+
# always yield an array of messages, so we have consistent API (always a batch)
|
32
|
+
kafka_consumer.each_message(*settings) { |message| yield([message]) }
|
33
|
+
end
|
34
|
+
rescue Kafka::ProcessingError => error
|
35
|
+
# If there was an error during consumption, we have to log it, pause current partition
|
36
|
+
# and process other things
|
37
|
+
Karafka.monitor.instrument(
|
38
|
+
'connection.client.fetch_loop.error',
|
39
|
+
caller: self,
|
40
|
+
error: error.cause
|
41
|
+
)
|
42
|
+
pause(error.topic, error.partition)
|
43
|
+
retry
|
44
|
+
# This is on purpose - see the notes for this method
|
45
|
+
# rubocop:disable RescueException
|
46
|
+
rescue Exception => error
|
47
|
+
# rubocop:enable RescueException
|
48
|
+
Karafka.monitor.instrument(
|
49
|
+
'connection.client.fetch_loop.error',
|
50
|
+
caller: self,
|
51
|
+
error: error
|
52
|
+
)
|
53
|
+
retry
|
54
|
+
end
|
55
|
+
|
56
|
+
# Gracefuly stops topic consumption
|
57
|
+
# @note Stopping running consumers without a really important reason is not recommended
|
58
|
+
# as until all the consumers are stopped, the server will keep running serving only
|
59
|
+
# part of the messages
|
60
|
+
def stop
|
61
|
+
@kafka_consumer&.stop
|
62
|
+
@kafka_consumer = nil
|
63
|
+
end
|
64
|
+
|
65
|
+
# Pauses fetching and consumption of a given topic partition
|
66
|
+
# @param topic [String] topic that we want to pause
|
67
|
+
# @param partition [Integer] number partition that we want to pause
|
68
|
+
def pause(topic, partition)
|
69
|
+
settings = ConfigAdapter.pausing(consumer_group)
|
70
|
+
timeout = settings[:timeout]
|
71
|
+
raise(Errors::InvalidPauseTimeout, timeout) unless timeout.positive?
|
72
|
+
kafka_consumer.pause(topic, partition, settings)
|
73
|
+
end
|
74
|
+
|
75
|
+
# Marks a given message as consumed and commit the offsets
|
76
|
+
# @note In opposite to ruby-kafka, we commit the offset for each manual marking to be sure
|
77
|
+
# that offset commit happen asap in case of a crash
|
78
|
+
# @param [Karafka::Params::Params] params message that we want to mark as processed
|
79
|
+
def mark_as_consumed(params)
|
80
|
+
kafka_consumer.mark_message_as_processed(params)
|
81
|
+
# Trigger an immediate, blocking offset commit in order to minimize the risk of crashing
|
82
|
+
# before the automatic triggers have kicked in.
|
83
|
+
kafka_consumer.commit_offsets
|
84
|
+
end
|
85
|
+
|
86
|
+
private
|
87
|
+
|
88
|
+
attr_reader :consumer_group
|
89
|
+
|
90
|
+
# @return [Kafka::Consumer] returns a ready to consume Kafka consumer
|
91
|
+
# that is set up to consume from topics of a given consumer group
|
92
|
+
def kafka_consumer
|
93
|
+
@kafka_consumer ||= kafka.consumer(
|
94
|
+
*ConfigAdapter.consumer(consumer_group)
|
95
|
+
).tap do |consumer|
|
96
|
+
consumer_group.topics.each do |topic|
|
97
|
+
consumer.subscribe(*ConfigAdapter.subscription(topic))
|
98
|
+
end
|
99
|
+
end
|
100
|
+
rescue Kafka::ConnectionError
|
101
|
+
# If we would not wait it would totally spam log file with failed
|
102
|
+
# attempts if Kafka is down
|
103
|
+
sleep(consumer_group.reconnect_timeout)
|
104
|
+
# We don't log and just reraise - this will be logged
|
105
|
+
# down the road
|
106
|
+
raise
|
107
|
+
end
|
108
|
+
|
109
|
+
# @return [Kafka] returns a Kafka
|
110
|
+
# @note We don't cache it internally because we cache kafka_consumer that uses kafka
|
111
|
+
# object instance
|
112
|
+
def kafka
|
113
|
+
Kafka.new(*ConfigAdapter.client(consumer_group))
|
114
|
+
end
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|