karafka 1.2.12 → 1.3.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (107) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data.tar.gz.sig +0 -0
  4. data/{.coditsu.yml → .coditsu/ci.yml} +1 -1
  5. data/.console_irbrc +1 -3
  6. data/.github/FUNDING.yml +3 -0
  7. data/.github/ISSUE_TEMPLATE/bug_report.md +50 -0
  8. data/.github/ISSUE_TEMPLATE/feature_request.md +20 -0
  9. data/.gitignore +1 -0
  10. data/.ruby-version +1 -1
  11. data/.travis.yml +8 -21
  12. data/CHANGELOG.md +87 -15
  13. data/CONTRIBUTING.md +1 -1
  14. data/Gemfile +2 -5
  15. data/Gemfile.lock +74 -59
  16. data/README.md +10 -11
  17. data/bin/karafka +1 -1
  18. data/certs/mensfeld.pem +25 -0
  19. data/config/errors.yml +38 -5
  20. data/karafka.gemspec +14 -12
  21. data/lib/karafka.rb +8 -15
  22. data/lib/karafka/app.rb +14 -6
  23. data/lib/karafka/attributes_map.rb +4 -5
  24. data/lib/karafka/base_consumer.rb +19 -30
  25. data/lib/karafka/base_responder.rb +45 -27
  26. data/lib/karafka/cli.rb +1 -1
  27. data/lib/karafka/cli/console.rb +11 -9
  28. data/lib/karafka/cli/flow.rb +0 -1
  29. data/lib/karafka/cli/info.rb +3 -1
  30. data/lib/karafka/cli/install.rb +30 -6
  31. data/lib/karafka/cli/server.rb +11 -6
  32. data/lib/karafka/code_reloader.rb +67 -0
  33. data/lib/karafka/connection/api_adapter.rb +11 -4
  34. data/lib/karafka/connection/batch_delegator.rb +51 -0
  35. data/lib/karafka/connection/builder.rb +1 -1
  36. data/lib/karafka/connection/client.rb +30 -20
  37. data/lib/karafka/connection/listener.rb +24 -13
  38. data/lib/karafka/connection/message_delegator.rb +36 -0
  39. data/lib/karafka/consumers/callbacks.rb +32 -15
  40. data/lib/karafka/consumers/includer.rb +30 -18
  41. data/lib/karafka/consumers/metadata.rb +10 -0
  42. data/lib/karafka/consumers/responders.rb +2 -2
  43. data/lib/karafka/contracts.rb +10 -0
  44. data/lib/karafka/contracts/config.rb +21 -0
  45. data/lib/karafka/contracts/consumer_group.rb +206 -0
  46. data/lib/karafka/contracts/consumer_group_topic.rb +19 -0
  47. data/lib/karafka/contracts/responder_usage.rb +54 -0
  48. data/lib/karafka/contracts/server_cli_options.rb +29 -0
  49. data/lib/karafka/errors.rb +17 -16
  50. data/lib/karafka/fetcher.rb +28 -30
  51. data/lib/karafka/helpers/class_matcher.rb +6 -2
  52. data/lib/karafka/helpers/config_retriever.rb +1 -1
  53. data/lib/karafka/helpers/inflector.rb +26 -0
  54. data/lib/karafka/helpers/multi_delegator.rb +0 -1
  55. data/lib/karafka/instrumentation/logger.rb +9 -6
  56. data/lib/karafka/instrumentation/monitor.rb +15 -9
  57. data/lib/karafka/instrumentation/proctitle_listener.rb +36 -0
  58. data/lib/karafka/instrumentation/stdout_listener.rb +138 -0
  59. data/lib/karafka/params/builders/metadata.rb +33 -0
  60. data/lib/karafka/params/builders/params.rb +36 -0
  61. data/lib/karafka/params/builders/params_batch.rb +25 -0
  62. data/lib/karafka/params/metadata.rb +35 -0
  63. data/lib/karafka/params/params.rb +68 -0
  64. data/lib/karafka/params/params_batch.rb +35 -20
  65. data/lib/karafka/patches/ruby_kafka.rb +21 -8
  66. data/lib/karafka/persistence/client.rb +15 -11
  67. data/lib/karafka/persistence/{consumer.rb → consumers.rb} +20 -13
  68. data/lib/karafka/persistence/topics.rb +48 -0
  69. data/lib/karafka/process.rb +0 -2
  70. data/lib/karafka/responders/topic.rb +6 -8
  71. data/lib/karafka/routing/builder.rb +36 -8
  72. data/lib/karafka/routing/consumer_group.rb +1 -1
  73. data/lib/karafka/routing/consumer_mapper.rb +9 -9
  74. data/lib/karafka/routing/proxy.rb +10 -1
  75. data/lib/karafka/routing/topic.rb +5 -3
  76. data/lib/karafka/routing/topic_mapper.rb +16 -18
  77. data/lib/karafka/serialization/json/deserializer.rb +27 -0
  78. data/lib/karafka/serialization/json/serializer.rb +31 -0
  79. data/lib/karafka/server.rb +25 -27
  80. data/lib/karafka/setup/config.rb +65 -37
  81. data/lib/karafka/setup/configurators/water_drop.rb +7 -3
  82. data/lib/karafka/setup/dsl.rb +0 -1
  83. data/lib/karafka/status.rb +7 -3
  84. data/lib/karafka/templates/{application_consumer.rb.example → application_consumer.rb.erb} +2 -1
  85. data/lib/karafka/templates/{application_responder.rb.example → application_responder.rb.erb} +0 -0
  86. data/lib/karafka/templates/karafka.rb.erb +92 -0
  87. data/lib/karafka/version.rb +1 -1
  88. metadata +90 -57
  89. metadata.gz.sig +1 -0
  90. data/lib/karafka/callbacks.rb +0 -30
  91. data/lib/karafka/callbacks/config.rb +0 -22
  92. data/lib/karafka/callbacks/dsl.rb +0 -16
  93. data/lib/karafka/connection/delegator.rb +0 -46
  94. data/lib/karafka/instrumentation/listener.rb +0 -112
  95. data/lib/karafka/loader.rb +0 -28
  96. data/lib/karafka/params/dsl.rb +0 -158
  97. data/lib/karafka/parsers/json.rb +0 -38
  98. data/lib/karafka/patches/dry_configurable.rb +0 -33
  99. data/lib/karafka/persistence/topic.rb +0 -29
  100. data/lib/karafka/schemas/config.rb +0 -24
  101. data/lib/karafka/schemas/consumer_group.rb +0 -78
  102. data/lib/karafka/schemas/consumer_group_topic.rb +0 -18
  103. data/lib/karafka/schemas/responder_usage.rb +0 -39
  104. data/lib/karafka/schemas/server_cli_options.rb +0 -43
  105. data/lib/karafka/setup/configurators/base.rb +0 -29
  106. data/lib/karafka/setup/configurators/params.rb +0 -25
  107. data/lib/karafka/templates/karafka.rb.example +0 -54
@@ -37,7 +37,7 @@ end
37
37
  # This is kinda trick - since we don't have a autoload and other magic stuff
38
38
  # like Rails does, so instead this method allows us to replace currently running
39
39
  # console with a new one via Kernel.exec. It will start console with new code loaded
40
- # Yes we know that it is not turbofast, however it is turbo convinient and small
40
+ # Yes, we know that it is not turbo fast, however it is turbo convenient and small
41
41
  #
42
42
  # Also - the KARAFKA_CONSOLE is used to detect that we're executing the irb session
43
43
  # so this method is only available when the Karafka console is running
@@ -8,15 +8,17 @@ module Karafka
8
8
  desc 'Start the Karafka console (short-cut alias: "c")'
9
9
  option aliases: 'c'
10
10
 
11
- # @return [String] Console executing command
12
- # @example
13
- # Karafka::Cli::Console.command #=> 'KARAFKA_CONSOLE=true bundle exec irb...'
14
- def self.command
15
- envs = [
16
- "IRBRC='#{Karafka.gem_root}/.console_irbrc'",
17
- 'KARAFKA_CONSOLE=true'
18
- ]
19
- "#{envs.join(' ')} bundle exec irb"
11
+ class << self
12
+ # @return [String] Console executing command
13
+ # @example
14
+ # Karafka::Cli::Console.command #=> 'KARAFKA_CONSOLE=true bundle exec irb...'
15
+ def command
16
+ envs = [
17
+ "IRBRC='#{Karafka.gem_root}/.console_irbrc'",
18
+ 'KARAFKA_CONSOLE=true'
19
+ ]
20
+ "#{envs.join(' ')} bundle exec irb -r #{Karafka.boot_file}"
21
+ end
20
22
  end
21
23
 
22
24
  # Start the Karafka console
@@ -18,7 +18,6 @@ module Karafka
18
18
  topic.responder.topics.each_value do |responder_topic|
19
19
  features = []
20
20
  features << (responder_topic.required? ? 'always' : 'conditionally')
21
- features << (responder_topic.multiple_usage? ? 'one or more' : 'exactly once')
22
21
 
23
22
  print responder_topic.name, "(#{features.join(', ')})"
24
23
  end
@@ -12,7 +12,9 @@ module Karafka
12
12
  config = Karafka::App.config
13
13
 
14
14
  info = [
15
- "Karafka framework version: #{Karafka::VERSION}",
15
+ "Karafka version: #{Karafka::VERSION}",
16
+ "Ruby version: #{RUBY_VERSION}",
17
+ "Ruby-kafka version: #{::Kafka::VERSION}",
16
18
  "Application client id: #{config.client_id}",
17
19
  "Backend: #{config.backend}",
18
20
  "Batch fetching: #{config.batch_fetching}",
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'erb'
4
+
3
5
  module Karafka
4
6
  # Karafka framework Cli
5
7
  class Cli < Thor
@@ -11,18 +13,30 @@ module Karafka
11
13
  INSTALL_DIRS = %w[
12
14
  app/consumers
13
15
  app/responders
16
+ app/workers
14
17
  config
18
+ lib
15
19
  log
16
20
  tmp/pids
17
21
  ].freeze
18
22
 
19
23
  # Where should we map proper files from templates
20
24
  INSTALL_FILES_MAP = {
21
- 'karafka.rb.example' => Karafka.boot_file.basename,
22
- 'application_consumer.rb.example' => 'app/consumers/application_consumer.rb',
23
- 'application_responder.rb.example' => 'app/responders/application_responder.rb'
25
+ 'karafka.rb.erb' => Karafka.boot_file.basename,
26
+ 'application_consumer.rb.erb' => 'app/consumers/application_consumer.rb',
27
+ 'application_responder.rb.erb' => 'app/responders/application_responder.rb'
24
28
  }.freeze
25
29
 
30
+ # @param args [Array] all the things that Thor CLI accepts
31
+ def initialize(*args)
32
+ super
33
+ @rails = Bundler::LockfileParser.new(
34
+ Bundler.read_file(
35
+ Bundler.default_lockfile
36
+ )
37
+ ).dependencies.key?('rails')
38
+ end
39
+
26
40
  # Install all required things for Karafka application in current directory
27
41
  def call
28
42
  INSTALL_DIRS.each do |dir|
@@ -31,12 +45,22 @@ module Karafka
31
45
 
32
46
  INSTALL_FILES_MAP.each do |source, target|
33
47
  target = Karafka.root.join(target)
34
- next if File.exist?(target)
35
48
 
36
- source = Karafka.core_root.join("templates/#{source}")
37
- FileUtils.cp_r(source, target)
49
+ template = File.read(Karafka.core_root.join("templates/#{source}"))
50
+ # @todo Replace with the keyword argument version once we don't have to support
51
+ # Ruby < 2.6
52
+ render = ::ERB.new(template, nil, '-').result(binding)
53
+
54
+ File.open(target, 'w') { |file| file.write(render) }
38
55
  end
39
56
  end
57
+
58
+ # @return [Boolean] true if we have Rails loaded
59
+ # This allows us to generate customized karafka.rb template with some tweaks specific for
60
+ # Rails
61
+ def rails?
62
+ @rails
63
+ end
40
64
  end
41
65
  end
42
66
  end
@@ -5,6 +5,11 @@ module Karafka
5
5
  class Cli < Thor
6
6
  # Server Karafka Cli action
7
7
  class Server < Base
8
+ # Server config settings contract
9
+ CONTRACT = Contracts::ServerCliOptions.new.freeze
10
+
11
+ private_constant :CONTRACT
12
+
8
13
  desc 'Start the Karafka server (short-cut alias: "s")'
9
14
  option aliases: 's'
10
15
  option :daemon, default: false, type: :boolean, aliases: :d
@@ -13,11 +18,10 @@ module Karafka
13
18
 
14
19
  # Start the Karafka server
15
20
  def call
16
- validate!
17
-
18
- puts 'Starting Karafka server'
19
21
  cli.info
20
22
 
23
+ validate!
24
+
21
25
  if cli.options[:daemon]
22
26
  FileUtils.mkdir_p File.dirname(cli.options[:pid])
23
27
  daemonize
@@ -31,7 +35,7 @@ module Karafka
31
35
  # We want to delay the moment in which the pidfile is removed as much as we can,
32
36
  # so instead of removing it after the server stops running, we rely on the gc moment
33
37
  # when this object gets removed (it is a bit later), so it is closer to the actual
34
- # system process end. We do that, so monitoring and deployment tools that rely on pids
38
+ # system process end. We do that, so monitoring and deployment tools that rely on a pid
35
39
  # won't alarm or start new system process up until the current one is finished
36
40
  ObjectSpace.define_finalizer(self, proc { send(:clean) })
37
41
 
@@ -43,9 +47,10 @@ module Karafka
43
47
  # Checks the server cli configuration
44
48
  # options validations in terms of app setup (topics, pid existence, etc)
45
49
  def validate!
46
- result = Schemas::ServerCliOptions.call(cli.options)
50
+ result = CONTRACT.call(cli.options)
47
51
  return if result.success?
48
- raise Errors::InvalidConfiguration, result.errors
52
+
53
+ raise Errors::InvalidConfigurationError, result.errors.to_h
49
54
  end
50
55
 
51
56
  # Detaches current process into background and writes its pidfile
@@ -0,0 +1,67 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ # Special type of a listener, that is not an instrumentation one, but one that triggers
5
+ # code reload in the development mode after each fetched batch (or message)
6
+ #
7
+ # Please refer to the development code reload sections for details on the benefits and downsides
8
+ # of the in-process code reloading
9
+ class CodeReloader
10
+ # This mutex is needed as we might have an application that has multiple consumer groups
11
+ # running in separate threads and we should not trigger reload before fully reloading the app
12
+ # in previous thread
13
+ MUTEX = Mutex.new
14
+
15
+ private_constant :MUTEX
16
+
17
+ # @param reloaders [Array<Object>] any code loaders that we use in this app. Whether it is
18
+ # the Rails loader, Zeitwerk or anything else that allows reloading triggering
19
+ # @param block [Proc] yields given block just before reloading. This can be used to hook custom
20
+ # reloading stuff, that ain't reloaders (for example for resetting dry-events registry)
21
+ def initialize(*reloaders, &block)
22
+ @reloaders = reloaders
23
+ @block = block
24
+ end
25
+
26
+ # Binds to the instrumentation events and triggers reload
27
+ # @param _event [Dry::Event] empty dry event
28
+ # @note Since we de-register all the user defined objects and redraw routes, it means that
29
+ # we won't be able to do a multi-batch buffering in the development mode as each of the
30
+ # batches will be buffered on a newly created "per fetch" instance.
31
+ def on_connection_listener_fetch_loop(_event)
32
+ reload
33
+ end
34
+
35
+ private
36
+
37
+ # Triggers reload of both standard and Rails reloaders as well as expires all internals of
38
+ # Karafka, so it can be rediscovered and rebuilt
39
+ def reload
40
+ MUTEX.synchronize do
41
+ if @reloaders[0].respond_to?(:execute)
42
+ reload_with_rails
43
+ else
44
+ reload_without_rails
45
+ end
46
+ end
47
+ end
48
+
49
+ # Rails reloading procedure
50
+ def reload_with_rails
51
+ updatable = @reloaders.select(&:updated?)
52
+
53
+ return if updatable.empty?
54
+
55
+ updatable.each(&:execute)
56
+ @block&.call
57
+ Karafka::App.reload
58
+ end
59
+
60
+ # Zeitwerk and other reloaders
61
+ def reload_without_rails
62
+ @reloaders.each(&:reload)
63
+ @block&.call
64
+ Karafka::App.reload
65
+ end
66
+ end
67
+ end
@@ -87,7 +87,11 @@ module Karafka
87
87
  [
88
88
  Karafka::App.config.topic_mapper.outgoing(topic),
89
89
  partition,
90
- { timeout: consumer_group.pause_timeout }
90
+ {
91
+ timeout: consumer_group.pause_timeout,
92
+ max_timeout: consumer_group.pause_max_timeout,
93
+ exponential_backoff: consumer_group.pause_exponential_backoff
94
+ }
91
95
  ]
92
96
  end
93
97
 
@@ -98,9 +102,10 @@ module Karafka
98
102
  # @note When default empty topic mapper is used, no need for any conversion as the
99
103
  # internal and external format are exactly the same
100
104
  def mark_message_as_processed(params)
101
- # Majority of non heroku users don't use custom topic mappers. No need to change
102
- # anything when it is a default mapper that does not change anything
103
- return [params] if Karafka::App.config.topic_mapper == Karafka::Routing::TopicMapper
105
+ # Majority of users don't use custom topic mappers. No need to change anything when it
106
+ # is a default mapper that does not change anything. Only some cloud providers require
107
+ # topics to be remapped
108
+ return [params] if Karafka::App.config.topic_mapper.is_a?(Karafka::Routing::TopicMapper)
104
109
 
105
110
  # @note We don't use tap as it is around 13% slower than non-dup version
106
111
  dupped = params.dup
@@ -119,9 +124,11 @@ module Karafka
119
124
  kafka_configs.each_key do |setting_name|
120
125
  # Ignore settings that are not related to our namespace
121
126
  next unless AttributesMap.api_adapter[namespace_key].include?(setting_name)
127
+
122
128
  # Ignore settings that are already initialized
123
129
  # In case they are in preexisting settings fetched differently
124
130
  next if preexisting_settings.key?(setting_name)
131
+
125
132
  # Fetch all the settings from a given layer object. Objects can handle the fallback
126
133
  # to the kafka settings, so
127
134
  preexisting_settings[setting_name] = route_layer.send(setting_name)
@@ -0,0 +1,51 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Connection
5
+ # Class that delegates processing of batch received messages for which we listen to
6
+ # a proper processor
7
+ module BatchDelegator
8
+ class << self
9
+ # Delegates messages (does something with them)
10
+ # It will either schedule or run a proper processor action for messages
11
+ # @param group_id [String] group_id of a group from which a given message came
12
+ # @param kafka_batch [<Kafka::FetchedBatch>] raw messages fetched batch
13
+ # @note This should be looped to obtain a constant delegating of new messages
14
+ def call(group_id, kafka_batch)
15
+ topic = Persistence::Topics.fetch(group_id, kafka_batch.topic)
16
+ consumer = Persistence::Consumers.fetch(topic, kafka_batch.partition)
17
+
18
+ Karafka.monitor.instrument(
19
+ 'connection.batch_delegator.call',
20
+ caller: self,
21
+ consumer: consumer,
22
+ kafka_batch: kafka_batch
23
+ ) do
24
+ # Due to how ruby-kafka is built, we have the metadata that is stored on the batch
25
+ # level only available for batch consuming
26
+ consumer.metadata = Params::Builders::Metadata.from_kafka_batch(kafka_batch, topic)
27
+ kafka_messages = kafka_batch.messages
28
+
29
+ # Depending on a case (persisted or not) we might use new consumer instance per
30
+ # each batch, or use the same one for all of them (for implementing buffering, etc.)
31
+ if topic.batch_consuming
32
+ consumer.params_batch = Params::Builders::ParamsBatch.from_kafka_messages(
33
+ kafka_messages,
34
+ topic
35
+ )
36
+ consumer.call
37
+ else
38
+ kafka_messages.each do |kafka_message|
39
+ consumer.params_batch = Params::Builders::ParamsBatch.from_kafka_messages(
40
+ [kafka_message],
41
+ topic
42
+ )
43
+ consumer.call
44
+ end
45
+ end
46
+ end
47
+ end
48
+ end
49
+ end
50
+ end
51
+ end
@@ -5,7 +5,7 @@ module Karafka
5
5
  # Builder used to construct Kafka client
6
6
  module Builder
7
7
  class << self
8
- # Builds a Kafka::Cient instance that we use to work with Kafka cluster
8
+ # Builds a Kafka::Client instance that we use to work with Kafka cluster
9
9
  # @return [::Kafka::Client] returns a Kafka client
10
10
  def call
11
11
  Kafka.new(*ApiAdapter.client)
@@ -7,7 +7,13 @@ module Karafka
7
7
  class Client
8
8
  extend Forwardable
9
9
 
10
- def_delegator :kafka_consumer, :seek
10
+ %i[
11
+ seek
12
+ trigger_heartbeat
13
+ trigger_heartbeat!
14
+ ].each do |delegated_method|
15
+ def_delegator :kafka_consumer, delegated_method
16
+ end
11
17
 
12
18
  # Creates a queue consumer client that will pull the data from Kafka
13
19
  # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group for which
@@ -20,30 +26,32 @@ module Karafka
20
26
  end
21
27
 
22
28
  # Opens connection, gets messages and calls a block for each of the incoming messages
23
- # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
29
+ # @yieldparam [Array<Kafka::FetchedMessage>, Symbol] kafka response with an info about
30
+ # the type of the fetcher that is being used
24
31
  # @note This will yield with raw messages - no preprocessing or reformatting.
25
32
  def fetch_loop
26
33
  settings = ApiAdapter.consumption(consumer_group)
27
34
 
28
35
  if consumer_group.batch_fetching
29
- kafka_consumer.each_batch(*settings) { |batch| yield(batch.messages) }
36
+ kafka_consumer.each_batch(*settings) { |batch| yield(batch, :batch) }
30
37
  else
31
- # always yield an array of messages, so we have consistent API (always a batch)
32
- kafka_consumer.each_message(*settings) { |message| yield([message]) }
38
+ kafka_consumer.each_message(*settings) { |message| yield(message, :message) }
33
39
  end
34
- rescue Kafka::ProcessingError => error
40
+ # @note We catch only the processing errors as any other are considered critical (exceptions)
41
+ # and should require a client restart with a backoff
42
+ rescue Kafka::ProcessingError => e
35
43
  # If there was an error during consumption, we have to log it, pause current partition
36
44
  # and process other things
37
45
  Karafka.monitor.instrument(
38
46
  'connection.client.fetch_loop.error',
39
47
  caller: self,
40
- error: error.cause
48
+ error: e.cause
41
49
  )
42
- pause(error.topic, error.partition)
50
+ pause(e.topic, e.partition)
43
51
  retry
44
52
  end
45
53
 
46
- # Gracefuly stops topic consumption
54
+ # Gracefully stops topic consumption
47
55
  # @note Stopping running consumers without a really important reason is not recommended
48
56
  # as until all the consumers are stopped, the server will keep running serving only
49
57
  # part of the messages
@@ -59,25 +67,27 @@ module Karafka
59
67
  kafka_consumer.pause(*ApiAdapter.pause(topic, partition, consumer_group))
60
68
  end
61
69
 
62
- # Marks a given message as consumed and commit the offsets
63
- # @note In opposite to ruby-kafka, we commit the offset for each manual marking to be sure
64
- # that offset commit happen asap in case of a crash
70
+ # Marks given message as consumed
65
71
  # @param [Karafka::Params::Params] params message that we want to mark as processed
72
+ # @note This method won't trigger automatic offsets commits, rather relying on the ruby-kafka
73
+ # offsets time-interval based committing
66
74
  def mark_as_consumed(params)
67
75
  kafka_consumer.mark_message_as_processed(
68
76
  *ApiAdapter.mark_message_as_processed(params)
69
77
  )
78
+ end
79
+
80
+ # Marks a given message as consumed and commit the offsets in a blocking way
81
+ # @param [Karafka::Params::Params] params message that we want to mark as processed
82
+ # @note This method commits the offset for each manual marking to be sure
83
+ # that offset commit happen asap in case of a crash
84
+ def mark_as_consumed!(params)
85
+ mark_as_consumed(params)
70
86
  # Trigger an immediate, blocking offset commit in order to minimize the risk of crashing
71
87
  # before the automatic triggers have kicked in.
72
88
  kafka_consumer.commit_offsets
73
89
  end
74
90
 
75
- # Triggers a non-optional blocking heartbeat that notifies Kafka about the fact, that this
76
- # consumer / client is still up and running
77
- def trigger_heartbeat
78
- kafka_consumer.trigger_heartbeat!
79
- end
80
-
81
91
  private
82
92
 
83
93
  attr_reader :consumer_group
@@ -95,10 +105,10 @@ module Karafka
95
105
  end
96
106
  end
97
107
  rescue Kafka::ConnectionError
98
- # If we would not wait it would totally spam log file with failed
108
+ # If we would not wait it will spam log file with failed
99
109
  # attempts if Kafka is down
100
110
  sleep(consumer_group.reconnect_timeout)
101
- # We don't log and just reraise - this will be logged
111
+ # We don't log and just re-raise - this will be logged
102
112
  # down the road
103
113
  raise
104
114
  end
@@ -16,9 +16,10 @@ module Karafka
16
16
 
17
17
  # Runs prefetch callbacks and executes the main listener fetch loop
18
18
  def call
19
- Karafka::Callbacks.before_fetch_loop(
20
- @consumer_group,
21
- client
19
+ Karafka.monitor.instrument(
20
+ 'connection.listener.before_fetch_loop',
21
+ consumer_group: @consumer_group,
22
+ client: client
22
23
  )
23
24
  fetch_loop
24
25
  end
@@ -26,27 +27,37 @@ module Karafka
26
27
  private
27
28
 
28
29
  # Opens connection, gets messages and calls a block for each of the incoming messages
29
- # @yieldparam [String] consumer group id
30
- # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
31
- # @note This will yield with a raw message - no preprocessing or reformatting
32
30
  # @note We catch all the errors here, so they don't affect other listeners (or this one)
33
31
  # so we will be able to listen and consume other incoming messages.
34
32
  # Since it is run inside Karafka::Connection::ActorCluster - catching all the exceptions
35
- # won't crash the whole cluster. Here we mostly focus on catchin the exceptions related to
33
+ # won't crash the whole cluster. Here we mostly focus on catching the exceptions related to
36
34
  # Kafka connections / Internet connection issues / Etc. Business logic problems should not
37
35
  # propagate this far
38
36
  def fetch_loop
39
- client.fetch_loop do |raw_messages|
40
- # @note What happens here is a delegation of processing to a proper processor based
41
- # on the incoming messages characteristics
42
- Karafka::Connection::Delegator.call(@consumer_group.id, raw_messages)
37
+ # @note What happens here is a delegation of processing to a proper processor based
38
+ # on the incoming messages characteristics
39
+ client.fetch_loop do |raw_data, type|
40
+ Karafka.monitor.instrument('connection.listener.fetch_loop')
41
+
42
+ case type
43
+ when :message
44
+ MessageDelegator.call(@consumer_group.id, raw_data)
45
+ when :batch
46
+ BatchDelegator.call(@consumer_group.id, raw_data)
47
+ end
43
48
  end
44
49
  # This is on purpose - see the notes for this method
45
- # rubocop:disable RescueException
50
+ # rubocop:disable Lint/RescueException
46
51
  rescue Exception => e
47
52
  Karafka.monitor.instrument('connection.listener.fetch_loop.error', caller: self, error: e)
48
- # rubocop:enable RescueException
53
+ # rubocop:enable Lint/RescueException
54
+ # We can stop client without a problem, as it will reinitialize itself when running the
55
+ # `fetch_loop` again
49
56
  @client.stop
57
+ # We need to clear the consumers cache for current connection when fatal error happens and
58
+ # we reset the connection. Otherwise for consumers with manual offset management, the
59
+ # persistence might have stored some data that would be reprocessed
60
+ Karafka::Persistence::Consumers.clear
50
61
  sleep(@consumer_group.reconnect_timeout) && retry
51
62
  end
52
63