karafka 1.2.0 → 1.3.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (108) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data.tar.gz.sig +0 -0
  4. data/.coditsu/ci.yml +3 -0
  5. data/.console_irbrc +1 -3
  6. data/.github/FUNDING.yml +3 -0
  7. data/.github/ISSUE_TEMPLATE/bug_report.md +50 -0
  8. data/.github/ISSUE_TEMPLATE/feature_request.md +20 -0
  9. data/.gitignore +1 -0
  10. data/.ruby-version +1 -1
  11. data/.travis.yml +35 -17
  12. data/CHANGELOG.md +113 -0
  13. data/CONTRIBUTING.md +1 -1
  14. data/Gemfile +3 -2
  15. data/Gemfile.lock +85 -71
  16. data/README.md +20 -8
  17. data/bin/karafka +1 -1
  18. data/certs/mensfeld.pem +25 -0
  19. data/config/errors.yml +38 -5
  20. data/karafka.gemspec +18 -11
  21. data/lib/karafka.rb +8 -15
  22. data/lib/karafka/app.rb +14 -6
  23. data/lib/karafka/attributes_map.rb +15 -14
  24. data/lib/karafka/base_consumer.rb +19 -30
  25. data/lib/karafka/base_responder.rb +51 -29
  26. data/lib/karafka/cli.rb +1 -1
  27. data/lib/karafka/cli/console.rb +11 -9
  28. data/lib/karafka/cli/flow.rb +0 -1
  29. data/lib/karafka/cli/info.rb +3 -1
  30. data/lib/karafka/cli/install.rb +30 -6
  31. data/lib/karafka/cli/server.rb +11 -6
  32. data/lib/karafka/code_reloader.rb +67 -0
  33. data/lib/karafka/connection/{config_adapter.rb → api_adapter.rb} +54 -19
  34. data/lib/karafka/connection/batch_delegator.rb +51 -0
  35. data/lib/karafka/connection/builder.rb +16 -0
  36. data/lib/karafka/connection/client.rb +40 -40
  37. data/lib/karafka/connection/listener.rb +26 -15
  38. data/lib/karafka/connection/message_delegator.rb +36 -0
  39. data/lib/karafka/consumers/callbacks.rb +32 -15
  40. data/lib/karafka/consumers/includer.rb +30 -18
  41. data/lib/karafka/consumers/metadata.rb +10 -0
  42. data/lib/karafka/consumers/responders.rb +2 -2
  43. data/lib/karafka/contracts.rb +10 -0
  44. data/lib/karafka/contracts/config.rb +21 -0
  45. data/lib/karafka/contracts/consumer_group.rb +206 -0
  46. data/lib/karafka/contracts/consumer_group_topic.rb +19 -0
  47. data/lib/karafka/contracts/responder_usage.rb +54 -0
  48. data/lib/karafka/contracts/server_cli_options.rb +29 -0
  49. data/lib/karafka/errors.rb +18 -17
  50. data/lib/karafka/fetcher.rb +28 -30
  51. data/lib/karafka/helpers/class_matcher.rb +11 -1
  52. data/lib/karafka/helpers/config_retriever.rb +1 -1
  53. data/lib/karafka/helpers/inflector.rb +26 -0
  54. data/lib/karafka/helpers/multi_delegator.rb +0 -1
  55. data/lib/karafka/instrumentation/logger.rb +9 -6
  56. data/lib/karafka/instrumentation/monitor.rb +15 -9
  57. data/lib/karafka/instrumentation/proctitle_listener.rb +36 -0
  58. data/lib/karafka/instrumentation/stdout_listener.rb +138 -0
  59. data/lib/karafka/params/builders/metadata.rb +33 -0
  60. data/lib/karafka/params/builders/params.rb +36 -0
  61. data/lib/karafka/params/builders/params_batch.rb +25 -0
  62. data/lib/karafka/params/metadata.rb +35 -0
  63. data/lib/karafka/params/params.rb +68 -0
  64. data/lib/karafka/params/params_batch.rb +35 -20
  65. data/lib/karafka/patches/ruby_kafka.rb +21 -8
  66. data/lib/karafka/persistence/client.rb +15 -11
  67. data/lib/karafka/persistence/{consumer.rb → consumers.rb} +20 -13
  68. data/lib/karafka/persistence/topics.rb +48 -0
  69. data/lib/karafka/process.rb +0 -4
  70. data/lib/karafka/responders/builder.rb +1 -1
  71. data/lib/karafka/responders/topic.rb +6 -8
  72. data/lib/karafka/routing/builder.rb +36 -8
  73. data/lib/karafka/routing/consumer_group.rb +1 -1
  74. data/lib/karafka/routing/consumer_mapper.rb +9 -9
  75. data/lib/karafka/routing/proxy.rb +10 -1
  76. data/lib/karafka/routing/topic.rb +5 -3
  77. data/lib/karafka/routing/topic_mapper.rb +16 -18
  78. data/lib/karafka/serialization/json/deserializer.rb +27 -0
  79. data/lib/karafka/serialization/json/serializer.rb +31 -0
  80. data/lib/karafka/server.rb +30 -41
  81. data/lib/karafka/setup/config.rb +72 -40
  82. data/lib/karafka/setup/configurators/water_drop.rb +8 -4
  83. data/lib/karafka/setup/dsl.rb +0 -1
  84. data/lib/karafka/status.rb +7 -3
  85. data/lib/karafka/templates/{application_consumer.rb.example → application_consumer.rb.erb} +2 -1
  86. data/lib/karafka/templates/{application_responder.rb.example → application_responder.rb.erb} +0 -0
  87. data/lib/karafka/templates/karafka.rb.erb +92 -0
  88. data/lib/karafka/version.rb +1 -1
  89. metadata +95 -60
  90. metadata.gz.sig +0 -0
  91. data/lib/karafka/callbacks.rb +0 -30
  92. data/lib/karafka/callbacks/config.rb +0 -22
  93. data/lib/karafka/callbacks/dsl.rb +0 -16
  94. data/lib/karafka/connection/delegator.rb +0 -46
  95. data/lib/karafka/instrumentation/listener.rb +0 -112
  96. data/lib/karafka/loader.rb +0 -28
  97. data/lib/karafka/params/dsl.rb +0 -156
  98. data/lib/karafka/parsers/json.rb +0 -38
  99. data/lib/karafka/patches/dry_configurable.rb +0 -35
  100. data/lib/karafka/persistence/topic.rb +0 -29
  101. data/lib/karafka/schemas/config.rb +0 -24
  102. data/lib/karafka/schemas/consumer_group.rb +0 -77
  103. data/lib/karafka/schemas/consumer_group_topic.rb +0 -18
  104. data/lib/karafka/schemas/responder_usage.rb +0 -39
  105. data/lib/karafka/schemas/server_cli_options.rb +0 -43
  106. data/lib/karafka/setup/configurators/base.rb +0 -29
  107. data/lib/karafka/setup/configurators/params.rb +0 -25
  108. data/lib/karafka/templates/karafka.rb.example +0 -54
@@ -37,7 +37,7 @@ end
37
37
  # This is kinda trick - since we don't have a autoload and other magic stuff
38
38
  # like Rails does, so instead this method allows us to replace currently running
39
39
  # console with a new one via Kernel.exec. It will start console with new code loaded
40
- # Yes we know that it is not turbofast, however it is turbo convinient and small
40
+ # Yes, we know that it is not turbo fast, however it is turbo convenient and small
41
41
  #
42
42
  # Also - the KARAFKA_CONSOLE is used to detect that we're executing the irb session
43
43
  # so this method is only available when the Karafka console is running
@@ -8,15 +8,17 @@ module Karafka
8
8
  desc 'Start the Karafka console (short-cut alias: "c")'
9
9
  option aliases: 'c'
10
10
 
11
- # @return [String] Console executing command
12
- # @example
13
- # Karafka::Cli::Console.command #=> 'KARAFKA_CONSOLE=true bundle exec irb...'
14
- def self.command
15
- envs = [
16
- "IRBRC='#{Karafka.gem_root}/.console_irbrc'",
17
- 'KARAFKA_CONSOLE=true'
18
- ]
19
- "#{envs.join(' ')} bundle exec irb"
11
+ class << self
12
+ # @return [String] Console executing command
13
+ # @example
14
+ # Karafka::Cli::Console.command #=> 'KARAFKA_CONSOLE=true bundle exec irb...'
15
+ def command
16
+ envs = [
17
+ "IRBRC='#{Karafka.gem_root}/.console_irbrc'",
18
+ 'KARAFKA_CONSOLE=true'
19
+ ]
20
+ "#{envs.join(' ')} bundle exec irb -r #{Karafka.boot_file}"
21
+ end
20
22
  end
21
23
 
22
24
  # Start the Karafka console
@@ -18,7 +18,6 @@ module Karafka
18
18
  topic.responder.topics.each_value do |responder_topic|
19
19
  features = []
20
20
  features << (responder_topic.required? ? 'always' : 'conditionally')
21
- features << (responder_topic.multiple_usage? ? 'one or more' : 'exactly once')
22
21
 
23
22
  print responder_topic.name, "(#{features.join(', ')})"
24
23
  end
@@ -12,7 +12,9 @@ module Karafka
12
12
  config = Karafka::App.config
13
13
 
14
14
  info = [
15
- "Karafka framework version: #{Karafka::VERSION}",
15
+ "Karafka version: #{Karafka::VERSION}",
16
+ "Ruby version: #{RUBY_VERSION}",
17
+ "Ruby-kafka version: #{::Kafka::VERSION}",
16
18
  "Application client id: #{config.client_id}",
17
19
  "Backend: #{config.backend}",
18
20
  "Batch fetching: #{config.batch_fetching}",
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'erb'
4
+
3
5
  module Karafka
4
6
  # Karafka framework Cli
5
7
  class Cli < Thor
@@ -11,18 +13,30 @@ module Karafka
11
13
  INSTALL_DIRS = %w[
12
14
  app/consumers
13
15
  app/responders
16
+ app/workers
14
17
  config
18
+ lib
15
19
  log
16
20
  tmp/pids
17
21
  ].freeze
18
22
 
19
23
  # Where should we map proper files from templates
20
24
  INSTALL_FILES_MAP = {
21
- 'karafka.rb.example' => Karafka.boot_file.basename,
22
- 'application_consumer.rb.example' => 'app/consumers/application_consumer.rb',
23
- 'application_responder.rb.example' => 'app/responders/application_responder.rb'
25
+ 'karafka.rb.erb' => Karafka.boot_file.basename,
26
+ 'application_consumer.rb.erb' => 'app/consumers/application_consumer.rb',
27
+ 'application_responder.rb.erb' => 'app/responders/application_responder.rb'
24
28
  }.freeze
25
29
 
30
+ # @param args [Array] all the things that Thor CLI accepts
31
+ def initialize(*args)
32
+ super
33
+ @rails = Bundler::LockfileParser.new(
34
+ Bundler.read_file(
35
+ Bundler.default_lockfile
36
+ )
37
+ ).dependencies.key?('rails')
38
+ end
39
+
26
40
  # Install all required things for Karafka application in current directory
27
41
  def call
28
42
  INSTALL_DIRS.each do |dir|
@@ -31,12 +45,22 @@ module Karafka
31
45
 
32
46
  INSTALL_FILES_MAP.each do |source, target|
33
47
  target = Karafka.root.join(target)
34
- next if File.exist?(target)
35
48
 
36
- source = Karafka.core_root.join("templates/#{source}")
37
- FileUtils.cp_r(source, target)
49
+ template = File.read(Karafka.core_root.join("templates/#{source}"))
50
+ # @todo Replace with the keyword argument version once we don't have to support
51
+ # Ruby < 2.6
52
+ render = ::ERB.new(template, nil, '-').result(binding)
53
+
54
+ File.open(target, 'w') { |file| file.write(render) }
38
55
  end
39
56
  end
57
+
58
+ # @return [Boolean] true if we have Rails loaded
59
+ # This allows us to generate customized karafka.rb template with some tweaks specific for
60
+ # Rails
61
+ def rails?
62
+ @rails
63
+ end
40
64
  end
41
65
  end
42
66
  end
@@ -5,6 +5,11 @@ module Karafka
5
5
  class Cli < Thor
6
6
  # Server Karafka Cli action
7
7
  class Server < Base
8
+ # Server config settings contract
9
+ CONTRACT = Contracts::ServerCliOptions.new.freeze
10
+
11
+ private_constant :CONTRACT
12
+
8
13
  desc 'Start the Karafka server (short-cut alias: "s")'
9
14
  option aliases: 's'
10
15
  option :daemon, default: false, type: :boolean, aliases: :d
@@ -13,11 +18,10 @@ module Karafka
13
18
 
14
19
  # Start the Karafka server
15
20
  def call
16
- validate!
17
-
18
- puts 'Starting Karafka server'
19
21
  cli.info
20
22
 
23
+ validate!
24
+
21
25
  if cli.options[:daemon]
22
26
  FileUtils.mkdir_p File.dirname(cli.options[:pid])
23
27
  daemonize
@@ -31,7 +35,7 @@ module Karafka
31
35
  # We want to delay the moment in which the pidfile is removed as much as we can,
32
36
  # so instead of removing it after the server stops running, we rely on the gc moment
33
37
  # when this object gets removed (it is a bit later), so it is closer to the actual
34
- # system process end. We do that, so monitoring and deployment tools that rely on pids
38
+ # system process end. We do that, so monitoring and deployment tools that rely on a pid
35
39
  # won't alarm or start new system process up until the current one is finished
36
40
  ObjectSpace.define_finalizer(self, proc { send(:clean) })
37
41
 
@@ -43,9 +47,10 @@ module Karafka
43
47
  # Checks the server cli configuration
44
48
  # options validations in terms of app setup (topics, pid existence, etc)
45
49
  def validate!
46
- result = Schemas::ServerCliOptions.call(cli.options)
50
+ result = CONTRACT.call(cli.options)
47
51
  return if result.success?
48
- raise Errors::InvalidConfiguration, result.errors
52
+
53
+ raise Errors::InvalidConfigurationError, result.errors.to_h
49
54
  end
50
55
 
51
56
  # Detaches current process into background and writes its pidfile
@@ -0,0 +1,67 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ # Special type of a listener, that is not an instrumentation one, but one that triggers
5
+ # code reload in the development mode after each fetched batch (or message)
6
+ #
7
+ # Please refer to the development code reload sections for details on the benefits and downsides
8
+ # of the in-process code reloading
9
+ class CodeReloader
10
+ # This mutex is needed as we might have an application that has multiple consumer groups
11
+ # running in separate threads and we should not trigger reload before fully reloading the app
12
+ # in previous thread
13
+ MUTEX = Mutex.new
14
+
15
+ private_constant :MUTEX
16
+
17
+ # @param reloaders [Array<Object>] any code loaders that we use in this app. Whether it is
18
+ # the Rails loader, Zeitwerk or anything else that allows reloading triggering
19
+ # @param block [Proc] yields given block just before reloading. This can be used to hook custom
20
+ # reloading stuff, that ain't reloaders (for example for resetting dry-events registry)
21
+ def initialize(*reloaders, &block)
22
+ @reloaders = reloaders
23
+ @block = block
24
+ end
25
+
26
+ # Binds to the instrumentation events and triggers reload
27
+ # @param _event [Dry::Event] empty dry event
28
+ # @note Since we de-register all the user defined objects and redraw routes, it means that
29
+ # we won't be able to do a multi-batch buffering in the development mode as each of the
30
+ # batches will be buffered on a newly created "per fetch" instance.
31
+ def on_connection_listener_fetch_loop(_event)
32
+ reload
33
+ end
34
+
35
+ private
36
+
37
+ # Triggers reload of both standard and Rails reloaders as well as expires all internals of
38
+ # Karafka, so it can be rediscovered and rebuilt
39
+ def reload
40
+ MUTEX.synchronize do
41
+ if @reloaders[0].respond_to?(:execute)
42
+ reload_with_rails
43
+ else
44
+ reload_without_rails
45
+ end
46
+ end
47
+ end
48
+
49
+ # Rails reloading procedure
50
+ def reload_with_rails
51
+ updatable = @reloaders.select(&:updated?)
52
+
53
+ return if updatable.empty?
54
+
55
+ updatable.each(&:execute)
56
+ @block&.call
57
+ Karafka::App.reload
58
+ end
59
+
60
+ # Zeitwerk and other reloaders
61
+ def reload_without_rails
62
+ @reloaders.each(&:reload)
63
+ @block&.call
64
+ Karafka::App.reload
65
+ end
66
+ end
67
+ end
@@ -3,22 +3,22 @@
3
3
  module Karafka
4
4
  # Namespace for all the things related to Kafka connection
5
5
  module Connection
6
- # Mapper used to convert our internal settings into ruby-kafka settings
6
+ # Mapper used to convert our internal settings into ruby-kafka settings based on their
7
+ # API requirements.
7
8
  # Since ruby-kafka has more and more options and there are few "levels" on which
8
9
  # we have to apply them (despite the fact, that in Karafka you configure all of it
9
10
  # in one place), we have to remap it into what ruby-kafka driver requires
10
11
  # @note The good thing about Kafka.new method is that it ignores all options that
11
12
  # do nothing. So we don't have to worry about injecting our internal settings
12
13
  # into the client and breaking stuff
13
- module ConfigAdapter
14
+ module ApiAdapter
14
15
  class << self
15
16
  # Builds all the configuration settings for Kafka.new method
16
- # @param _consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
17
17
  # @return [Array<Hash>] Array with all the client arguments including hash with all
18
18
  # the settings required by Kafka.new method
19
19
  # @note We return array, so we can inject any arguments we want, in case of changes in the
20
20
  # raw driver
21
- def client(_consumer_group)
21
+ def client
22
22
  # This one is a default that takes all the settings except special
23
23
  # cases defined in the map
24
24
  settings = {
@@ -29,9 +29,9 @@ module Karafka
29
29
  kafka_configs.each do |setting_name, setting_value|
30
30
  # All options for config adapter should be ignored as we're just interested
31
31
  # in what is left, as we want to pass all the options that are "typical"
32
- # and not listed in the config_adapter special cases mapping. All the values
33
- # from the config_adapter mapping go somewhere else, not to the client directly
34
- next if AttributesMap.config_adapter.values.flatten.include?(setting_name)
32
+ # and not listed in the api_adapter special cases mapping. All the values
33
+ # from the api_adapter mapping go somewhere else, not to the client directly
34
+ next if AttributesMap.api_adapter.values.flatten.include?(setting_name)
35
35
 
36
36
  settings[setting_name] = setting_value
37
37
  end
@@ -58,26 +58,59 @@ module Karafka
58
58
  # @return [Array<Hash>] Array with all the arguments required by consuming method
59
59
  # including hash with all the settings required by
60
60
  # Kafka::Consumer#consume_each_message and Kafka::Consumer#consume_each_batch method
61
- def consuming(consumer_group)
62
- settings = {
63
- automatically_mark_as_processed: consumer_group.automatically_mark_as_consumed
64
- }
65
- [sanitize(fetch_for(:consuming, consumer_group, settings))]
61
+ def consumption(consumer_group)
62
+ [
63
+ sanitize(
64
+ fetch_for(
65
+ :consumption,
66
+ consumer_group,
67
+ automatically_mark_as_processed: consumer_group.automatically_mark_as_consumed
68
+ )
69
+ )
70
+ ]
66
71
  end
67
72
 
68
73
  # Builds all the configuration settings for kafka consumer#subscribe method
69
74
  # @param topic [Karafka::Routing::Topic] topic that holds details for a given subscription
70
75
  # @return [Hash] hash with all the settings required by kafka consumer#subscribe method
71
- def subscription(topic)
72
- settings = fetch_for(:subscription, topic)
76
+ def subscribe(topic)
77
+ settings = fetch_for(:subscribe, topic)
73
78
  [Karafka::App.config.topic_mapper.outgoing(topic.name), sanitize(settings)]
74
79
  end
75
80
 
76
81
  # Builds all the configuration settings required by kafka consumer#pause method
82
+ # @param topic [String] topic that we want to pause
83
+ # @param partition [Integer] number partition that we want to pause
77
84
  # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
78
- # @return [Hash] hash with all the settings required to pause kafka consumer
79
- def pausing(consumer_group)
80
- { timeout: consumer_group.pause_timeout }
85
+ # @return [Array] array with all the details required to pause kafka consumer
86
+ def pause(topic, partition, consumer_group)
87
+ [
88
+ Karafka::App.config.topic_mapper.outgoing(topic),
89
+ partition,
90
+ {
91
+ timeout: consumer_group.pause_timeout,
92
+ max_timeout: consumer_group.pause_max_timeout,
93
+ exponential_backoff: consumer_group.pause_exponential_backoff
94
+ }
95
+ ]
96
+ end
97
+
98
+ # Remaps topic details taking the topic mapper feature into consideration.
99
+ # @param params [Karafka::Params::Params] params instance
100
+ # @return [Array] array with all the details needed by ruby-kafka to mark message
101
+ # as processed
102
+ # @note When default empty topic mapper is used, no need for any conversion as the
103
+ # internal and external format are exactly the same
104
+ def mark_message_as_processed(params)
105
+ # Majority of users don't use custom topic mappers. No need to change anything when it
106
+ # is a default mapper that does not change anything. Only some cloud providers require
107
+ # topics to be remapped
108
+ return [params] if Karafka::App.config.topic_mapper.is_a?(Karafka::Routing::TopicMapper)
109
+
110
+ # @note We don't use tap as it is around 13% slower than non-dup version
111
+ dupped = params.dup
112
+ dupped['topic'] = Karafka::App.config.topic_mapper.outgoing(params.topic)
113
+ [dupped]
81
114
  end
82
115
 
83
116
  private
@@ -90,10 +123,12 @@ module Karafka
90
123
  def fetch_for(namespace_key, route_layer, preexisting_settings = {})
91
124
  kafka_configs.each_key do |setting_name|
92
125
  # Ignore settings that are not related to our namespace
93
- next unless AttributesMap.config_adapter[namespace_key].include?(setting_name)
126
+ next unless AttributesMap.api_adapter[namespace_key].include?(setting_name)
127
+
94
128
  # Ignore settings that are already initialized
95
129
  # In case they are in preexisting settings fetched differently
96
- next if preexisting_settings.keys.include?(setting_name)
130
+ next if preexisting_settings.key?(setting_name)
131
+
97
132
  # Fetch all the settings from a given layer object. Objects can handle the fallback
98
133
  # to the kafka settings, so
99
134
  preexisting_settings[setting_name] = route_layer.send(setting_name)
@@ -0,0 +1,51 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Connection
5
+ # Class that delegates processing of batch received messages for which we listen to
6
+ # a proper processor
7
+ module BatchDelegator
8
+ class << self
9
+ # Delegates messages (does something with them)
10
+ # It will either schedule or run a proper processor action for messages
11
+ # @param group_id [String] group_id of a group from which a given message came
12
+ # @param kafka_batch [<Kafka::FetchedBatch>] raw messages fetched batch
13
+ # @note This should be looped to obtain a constant delegating of new messages
14
+ def call(group_id, kafka_batch)
15
+ topic = Persistence::Topics.fetch(group_id, kafka_batch.topic)
16
+ consumer = Persistence::Consumers.fetch(topic, kafka_batch.partition)
17
+
18
+ Karafka.monitor.instrument(
19
+ 'connection.batch_delegator.call',
20
+ caller: self,
21
+ consumer: consumer,
22
+ kafka_batch: kafka_batch
23
+ ) do
24
+ # Due to how ruby-kafka is built, we have the metadata that is stored on the batch
25
+ # level only available for batch consuming
26
+ consumer.metadata = Params::Builders::Metadata.from_kafka_batch(kafka_batch, topic)
27
+ kafka_messages = kafka_batch.messages
28
+
29
+ # Depending on a case (persisted or not) we might use new consumer instance per
30
+ # each batch, or use the same one for all of them (for implementing buffering, etc.)
31
+ if topic.batch_consuming
32
+ consumer.params_batch = Params::Builders::ParamsBatch.from_kafka_messages(
33
+ kafka_messages,
34
+ topic
35
+ )
36
+ consumer.call
37
+ else
38
+ kafka_messages.each do |kafka_message|
39
+ consumer.params_batch = Params::Builders::ParamsBatch.from_kafka_messages(
40
+ [kafka_message],
41
+ topic
42
+ )
43
+ consumer.call
44
+ end
45
+ end
46
+ end
47
+ end
48
+ end
49
+ end
50
+ end
51
+ end
@@ -0,0 +1,16 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Connection
5
+ # Builder used to construct Kafka client
6
+ module Builder
7
+ class << self
8
+ # Builds a Kafka::Client instance that we use to work with Kafka cluster
9
+ # @return [::Kafka::Client] returns a Kafka client
10
+ def call
11
+ Kafka.new(*ApiAdapter.client)
12
+ end
13
+ end
14
+ end
15
+ end
16
+ end
@@ -7,7 +7,13 @@ module Karafka
7
7
  class Client
8
8
  extend Forwardable
9
9
 
10
- def_delegator :kafka_consumer, :seek
10
+ %i[
11
+ seek
12
+ trigger_heartbeat
13
+ trigger_heartbeat!
14
+ ].each do |delegated_method|
15
+ def_delegator :kafka_consumer, delegated_method
16
+ end
11
17
 
12
18
  # Creates a queue consumer client that will pull the data from Kafka
13
19
  # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group for which
@@ -20,40 +26,32 @@ module Karafka
20
26
  end
21
27
 
22
28
  # Opens connection, gets messages and calls a block for each of the incoming messages
23
- # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
29
+ # @yieldparam [Array<Kafka::FetchedMessage>, Symbol] kafka response with an info about
30
+ # the type of the fetcher that is being used
24
31
  # @note This will yield with raw messages - no preprocessing or reformatting.
25
32
  def fetch_loop
26
- settings = ConfigAdapter.consuming(consumer_group)
33
+ settings = ApiAdapter.consumption(consumer_group)
27
34
 
28
35
  if consumer_group.batch_fetching
29
- kafka_consumer.each_batch(*settings) { |batch| yield(batch.messages) }
36
+ kafka_consumer.each_batch(*settings) { |batch| yield(batch, :batch) }
30
37
  else
31
- # always yield an array of messages, so we have consistent API (always a batch)
32
- kafka_consumer.each_message(*settings) { |message| yield([message]) }
38
+ kafka_consumer.each_message(*settings) { |message| yield(message, :message) }
33
39
  end
34
- rescue Kafka::ProcessingError => error
40
+ # @note We catch only the processing errors as any other are considered critical (exceptions)
41
+ # and should require a client restart with a backoff
42
+ rescue Kafka::ProcessingError => e
35
43
  # If there was an error during consumption, we have to log it, pause current partition
36
44
  # and process other things
37
45
  Karafka.monitor.instrument(
38
46
  'connection.client.fetch_loop.error',
39
47
  caller: self,
40
- error: error.cause
41
- )
42
- pause(error.topic, error.partition)
43
- retry
44
- # This is on purpose - see the notes for this method
45
- # rubocop:disable RescueException
46
- rescue Exception => error
47
- # rubocop:enable RescueException
48
- Karafka.monitor.instrument(
49
- 'connection.client.fetch_loop.error',
50
- caller: self,
51
- error: error
48
+ error: e.cause
52
49
  )
50
+ pause(e.topic, e.partition)
53
51
  retry
54
52
  end
55
53
 
56
- # Gracefuly stops topic consumption
54
+ # Gracefully stops topic consumption
57
55
  # @note Stopping running consumers without a really important reason is not recommended
58
56
  # as until all the consumers are stopped, the server will keep running serving only
59
57
  # part of the messages
@@ -66,18 +64,25 @@ module Karafka
66
64
  # @param topic [String] topic that we want to pause
67
65
  # @param partition [Integer] number partition that we want to pause
68
66
  def pause(topic, partition)
69
- settings = ConfigAdapter.pausing(consumer_group)
70
- timeout = settings[:timeout]
71
- raise(Errors::InvalidPauseTimeout, timeout) unless timeout.positive?
72
- kafka_consumer.pause(topic, partition, settings)
67
+ kafka_consumer.pause(*ApiAdapter.pause(topic, partition, consumer_group))
73
68
  end
74
69
 
75
- # Marks a given message as consumed and commit the offsets
76
- # @note In opposite to ruby-kafka, we commit the offset for each manual marking to be sure
77
- # that offset commit happen asap in case of a crash
70
+ # Marks given message as consumed
78
71
  # @param [Karafka::Params::Params] params message that we want to mark as processed
72
+ # @note This method won't trigger automatic offsets commits, rather relying on the ruby-kafka
73
+ # offsets time-interval based committing
79
74
  def mark_as_consumed(params)
80
- kafka_consumer.mark_message_as_processed(params)
75
+ kafka_consumer.mark_message_as_processed(
76
+ *ApiAdapter.mark_message_as_processed(params)
77
+ )
78
+ end
79
+
80
+ # Marks a given message as consumed and commit the offsets in a blocking way
81
+ # @param [Karafka::Params::Params] params message that we want to mark as processed
82
+ # @note This method commits the offset for each manual marking to be sure
83
+ # that offset commit happen asap in case of a crash
84
+ def mark_as_consumed!(params)
85
+ mark_as_consumed(params)
81
86
  # Trigger an immediate, blocking offset commit in order to minimize the risk of crashing
82
87
  # before the automatic triggers have kicked in.
83
88
  kafka_consumer.commit_offsets
@@ -90,28 +95,23 @@ module Karafka
90
95
  # @return [Kafka::Consumer] returns a ready to consume Kafka consumer
91
96
  # that is set up to consume from topics of a given consumer group
92
97
  def kafka_consumer
93
- @kafka_consumer ||= kafka.consumer(
94
- *ConfigAdapter.consumer(consumer_group)
98
+ # @note We don't cache the connection internally because we cache kafka_consumer that uses
99
+ # kafka client object instance
100
+ @kafka_consumer ||= Builder.call.consumer(
101
+ *ApiAdapter.consumer(consumer_group)
95
102
  ).tap do |consumer|
96
103
  consumer_group.topics.each do |topic|
97
- consumer.subscribe(*ConfigAdapter.subscription(topic))
104
+ consumer.subscribe(*ApiAdapter.subscribe(topic))
98
105
  end
99
106
  end
100
107
  rescue Kafka::ConnectionError
101
- # If we would not wait it would totally spam log file with failed
108
+ # If we would not wait it will spam log file with failed
102
109
  # attempts if Kafka is down
103
110
  sleep(consumer_group.reconnect_timeout)
104
- # We don't log and just reraise - this will be logged
111
+ # We don't log and just re-raise - this will be logged
105
112
  # down the road
106
113
  raise
107
114
  end
108
-
109
- # @return [Kafka] returns a Kafka
110
- # @note We don't cache it internally because we cache kafka_consumer that uses kafka
111
- # object instance
112
- def kafka
113
- Kafka.new(*ConfigAdapter.client(consumer_group))
114
- end
115
115
  end
116
116
  end
117
117
  end