karafka 2.0.0.beta4 → 2.0.0.beta5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/.github/workflows/ci.yml +18 -1
  4. data/CHANGELOG.md +15 -0
  5. data/Gemfile.lock +1 -1
  6. data/bin/benchmarks +2 -2
  7. data/bin/integrations +10 -3
  8. data/bin/{stress → stress_many} +0 -0
  9. data/bin/stress_one +13 -0
  10. data/docker-compose.yml +23 -18
  11. data/lib/karafka/active_job/routing/extensions.rb +1 -1
  12. data/lib/karafka/app.rb +2 -1
  13. data/lib/karafka/base_consumer.rb +26 -19
  14. data/lib/karafka/connection/client.rb +24 -4
  15. data/lib/karafka/connection/listener.rb +49 -11
  16. data/lib/karafka/connection/pauses_manager.rb +8 -0
  17. data/lib/karafka/connection/rebalance_manager.rb +20 -19
  18. data/lib/karafka/contracts/config.rb +17 -4
  19. data/lib/karafka/contracts/server_cli_options.rb +1 -1
  20. data/lib/karafka/errors.rb +3 -0
  21. data/lib/karafka/pro/active_job/consumer.rb +1 -8
  22. data/lib/karafka/pro/base_consumer.rb +10 -13
  23. data/lib/karafka/pro/loader.rb +11 -6
  24. data/lib/karafka/pro/processing/coordinator.rb +12 -0
  25. data/lib/karafka/pro/processing/jobs_builder.rb +3 -2
  26. data/lib/karafka/pro/processing/scheduler.rb +56 -0
  27. data/lib/karafka/processing/coordinator.rb +84 -0
  28. data/lib/karafka/processing/coordinators_buffer.rb +58 -0
  29. data/lib/karafka/processing/executor.rb +6 -16
  30. data/lib/karafka/processing/executors_buffer.rb +46 -15
  31. data/lib/karafka/processing/jobs/consume.rb +4 -2
  32. data/lib/karafka/processing/jobs_builder.rb +3 -2
  33. data/lib/karafka/processing/result.rb +0 -5
  34. data/lib/karafka/processing/scheduler.rb +22 -0
  35. data/lib/karafka/routing/consumer_group.rb +1 -1
  36. data/lib/karafka/routing/topic.rb +9 -0
  37. data/lib/karafka/setup/config.rb +18 -10
  38. data/lib/karafka/version.rb +1 -1
  39. data.tar.gz.sig +0 -0
  40. metadata +9 -5
  41. metadata.gz.sig +4 -1
  42. data/lib/karafka/pro/scheduler.rb +0 -54
  43. data/lib/karafka/scheduler.rb +0 -20
@@ -30,12 +30,25 @@ module Karafka
30
30
 
31
31
  # We validate internals just to be sure, that they are present and working
32
32
  required(:internal).schema do
33
- required(:routing_builder)
34
- required(:subscription_groups_builder)
35
- required(:jobs_builder)
36
33
  required(:status)
37
34
  required(:process)
38
- required(:scheduler)
35
+
36
+ required(:routing).schema do
37
+ required(:builder)
38
+ required(:subscription_groups_builder)
39
+ end
40
+
41
+ required(:processing).schema do
42
+ required(:jobs_builder)
43
+ required(:scheduler)
44
+ required(:coordinator_class)
45
+ end
46
+
47
+ required(:active_job).schema do
48
+ required(:dispatcher)
49
+ required(:job_options_contract)
50
+ required(:consumer_class)
51
+ end
39
52
  end
40
53
  end
41
54
 
@@ -12,7 +12,7 @@ module Karafka
12
12
  # If there were no consumer_groups declared in the server cli, it means that we will
13
13
  # run all of them and no need to validate them here at all
14
14
  if !value.nil? &&
15
- !(value - Karafka::App.config.internal.routing_builder.map(&:name)).empty?
15
+ !(value - Karafka::App.config.internal.routing.builder.map(&:name)).empty?
16
16
  key(:consumer_groups).failure(:consumer_groups_inclusion)
17
17
  end
18
18
  end
@@ -47,5 +47,8 @@ module Karafka
47
47
  # Used to instrument this error into the error notifications
48
48
  # We do not raise it so we won't crash deployed systems
49
49
  ExpiredLicenseTokenError = Class.new(BaseError)
50
+
51
+ # This should never happen. Please open an issue if it does.
52
+ InvalidCoordinatorState = Class.new(BaseError)
50
53
  end
51
54
  end
@@ -26,7 +26,7 @@ module Karafka
26
26
  messages.each do |message|
27
27
  # If for any reason we've lost this partition, not worth iterating over new messages
28
28
  # as they are no longer ours
29
- return if revoked?
29
+ break if revoked?
30
30
  break if Karafka::App.stopping?
31
31
 
32
32
  ::ActiveJob::Base.execute(
@@ -34,13 +34,6 @@ module Karafka
34
34
  )
35
35
 
36
36
  mark_as_consumed(message)
37
-
38
- # We check it twice as the job may be long running
39
- # If marking fails, it also means it got revoked and we can stop consuming
40
- return if revoked?
41
-
42
- # Do not process more if we are shutting down
43
- break if Karafka::App.stopping?
44
37
  end
45
38
  end
46
39
  end
@@ -39,38 +39,35 @@ module Karafka
39
39
  # Nothing to do if we lost the partition
40
40
  return if revoked?
41
41
 
42
- if @consumption.success?
43
- pause_tracker.reset
42
+ if @coordinator.success?
43
+ coordinator.pause_tracker.reset
44
44
 
45
45
  # We use the non-blocking one here. If someone needs the blocking one, can implement it
46
46
  # with manual offset management
47
47
  # Mark as consumed only if manual offset management is not on
48
48
  mark_as_consumed(messages.last) unless topic.manual_offset_management?
49
49
 
50
+ # We check it twice as marking could change this state
51
+ return if revoked?
52
+
50
53
  # If this is not a long running job there is nothing for us to do here
51
54
  return unless topic.long_running_job?
52
55
 
53
56
  # Once processing is done, we move to the new offset based on commits
54
57
  # Here, in case manual offset management is off, we have the new proper offset of a
55
58
  # first message from another batch from `@seek_offset`. If manual offset management
56
- # is on, we move to place where the user indicated it was finished.
59
+ # is on, we move to place where the user indicated it was finished. This can create an
60
+ # interesting (yet valid) corner case, where with manual offset management on and no
61
+ # marking as consumed, we end up with an infinite loop processing same messages over and
62
+ # over again
57
63
  seek(@seek_offset || messages.first.offset)
64
+
58
65
  resume
59
66
  else
60
67
  # If processing failed, we need to pause
61
68
  pause(@seek_offset || messages.first.offset)
62
69
  end
63
70
  end
64
-
65
- # Marks this consumer revoked state as true
66
- # This allows us for things like lrj to finish early as this state may change during lrj
67
- # execution
68
- def on_revoked
69
- # @note This may already be set to true if we tried to commit offsets and failed. In case
70
- # like this it will automatically be marked as revoked.
71
- @revoked = true
72
- super
73
- end
74
71
  end
75
72
  end
76
73
  end
@@ -17,9 +17,10 @@ module Karafka
17
17
  COMPONENTS = %w[
18
18
  base_consumer
19
19
  performance_tracker
20
- scheduler
20
+ processing/scheduler
21
21
  processing/jobs/consume_non_blocking
22
22
  processing/jobs_builder
23
+ processing/coordinator
23
24
  routing/extensions
24
25
  active_job/consumer
25
26
  active_job/dispatcher
@@ -35,11 +36,15 @@ module Karafka
35
36
  def setup(config)
36
37
  COMPONENTS.each { |component| require_relative(component) }
37
38
 
38
- config.internal.scheduler = Scheduler.new
39
- config.internal.jobs_builder = Processing::JobsBuilder.new
40
- config.internal.active_job.consumer = ActiveJob::Consumer
41
- config.internal.active_job.dispatcher = ActiveJob::Dispatcher.new
42
- config.internal.active_job.job_options_contract = ActiveJob::JobOptionsContract.new
39
+ icfg = config.internal
40
+
41
+ icfg.processing.coordinator_class = Processing::Coordinator
42
+ icfg.processing.scheduler = Processing::Scheduler.new
43
+ icfg.processing.jobs_builder = Processing::JobsBuilder.new
44
+
45
+ icfg.active_job.consumer_class = ActiveJob::Consumer
46
+ icfg.active_job.dispatcher = ActiveJob::Dispatcher.new
47
+ icfg.active_job.job_options_contract = ActiveJob::JobOptionsContract.new
43
48
 
44
49
  ::Karafka::Routing::Topic.include(Routing::Extensions)
45
50
 
@@ -0,0 +1,12 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Pro
5
+ module Processing
6
+ # Pro coordinator that provides extra orchestration methods useful for parallel processing
7
+ # within the same partition
8
+ class Coordinator < ::Karafka::Processing::Coordinator
9
+ end
10
+ end
11
+ end
12
+ end
@@ -16,11 +16,12 @@ module Karafka
16
16
  class JobsBuilder < ::Karafka::Processing::JobsBuilder
17
17
  # @param executor [Karafka::Processing::Executor]
18
18
  # @param messages [Karafka::Messages::Messages] messages batch to be consumed
19
+ # @param coordinator [Karafka::Processing::Coordinator]
19
20
  # @return [Karafka::Processing::Jobs::Consume] blocking job
20
21
  # @return [Karafka::Pro::Processing::Jobs::ConsumeNonBlocking] non blocking for lrj
21
- def consume(executor, messages)
22
+ def consume(executor, messages, coordinator)
22
23
  if executor.topic.long_running_job?
23
- Jobs::ConsumeNonBlocking.new(executor, messages)
24
+ Jobs::ConsumeNonBlocking.new(executor, messages, coordinator)
24
25
  else
25
26
  super
26
27
  end
@@ -0,0 +1,56 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component.
4
+ # All of the commercial components are present in the lib/karafka/pro directory of this
5
+ # repository and their usage requires commercial license agreement.
6
+ #
7
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
8
+ #
9
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
10
+ # your code to Maciej Mensfeld.
11
+
12
+ module Karafka
13
+ module Pro
14
+ module Processing
15
+ # Optimizes scheduler that takes into consideration of execution time needed to process
16
+ # messages from given topics partitions. It uses the non-preemptive LJF algorithm
17
+ #
18
+ # This scheduler is designed to optimize execution times on jobs that perform IO operations
19
+ # as when taking IO into consideration, the can achieve optimized parallel processing.
20
+ #
21
+ # This scheduler can also work with virtual partitions.
22
+ #
23
+ # Aside from consumption jobs, other jobs do not run often, thus we can leave them with
24
+ # default FIFO scheduler from the default Karafka scheduler
25
+ class Scheduler < ::Karafka::Processing::Scheduler
26
+ # Schedules jobs in the LJF order for consumption
27
+ #
28
+ # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
29
+ # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
30
+ #
31
+ def schedule_consumption(queue, jobs_array)
32
+ pt = PerformanceTracker.instance
33
+
34
+ ordered = []
35
+
36
+ jobs_array.each do |job|
37
+ messages = job.messages
38
+ message = messages.first
39
+
40
+ cost = pt.processing_time_p95(message.topic, message.partition) * messages.size
41
+
42
+ ordered << [job, cost]
43
+ end
44
+
45
+ ordered.sort_by!(&:last)
46
+ ordered.reverse!
47
+ ordered.map!(&:first)
48
+
49
+ ordered.each do |job|
50
+ queue << job
51
+ end
52
+ end
53
+ end
54
+ end
55
+ end
56
+ end
@@ -0,0 +1,84 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Processing
5
+ # Basic coordinator that allows us to provide coordination objects into consumers.
6
+ #
7
+ # This is a wrapping layer to simplify management of work to be handled around consumption.
8
+ #
9
+ # @note This coordinator needs to be thread safe. Some operations are performed only in the
10
+ # listener thread, but we go with thread-safe by default for all not to worry about potential
11
+ # future mistakes.
12
+ class Coordinator
13
+ # @return [Karafka::TimeTrackers::Pause]
14
+ attr_reader :pause_tracker
15
+
16
+ # @param pause_tracker [Karafka::TimeTrackers::Pause] pause tracker for given topic partition
17
+ def initialize(pause_tracker)
18
+ @pause_tracker = pause_tracker
19
+ @revoked = false
20
+ @consumptions = {}
21
+ @running_jobs = 0
22
+ @mutex = Mutex.new
23
+ end
24
+
25
+ # Starts the coordinator for given consumption jobs
26
+ def start
27
+ @mutex.synchronize do
28
+ @running_jobs = 0
29
+ # We need to clear the consumption results hash here, otherwise we could end up storing
30
+ # consumption results of consumer instances we no longer control
31
+ @consumptions.clear
32
+ end
33
+ end
34
+
35
+ # Increases number of jobs that we handle with this coordinator
36
+ def increment
37
+ @mutex.synchronize { @running_jobs += 1 }
38
+ end
39
+
40
+ # Decrements number of jobs we handle at the moment
41
+ def decrement
42
+ @mutex.synchronize do
43
+ @running_jobs -= 1
44
+
45
+ return @running_jobs unless @running_jobs.negative?
46
+
47
+ raise Karafka::Errors::InvalidCoordinatorState, @running_jobs
48
+ end
49
+ end
50
+
51
+ # @param consumer [Object] karafka consumer (normal or pro)
52
+ # @return [Karafka::Processing::Result] result object which we can use to indicate
53
+ # consumption processing state.
54
+ def consumption(consumer)
55
+ @mutex.synchronize do
56
+ @consumptions[consumer] ||= Processing::Result.new
57
+ end
58
+ end
59
+
60
+ # Is all the consumption done and finished successfully for this coordinator
61
+ def success?
62
+ @mutex.synchronize { @running_jobs.zero? && @consumptions.values.all?(&:success?) }
63
+ end
64
+
65
+ # Marks given coordinator for processing group as revoked
66
+ #
67
+ # This is invoked in two places:
68
+ # - from the main listener loop when we detect revoked partitions
69
+ # - from the consumer in case checkpointing fails
70
+ #
71
+ # This means, we can end up having consumer being aware that it was revoked prior to the
72
+ # listener loop dispatching the revocation job. It is ok, as effectively nothing will be
73
+ # processed until revocation jobs are done.
74
+ def revoke
75
+ @mutex.synchronize { @revoked = true }
76
+ end
77
+
78
+ # @return [Boolean] is the partition we are processing revoked or not
79
+ def revoked?
80
+ @revoked
81
+ end
82
+ end
83
+ end
84
+ end
@@ -0,0 +1,58 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Processing
5
+ # Buffer used to build and store coordinators per topic partition
6
+ #
7
+ # It provides direct pauses access for revocation
8
+ #
9
+ # @note This buffer operates only from the listener loop, thus we do not have to make it
10
+ # thread-safe.
11
+ class CoordinatorsBuffer
12
+ def initialize
13
+ @pauses_manager = Connection::PausesManager.new
14
+ @coordinator_class = ::Karafka::App.config.internal.processing.coordinator_class
15
+ @coordinators = Hash.new { |h, k| h[k] = {} }
16
+ end
17
+
18
+ # @param topic [String] topic name
19
+ # @param partition [Integer] partition number
20
+ def find_or_create(topic, partition)
21
+ @coordinators[topic][partition] ||= @coordinator_class.new(
22
+ @pauses_manager.fetch(topic, partition)
23
+ )
24
+ end
25
+
26
+ # Resumes processing of partitions for which pause time has ended.
27
+ # @param block we want to run for resumed topic partitions
28
+ # @yieldparam [String] topic name
29
+ # @yieldparam [Integer] partition number
30
+ def resume(&block)
31
+ @pauses_manager.resume(&block)
32
+ end
33
+
34
+ # @param topic [String] topic name
35
+ # @param partition [Integer] partition number
36
+ def revoke(topic, partition)
37
+ @pauses_manager.revoke(topic, partition)
38
+
39
+ # The fact that we delete here does not change the fact that the executor still holds the
40
+ # reference to this coordinator. We delete it here, as we will no longer process any
41
+ # new stuff with it and we may need a new coordinator if we regain this partition, but the
42
+ # coordinator may still be in use
43
+ coordinator = @coordinators[topic].delete(partition)
44
+
45
+ return unless coordinator
46
+
47
+ coordinator.revoke
48
+ end
49
+
50
+ # Clears coordinators and re-created the pauses manager
51
+ # This should be used only for critical errors recovery
52
+ def reset
53
+ @pauses_manager = Connection::PausesManager.new
54
+ @coordinators.clear
55
+ end
56
+ end
57
+ end
58
+ end
@@ -30,13 +30,11 @@ module Karafka
30
30
  # @param group_id [String] id of the subscription group to which the executor belongs
31
31
  # @param client [Karafka::Connection::Client] kafka client
32
32
  # @param topic [Karafka::Routing::Topic] topic for which this executor will run
33
- # @param pause_tracker [Karafka::TimeTrackers::Pause] fetch pause tracker for pausing
34
- def initialize(group_id, client, topic, pause_tracker)
33
+ def initialize(group_id, client, topic)
35
34
  @id = SecureRandom.uuid
36
35
  @group_id = group_id
37
36
  @client = client
38
37
  @topic = topic
39
- @pause_tracker = pause_tracker
40
38
  end
41
39
 
42
40
  # Builds the consumer instance, builds messages batch and sets all that is needed to run the
@@ -45,20 +43,15 @@ module Karafka
45
43
  # @param messages [Array<Karafka::Messages::Message>]
46
44
  # @param received_at [Time] the moment we've received the batch (actually the moment we've)
47
45
  # enqueued it, but good enough
48
- def before_consume(messages, received_at)
46
+ # @param coordinator [Karafka::Processing::Coordinator] coordinator for processing management
47
+ def before_consume(messages, received_at, coordinator)
49
48
  # Recreate consumer with each batch if persistence is not enabled
50
49
  # We reload the consumers with each batch instead of relying on some external signals
51
50
  # when needed for consistency. That way devs may have it on or off and not in this
52
51
  # middle state, where re-creation of a consumer instance would occur only sometimes
53
- @recreate = true unless ::Karafka::App.config.consumer_persistence
52
+ @consumer = nil unless ::Karafka::App.config.consumer_persistence
54
53
 
55
- # If @recreate was set to true (aside from non persistent), it means, that revocation or
56
- # a shutdown happened and we need to have a new instance for running another consume for
57
- # this topic partition
58
- if @recreate
59
- @consumer = nil
60
- @recreate = false
61
- end
54
+ consumer.coordinator = coordinator
62
55
 
63
56
  # First we build messages batch...
64
57
  consumer.messages = Messages::Builders::Messages.call(
@@ -95,7 +88,6 @@ module Karafka
95
88
  # consumer instance.
96
89
  def revoked
97
90
  consumer.on_revoked if @consumer
98
- @recreate = true
99
91
  end
100
92
 
101
93
  # Runs the controller `#shutdown` method that should be triggered when a given consumer is
@@ -107,7 +99,6 @@ module Karafka
107
99
  # There is a case, where the consumer no longer exists because it was revoked, in case like
108
100
  # that we do not build a new instance and shutdown should not be triggered.
109
101
  consumer.on_shutdown if @consumer
110
- @recreate = true
111
102
  end
112
103
 
113
104
  private
@@ -115,10 +106,9 @@ module Karafka
115
106
  # @return [Object] cached consumer instance
116
107
  def consumer
117
108
  @consumer ||= begin
118
- consumer = @topic.consumer.new
109
+ consumer = @topic.consumer_class.new
119
110
  consumer.topic = @topic
120
111
  consumer.client = @client
121
- consumer.pause_tracker = @pause_tracker
122
112
  consumer.producer = ::Karafka::App.producer
123
113
  consumer
124
114
  end
@@ -11,30 +11,48 @@ module Karafka
11
11
  def initialize(client, subscription_group)
12
12
  @subscription_group = subscription_group
13
13
  @client = client
14
- @buffer = Hash.new { |h, k| h[k] = {} }
14
+ # We need two layers here to keep track of topics, partitions and processing groups
15
+ @buffer = Hash.new { |h, k| h[k] = Hash.new { |h2, k2| h2[k2] = {} } }
15
16
  end
16
17
 
18
+ # Finds or creates an executor based on the provided details
19
+ #
17
20
  # @param topic [String] topic name
18
21
  # @param partition [Integer] partition number
19
- # @param pause [TimeTrackers::Pause] pause corresponding with provided topic and partition
22
+ # @param parallel_key [String] parallel group key
20
23
  # @return [Executor] consumer executor
21
- def fetch(
22
- topic,
23
- partition,
24
- pause
25
- )
26
- ktopic = @subscription_group.topics.find(topic)
24
+ def find_or_create(topic, partition, parallel_key)
25
+ ktopic = find_topic(topic)
27
26
 
28
- ktopic || raise(Errors::TopicNotFoundError, topic)
29
-
30
- @buffer[ktopic][partition] ||= Executor.new(
27
+ @buffer[ktopic][partition][parallel_key] ||= Executor.new(
31
28
  @subscription_group.id,
32
29
  @client,
33
- ktopic,
34
- pause
30
+ ktopic
35
31
  )
36
32
  end
37
33
 
34
+ # Revokes executors of a given topic partition, so they won't be used anymore for incoming
35
+ # messages
36
+ #
37
+ # @param topic [String] topic name
38
+ # @param partition [Integer] partition number
39
+ def revoke(topic, partition)
40
+ ktopic = find_topic(topic)
41
+
42
+ @buffer[ktopic][partition].clear
43
+ end
44
+
45
+ # Finds all the executors available for a given topic partition
46
+ #
47
+ # @param topic [String] topic name
48
+ # @param partition [Integer] partition number
49
+ # @return [Array<Executor>] executors in use for this topic + partition
50
+ def find_all(topic, partition)
51
+ ktopic = find_topic(topic)
52
+
53
+ @buffer[ktopic][partition].values
54
+ end
55
+
38
56
  # Iterates over all available executors and yields them together with topic and partition
39
57
  # info
40
58
  # @yieldparam [Routing::Topic] karafka routing topic object
@@ -42,8 +60,11 @@ module Karafka
42
60
  # @yieldparam [Executor] given executor
43
61
  def each
44
62
  @buffer.each do |ktopic, partitions|
45
- partitions.each do |partition, executor|
46
- yield(ktopic, partition, executor)
63
+ partitions.each do |partition, executors|
64
+ executors.each do |_parallel_key, executor|
65
+ # We skip the parallel key here as it does not serve any value when iterating
66
+ yield(ktopic, partition, executor)
67
+ end
47
68
  end
48
69
  end
49
70
  end
@@ -52,6 +73,16 @@ module Karafka
52
73
  def clear
53
74
  @buffer.clear
54
75
  end
76
+
77
+ private
78
+
79
+ # Finds topic based on its name
80
+ #
81
+ # @param topic [String] topic we're looking for
82
+ # @return [Karafka::Routing::Topic] topic we're interested in
83
+ def find_topic(topic)
84
+ @subscription_group.topics.find(topic) || raise(Errors::TopicNotFoundError, topic)
85
+ end
55
86
  end
56
87
  end
57
88
  end
@@ -12,17 +12,19 @@ module Karafka
12
12
  # @param executor [Karafka::Processing::Executor] executor that is suppose to run a given
13
13
  # job
14
14
  # @param messages [Karafka::Messages::Messages] karafka messages batch
15
+ # @param coordinator [Karafka::Processing::Coordinator] processing coordinator
15
16
  # @return [Consume]
16
- def initialize(executor, messages)
17
+ def initialize(executor, messages, coordinator)
17
18
  @executor = executor
18
19
  @messages = messages
20
+ @coordinator = coordinator
19
21
  @created_at = Time.now
20
22
  super()
21
23
  end
22
24
 
23
25
  # Runs the before consumption preparations on the executor
24
26
  def before_call
25
- executor.before_consume(@messages, @created_at)
27
+ executor.before_consume(@messages, @created_at, @coordinator)
26
28
  end
27
29
 
28
30
  # Runs the given executor
@@ -7,9 +7,10 @@ module Karafka
7
7
  class JobsBuilder
8
8
  # @param executor [Karafka::Processing::Executor]
9
9
  # @param messages [Karafka::Messages::Messages] messages batch to be consumed
10
+ # @param coordinator [Karafka::Processing::Coordinator]
10
11
  # @return [Karafka::Processing::Jobs::Consume] consumption job
11
- def consume(executor, messages)
12
- Jobs::Consume.new(executor, messages)
12
+ def consume(executor, messages, coordinator)
13
+ Jobs::Consume.new(executor, messages, coordinator)
13
14
  end
14
15
 
15
16
  # @param executor [Karafka::Processing::Executor]
@@ -10,11 +10,6 @@ module Karafka
10
10
  @success = true
11
11
  end
12
12
 
13
- # @return [Boolean]
14
- def failure?
15
- !success?
16
- end
17
-
18
13
  # @return [Boolean]
19
14
  def success?
20
15
  @success
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Processing
5
+ # FIFO scheduler for messages coming from various topics and partitions
6
+ class Scheduler
7
+ # Schedules jobs in the fifo order
8
+ #
9
+ # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
10
+ # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
11
+ def schedule_consumption(queue, jobs_array)
12
+ jobs_array.each do |job|
13
+ queue << job
14
+ end
15
+ end
16
+
17
+ # Both revocation and shutdown jobs can also run in fifo by default
18
+ alias schedule_revocation schedule_consumption
19
+ alias schedule_shutdown schedule_consumption
20
+ end
21
+ end
22
+ end
@@ -38,7 +38,7 @@ module Karafka
38
38
  # @return [Array<Routing::SubscriptionGroup>] all the subscription groups build based on
39
39
  # the consumer group topics
40
40
  def subscription_groups
41
- App.config.internal.subscription_groups_builder.call(topics)
41
+ App.config.internal.routing.subscription_groups_builder.call(topics)
42
42
  end
43
43
 
44
44
  # Hashed version of consumer group that can be used for validation purposes
@@ -66,6 +66,15 @@ module Karafka
66
66
  end
67
67
  end
68
68
 
69
+ # @return [Class] consumer class that we should use
70
+ # @note This is just an alias to the `#consumer` method. We however want to use it internally
71
+ # instead of referencing the `#consumer`. We use this to indicate that this method returns
72
+ # class and not an instance. In the routing we want to keep the `#consumer Consumer`
73
+ # routing syntax, but for references outside, we should use this one.
74
+ def consumer_class
75
+ consumer
76
+ end
77
+
69
78
  # @return [Boolean] true if this topic offset is handled by the end user
70
79
  def manual_offset_management?
71
80
  manual_offset_management