karafka 2.0.0.beta4 → 2.0.0.beta5

Sign up to get free protection for your applications and to get access to all the features.
Files changed (43) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/.github/workflows/ci.yml +18 -1
  4. data/CHANGELOG.md +15 -0
  5. data/Gemfile.lock +1 -1
  6. data/bin/benchmarks +2 -2
  7. data/bin/integrations +10 -3
  8. data/bin/{stress → stress_many} +0 -0
  9. data/bin/stress_one +13 -0
  10. data/docker-compose.yml +23 -18
  11. data/lib/karafka/active_job/routing/extensions.rb +1 -1
  12. data/lib/karafka/app.rb +2 -1
  13. data/lib/karafka/base_consumer.rb +26 -19
  14. data/lib/karafka/connection/client.rb +24 -4
  15. data/lib/karafka/connection/listener.rb +49 -11
  16. data/lib/karafka/connection/pauses_manager.rb +8 -0
  17. data/lib/karafka/connection/rebalance_manager.rb +20 -19
  18. data/lib/karafka/contracts/config.rb +17 -4
  19. data/lib/karafka/contracts/server_cli_options.rb +1 -1
  20. data/lib/karafka/errors.rb +3 -0
  21. data/lib/karafka/pro/active_job/consumer.rb +1 -8
  22. data/lib/karafka/pro/base_consumer.rb +10 -13
  23. data/lib/karafka/pro/loader.rb +11 -6
  24. data/lib/karafka/pro/processing/coordinator.rb +12 -0
  25. data/lib/karafka/pro/processing/jobs_builder.rb +3 -2
  26. data/lib/karafka/pro/processing/scheduler.rb +56 -0
  27. data/lib/karafka/processing/coordinator.rb +84 -0
  28. data/lib/karafka/processing/coordinators_buffer.rb +58 -0
  29. data/lib/karafka/processing/executor.rb +6 -16
  30. data/lib/karafka/processing/executors_buffer.rb +46 -15
  31. data/lib/karafka/processing/jobs/consume.rb +4 -2
  32. data/lib/karafka/processing/jobs_builder.rb +3 -2
  33. data/lib/karafka/processing/result.rb +0 -5
  34. data/lib/karafka/processing/scheduler.rb +22 -0
  35. data/lib/karafka/routing/consumer_group.rb +1 -1
  36. data/lib/karafka/routing/topic.rb +9 -0
  37. data/lib/karafka/setup/config.rb +18 -10
  38. data/lib/karafka/version.rb +1 -1
  39. data.tar.gz.sig +0 -0
  40. metadata +9 -5
  41. metadata.gz.sig +4 -1
  42. data/lib/karafka/pro/scheduler.rb +0 -54
  43. data/lib/karafka/scheduler.rb +0 -20
@@ -30,12 +30,25 @@ module Karafka
30
30
 
31
31
  # We validate internals just to be sure, that they are present and working
32
32
  required(:internal).schema do
33
- required(:routing_builder)
34
- required(:subscription_groups_builder)
35
- required(:jobs_builder)
36
33
  required(:status)
37
34
  required(:process)
38
- required(:scheduler)
35
+
36
+ required(:routing).schema do
37
+ required(:builder)
38
+ required(:subscription_groups_builder)
39
+ end
40
+
41
+ required(:processing).schema do
42
+ required(:jobs_builder)
43
+ required(:scheduler)
44
+ required(:coordinator_class)
45
+ end
46
+
47
+ required(:active_job).schema do
48
+ required(:dispatcher)
49
+ required(:job_options_contract)
50
+ required(:consumer_class)
51
+ end
39
52
  end
40
53
  end
41
54
 
@@ -12,7 +12,7 @@ module Karafka
12
12
  # If there were no consumer_groups declared in the server cli, it means that we will
13
13
  # run all of them and no need to validate them here at all
14
14
  if !value.nil? &&
15
- !(value - Karafka::App.config.internal.routing_builder.map(&:name)).empty?
15
+ !(value - Karafka::App.config.internal.routing.builder.map(&:name)).empty?
16
16
  key(:consumer_groups).failure(:consumer_groups_inclusion)
17
17
  end
18
18
  end
@@ -47,5 +47,8 @@ module Karafka
47
47
  # Used to instrument this error into the error notifications
48
48
  # We do not raise it so we won't crash deployed systems
49
49
  ExpiredLicenseTokenError = Class.new(BaseError)
50
+
51
+ # This should never happen. Please open an issue if it does.
52
+ InvalidCoordinatorState = Class.new(BaseError)
50
53
  end
51
54
  end
@@ -26,7 +26,7 @@ module Karafka
26
26
  messages.each do |message|
27
27
  # If for any reason we've lost this partition, not worth iterating over new messages
28
28
  # as they are no longer ours
29
- return if revoked?
29
+ break if revoked?
30
30
  break if Karafka::App.stopping?
31
31
 
32
32
  ::ActiveJob::Base.execute(
@@ -34,13 +34,6 @@ module Karafka
34
34
  )
35
35
 
36
36
  mark_as_consumed(message)
37
-
38
- # We check it twice as the job may be long running
39
- # If marking fails, it also means it got revoked and we can stop consuming
40
- return if revoked?
41
-
42
- # Do not process more if we are shutting down
43
- break if Karafka::App.stopping?
44
37
  end
45
38
  end
46
39
  end
@@ -39,38 +39,35 @@ module Karafka
39
39
  # Nothing to do if we lost the partition
40
40
  return if revoked?
41
41
 
42
- if @consumption.success?
43
- pause_tracker.reset
42
+ if @coordinator.success?
43
+ coordinator.pause_tracker.reset
44
44
 
45
45
  # We use the non-blocking one here. If someone needs the blocking one, can implement it
46
46
  # with manual offset management
47
47
  # Mark as consumed only if manual offset management is not on
48
48
  mark_as_consumed(messages.last) unless topic.manual_offset_management?
49
49
 
50
+ # We check it twice as marking could change this state
51
+ return if revoked?
52
+
50
53
  # If this is not a long running job there is nothing for us to do here
51
54
  return unless topic.long_running_job?
52
55
 
53
56
  # Once processing is done, we move to the new offset based on commits
54
57
  # Here, in case manual offset management is off, we have the new proper offset of a
55
58
  # first message from another batch from `@seek_offset`. If manual offset management
56
- # is on, we move to place where the user indicated it was finished.
59
+ # is on, we move to place where the user indicated it was finished. This can create an
60
+ # interesting (yet valid) corner case, where with manual offset management on and no
61
+ # marking as consumed, we end up with an infinite loop processing same messages over and
62
+ # over again
57
63
  seek(@seek_offset || messages.first.offset)
64
+
58
65
  resume
59
66
  else
60
67
  # If processing failed, we need to pause
61
68
  pause(@seek_offset || messages.first.offset)
62
69
  end
63
70
  end
64
-
65
- # Marks this consumer revoked state as true
66
- # This allows us for things like lrj to finish early as this state may change during lrj
67
- # execution
68
- def on_revoked
69
- # @note This may already be set to true if we tried to commit offsets and failed. In case
70
- # like this it will automatically be marked as revoked.
71
- @revoked = true
72
- super
73
- end
74
71
  end
75
72
  end
76
73
  end
@@ -17,9 +17,10 @@ module Karafka
17
17
  COMPONENTS = %w[
18
18
  base_consumer
19
19
  performance_tracker
20
- scheduler
20
+ processing/scheduler
21
21
  processing/jobs/consume_non_blocking
22
22
  processing/jobs_builder
23
+ processing/coordinator
23
24
  routing/extensions
24
25
  active_job/consumer
25
26
  active_job/dispatcher
@@ -35,11 +36,15 @@ module Karafka
35
36
  def setup(config)
36
37
  COMPONENTS.each { |component| require_relative(component) }
37
38
 
38
- config.internal.scheduler = Scheduler.new
39
- config.internal.jobs_builder = Processing::JobsBuilder.new
40
- config.internal.active_job.consumer = ActiveJob::Consumer
41
- config.internal.active_job.dispatcher = ActiveJob::Dispatcher.new
42
- config.internal.active_job.job_options_contract = ActiveJob::JobOptionsContract.new
39
+ icfg = config.internal
40
+
41
+ icfg.processing.coordinator_class = Processing::Coordinator
42
+ icfg.processing.scheduler = Processing::Scheduler.new
43
+ icfg.processing.jobs_builder = Processing::JobsBuilder.new
44
+
45
+ icfg.active_job.consumer_class = ActiveJob::Consumer
46
+ icfg.active_job.dispatcher = ActiveJob::Dispatcher.new
47
+ icfg.active_job.job_options_contract = ActiveJob::JobOptionsContract.new
43
48
 
44
49
  ::Karafka::Routing::Topic.include(Routing::Extensions)
45
50
 
@@ -0,0 +1,12 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Pro
5
+ module Processing
6
+ # Pro coordinator that provides extra orchestration methods useful for parallel processing
7
+ # within the same partition
8
+ class Coordinator < ::Karafka::Processing::Coordinator
9
+ end
10
+ end
11
+ end
12
+ end
@@ -16,11 +16,12 @@ module Karafka
16
16
  class JobsBuilder < ::Karafka::Processing::JobsBuilder
17
17
  # @param executor [Karafka::Processing::Executor]
18
18
  # @param messages [Karafka::Messages::Messages] messages batch to be consumed
19
+ # @param coordinator [Karafka::Processing::Coordinator]
19
20
  # @return [Karafka::Processing::Jobs::Consume] blocking job
20
21
  # @return [Karafka::Pro::Processing::Jobs::ConsumeNonBlocking] non blocking for lrj
21
- def consume(executor, messages)
22
+ def consume(executor, messages, coordinator)
22
23
  if executor.topic.long_running_job?
23
- Jobs::ConsumeNonBlocking.new(executor, messages)
24
+ Jobs::ConsumeNonBlocking.new(executor, messages, coordinator)
24
25
  else
25
26
  super
26
27
  end
@@ -0,0 +1,56 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component.
4
+ # All of the commercial components are present in the lib/karafka/pro directory of this
5
+ # repository and their usage requires commercial license agreement.
6
+ #
7
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
8
+ #
9
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
10
+ # your code to Maciej Mensfeld.
11
+
12
+ module Karafka
13
+ module Pro
14
+ module Processing
15
+ # Optimizes scheduler that takes into consideration of execution time needed to process
16
+ # messages from given topics partitions. It uses the non-preemptive LJF algorithm
17
+ #
18
+ # This scheduler is designed to optimize execution times on jobs that perform IO operations
19
+ # as when taking IO into consideration, the can achieve optimized parallel processing.
20
+ #
21
+ # This scheduler can also work with virtual partitions.
22
+ #
23
+ # Aside from consumption jobs, other jobs do not run often, thus we can leave them with
24
+ # default FIFO scheduler from the default Karafka scheduler
25
+ class Scheduler < ::Karafka::Processing::Scheduler
26
+ # Schedules jobs in the LJF order for consumption
27
+ #
28
+ # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
29
+ # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
30
+ #
31
+ def schedule_consumption(queue, jobs_array)
32
+ pt = PerformanceTracker.instance
33
+
34
+ ordered = []
35
+
36
+ jobs_array.each do |job|
37
+ messages = job.messages
38
+ message = messages.first
39
+
40
+ cost = pt.processing_time_p95(message.topic, message.partition) * messages.size
41
+
42
+ ordered << [job, cost]
43
+ end
44
+
45
+ ordered.sort_by!(&:last)
46
+ ordered.reverse!
47
+ ordered.map!(&:first)
48
+
49
+ ordered.each do |job|
50
+ queue << job
51
+ end
52
+ end
53
+ end
54
+ end
55
+ end
56
+ end
@@ -0,0 +1,84 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Processing
5
+ # Basic coordinator that allows us to provide coordination objects into consumers.
6
+ #
7
+ # This is a wrapping layer to simplify management of work to be handled around consumption.
8
+ #
9
+ # @note This coordinator needs to be thread safe. Some operations are performed only in the
10
+ # listener thread, but we go with thread-safe by default for all not to worry about potential
11
+ # future mistakes.
12
+ class Coordinator
13
+ # @return [Karafka::TimeTrackers::Pause]
14
+ attr_reader :pause_tracker
15
+
16
+ # @param pause_tracker [Karafka::TimeTrackers::Pause] pause tracker for given topic partition
17
+ def initialize(pause_tracker)
18
+ @pause_tracker = pause_tracker
19
+ @revoked = false
20
+ @consumptions = {}
21
+ @running_jobs = 0
22
+ @mutex = Mutex.new
23
+ end
24
+
25
+ # Starts the coordinator for given consumption jobs
26
+ def start
27
+ @mutex.synchronize do
28
+ @running_jobs = 0
29
+ # We need to clear the consumption results hash here, otherwise we could end up storing
30
+ # consumption results of consumer instances we no longer control
31
+ @consumptions.clear
32
+ end
33
+ end
34
+
35
+ # Increases number of jobs that we handle with this coordinator
36
+ def increment
37
+ @mutex.synchronize { @running_jobs += 1 }
38
+ end
39
+
40
+ # Decrements number of jobs we handle at the moment
41
+ def decrement
42
+ @mutex.synchronize do
43
+ @running_jobs -= 1
44
+
45
+ return @running_jobs unless @running_jobs.negative?
46
+
47
+ raise Karafka::Errors::InvalidCoordinatorState, @running_jobs
48
+ end
49
+ end
50
+
51
+ # @param consumer [Object] karafka consumer (normal or pro)
52
+ # @return [Karafka::Processing::Result] result object which we can use to indicate
53
+ # consumption processing state.
54
+ def consumption(consumer)
55
+ @mutex.synchronize do
56
+ @consumptions[consumer] ||= Processing::Result.new
57
+ end
58
+ end
59
+
60
+ # Is all the consumption done and finished successfully for this coordinator
61
+ def success?
62
+ @mutex.synchronize { @running_jobs.zero? && @consumptions.values.all?(&:success?) }
63
+ end
64
+
65
+ # Marks given coordinator for processing group as revoked
66
+ #
67
+ # This is invoked in two places:
68
+ # - from the main listener loop when we detect revoked partitions
69
+ # - from the consumer in case checkpointing fails
70
+ #
71
+ # This means, we can end up having consumer being aware that it was revoked prior to the
72
+ # listener loop dispatching the revocation job. It is ok, as effectively nothing will be
73
+ # processed until revocation jobs are done.
74
+ def revoke
75
+ @mutex.synchronize { @revoked = true }
76
+ end
77
+
78
+ # @return [Boolean] is the partition we are processing revoked or not
79
+ def revoked?
80
+ @revoked
81
+ end
82
+ end
83
+ end
84
+ end
@@ -0,0 +1,58 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Processing
5
+ # Buffer used to build and store coordinators per topic partition
6
+ #
7
+ # It provides direct pauses access for revocation
8
+ #
9
+ # @note This buffer operates only from the listener loop, thus we do not have to make it
10
+ # thread-safe.
11
+ class CoordinatorsBuffer
12
+ def initialize
13
+ @pauses_manager = Connection::PausesManager.new
14
+ @coordinator_class = ::Karafka::App.config.internal.processing.coordinator_class
15
+ @coordinators = Hash.new { |h, k| h[k] = {} }
16
+ end
17
+
18
+ # @param topic [String] topic name
19
+ # @param partition [Integer] partition number
20
+ def find_or_create(topic, partition)
21
+ @coordinators[topic][partition] ||= @coordinator_class.new(
22
+ @pauses_manager.fetch(topic, partition)
23
+ )
24
+ end
25
+
26
+ # Resumes processing of partitions for which pause time has ended.
27
+ # @param block we want to run for resumed topic partitions
28
+ # @yieldparam [String] topic name
29
+ # @yieldparam [Integer] partition number
30
+ def resume(&block)
31
+ @pauses_manager.resume(&block)
32
+ end
33
+
34
+ # @param topic [String] topic name
35
+ # @param partition [Integer] partition number
36
+ def revoke(topic, partition)
37
+ @pauses_manager.revoke(topic, partition)
38
+
39
+ # The fact that we delete here does not change the fact that the executor still holds the
40
+ # reference to this coordinator. We delete it here, as we will no longer process any
41
+ # new stuff with it and we may need a new coordinator if we regain this partition, but the
42
+ # coordinator may still be in use
43
+ coordinator = @coordinators[topic].delete(partition)
44
+
45
+ return unless coordinator
46
+
47
+ coordinator.revoke
48
+ end
49
+
50
+ # Clears coordinators and re-created the pauses manager
51
+ # This should be used only for critical errors recovery
52
+ def reset
53
+ @pauses_manager = Connection::PausesManager.new
54
+ @coordinators.clear
55
+ end
56
+ end
57
+ end
58
+ end
@@ -30,13 +30,11 @@ module Karafka
30
30
  # @param group_id [String] id of the subscription group to which the executor belongs
31
31
  # @param client [Karafka::Connection::Client] kafka client
32
32
  # @param topic [Karafka::Routing::Topic] topic for which this executor will run
33
- # @param pause_tracker [Karafka::TimeTrackers::Pause] fetch pause tracker for pausing
34
- def initialize(group_id, client, topic, pause_tracker)
33
+ def initialize(group_id, client, topic)
35
34
  @id = SecureRandom.uuid
36
35
  @group_id = group_id
37
36
  @client = client
38
37
  @topic = topic
39
- @pause_tracker = pause_tracker
40
38
  end
41
39
 
42
40
  # Builds the consumer instance, builds messages batch and sets all that is needed to run the
@@ -45,20 +43,15 @@ module Karafka
45
43
  # @param messages [Array<Karafka::Messages::Message>]
46
44
  # @param received_at [Time] the moment we've received the batch (actually the moment we've)
47
45
  # enqueued it, but good enough
48
- def before_consume(messages, received_at)
46
+ # @param coordinator [Karafka::Processing::Coordinator] coordinator for processing management
47
+ def before_consume(messages, received_at, coordinator)
49
48
  # Recreate consumer with each batch if persistence is not enabled
50
49
  # We reload the consumers with each batch instead of relying on some external signals
51
50
  # when needed for consistency. That way devs may have it on or off and not in this
52
51
  # middle state, where re-creation of a consumer instance would occur only sometimes
53
- @recreate = true unless ::Karafka::App.config.consumer_persistence
52
+ @consumer = nil unless ::Karafka::App.config.consumer_persistence
54
53
 
55
- # If @recreate was set to true (aside from non persistent), it means, that revocation or
56
- # a shutdown happened and we need to have a new instance for running another consume for
57
- # this topic partition
58
- if @recreate
59
- @consumer = nil
60
- @recreate = false
61
- end
54
+ consumer.coordinator = coordinator
62
55
 
63
56
  # First we build messages batch...
64
57
  consumer.messages = Messages::Builders::Messages.call(
@@ -95,7 +88,6 @@ module Karafka
95
88
  # consumer instance.
96
89
  def revoked
97
90
  consumer.on_revoked if @consumer
98
- @recreate = true
99
91
  end
100
92
 
101
93
  # Runs the controller `#shutdown` method that should be triggered when a given consumer is
@@ -107,7 +99,6 @@ module Karafka
107
99
  # There is a case, where the consumer no longer exists because it was revoked, in case like
108
100
  # that we do not build a new instance and shutdown should not be triggered.
109
101
  consumer.on_shutdown if @consumer
110
- @recreate = true
111
102
  end
112
103
 
113
104
  private
@@ -115,10 +106,9 @@ module Karafka
115
106
  # @return [Object] cached consumer instance
116
107
  def consumer
117
108
  @consumer ||= begin
118
- consumer = @topic.consumer.new
109
+ consumer = @topic.consumer_class.new
119
110
  consumer.topic = @topic
120
111
  consumer.client = @client
121
- consumer.pause_tracker = @pause_tracker
122
112
  consumer.producer = ::Karafka::App.producer
123
113
  consumer
124
114
  end
@@ -11,30 +11,48 @@ module Karafka
11
11
  def initialize(client, subscription_group)
12
12
  @subscription_group = subscription_group
13
13
  @client = client
14
- @buffer = Hash.new { |h, k| h[k] = {} }
14
+ # We need two layers here to keep track of topics, partitions and processing groups
15
+ @buffer = Hash.new { |h, k| h[k] = Hash.new { |h2, k2| h2[k2] = {} } }
15
16
  end
16
17
 
18
+ # Finds or creates an executor based on the provided details
19
+ #
17
20
  # @param topic [String] topic name
18
21
  # @param partition [Integer] partition number
19
- # @param pause [TimeTrackers::Pause] pause corresponding with provided topic and partition
22
+ # @param parallel_key [String] parallel group key
20
23
  # @return [Executor] consumer executor
21
- def fetch(
22
- topic,
23
- partition,
24
- pause
25
- )
26
- ktopic = @subscription_group.topics.find(topic)
24
+ def find_or_create(topic, partition, parallel_key)
25
+ ktopic = find_topic(topic)
27
26
 
28
- ktopic || raise(Errors::TopicNotFoundError, topic)
29
-
30
- @buffer[ktopic][partition] ||= Executor.new(
27
+ @buffer[ktopic][partition][parallel_key] ||= Executor.new(
31
28
  @subscription_group.id,
32
29
  @client,
33
- ktopic,
34
- pause
30
+ ktopic
35
31
  )
36
32
  end
37
33
 
34
+ # Revokes executors of a given topic partition, so they won't be used anymore for incoming
35
+ # messages
36
+ #
37
+ # @param topic [String] topic name
38
+ # @param partition [Integer] partition number
39
+ def revoke(topic, partition)
40
+ ktopic = find_topic(topic)
41
+
42
+ @buffer[ktopic][partition].clear
43
+ end
44
+
45
+ # Finds all the executors available for a given topic partition
46
+ #
47
+ # @param topic [String] topic name
48
+ # @param partition [Integer] partition number
49
+ # @return [Array<Executor>] executors in use for this topic + partition
50
+ def find_all(topic, partition)
51
+ ktopic = find_topic(topic)
52
+
53
+ @buffer[ktopic][partition].values
54
+ end
55
+
38
56
  # Iterates over all available executors and yields them together with topic and partition
39
57
  # info
40
58
  # @yieldparam [Routing::Topic] karafka routing topic object
@@ -42,8 +60,11 @@ module Karafka
42
60
  # @yieldparam [Executor] given executor
43
61
  def each
44
62
  @buffer.each do |ktopic, partitions|
45
- partitions.each do |partition, executor|
46
- yield(ktopic, partition, executor)
63
+ partitions.each do |partition, executors|
64
+ executors.each do |_parallel_key, executor|
65
+ # We skip the parallel key here as it does not serve any value when iterating
66
+ yield(ktopic, partition, executor)
67
+ end
47
68
  end
48
69
  end
49
70
  end
@@ -52,6 +73,16 @@ module Karafka
52
73
  def clear
53
74
  @buffer.clear
54
75
  end
76
+
77
+ private
78
+
79
+ # Finds topic based on its name
80
+ #
81
+ # @param topic [String] topic we're looking for
82
+ # @return [Karafka::Routing::Topic] topic we're interested in
83
+ def find_topic(topic)
84
+ @subscription_group.topics.find(topic) || raise(Errors::TopicNotFoundError, topic)
85
+ end
55
86
  end
56
87
  end
57
88
  end
@@ -12,17 +12,19 @@ module Karafka
12
12
  # @param executor [Karafka::Processing::Executor] executor that is suppose to run a given
13
13
  # job
14
14
  # @param messages [Karafka::Messages::Messages] karafka messages batch
15
+ # @param coordinator [Karafka::Processing::Coordinator] processing coordinator
15
16
  # @return [Consume]
16
- def initialize(executor, messages)
17
+ def initialize(executor, messages, coordinator)
17
18
  @executor = executor
18
19
  @messages = messages
20
+ @coordinator = coordinator
19
21
  @created_at = Time.now
20
22
  super()
21
23
  end
22
24
 
23
25
  # Runs the before consumption preparations on the executor
24
26
  def before_call
25
- executor.before_consume(@messages, @created_at)
27
+ executor.before_consume(@messages, @created_at, @coordinator)
26
28
  end
27
29
 
28
30
  # Runs the given executor
@@ -7,9 +7,10 @@ module Karafka
7
7
  class JobsBuilder
8
8
  # @param executor [Karafka::Processing::Executor]
9
9
  # @param messages [Karafka::Messages::Messages] messages batch to be consumed
10
+ # @param coordinator [Karafka::Processing::Coordinator]
10
11
  # @return [Karafka::Processing::Jobs::Consume] consumption job
11
- def consume(executor, messages)
12
- Jobs::Consume.new(executor, messages)
12
+ def consume(executor, messages, coordinator)
13
+ Jobs::Consume.new(executor, messages, coordinator)
13
14
  end
14
15
 
15
16
  # @param executor [Karafka::Processing::Executor]
@@ -10,11 +10,6 @@ module Karafka
10
10
  @success = true
11
11
  end
12
12
 
13
- # @return [Boolean]
14
- def failure?
15
- !success?
16
- end
17
-
18
13
  # @return [Boolean]
19
14
  def success?
20
15
  @success
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Processing
5
+ # FIFO scheduler for messages coming from various topics and partitions
6
+ class Scheduler
7
+ # Schedules jobs in the fifo order
8
+ #
9
+ # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
10
+ # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
11
+ def schedule_consumption(queue, jobs_array)
12
+ jobs_array.each do |job|
13
+ queue << job
14
+ end
15
+ end
16
+
17
+ # Both revocation and shutdown jobs can also run in fifo by default
18
+ alias schedule_revocation schedule_consumption
19
+ alias schedule_shutdown schedule_consumption
20
+ end
21
+ end
22
+ end
@@ -38,7 +38,7 @@ module Karafka
38
38
  # @return [Array<Routing::SubscriptionGroup>] all the subscription groups build based on
39
39
  # the consumer group topics
40
40
  def subscription_groups
41
- App.config.internal.subscription_groups_builder.call(topics)
41
+ App.config.internal.routing.subscription_groups_builder.call(topics)
42
42
  end
43
43
 
44
44
  # Hashed version of consumer group that can be used for validation purposes
@@ -66,6 +66,15 @@ module Karafka
66
66
  end
67
67
  end
68
68
 
69
+ # @return [Class] consumer class that we should use
70
+ # @note This is just an alias to the `#consumer` method. We however want to use it internally
71
+ # instead of referencing the `#consumer`. We use this to indicate that this method returns
72
+ # class and not an instance. In the routing we want to keep the `#consumer Consumer`
73
+ # routing syntax, but for references outside, we should use this one.
74
+ def consumer_class
75
+ consumer
76
+ end
77
+
69
78
  # @return [Boolean] true if this topic offset is handled by the end user
70
79
  def manual_offset_management?
71
80
  manual_offset_management