karafka 2.0.0.beta4 → 2.0.0.beta5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.github/workflows/ci.yml +18 -1
- data/CHANGELOG.md +15 -0
- data/Gemfile.lock +1 -1
- data/bin/benchmarks +2 -2
- data/bin/integrations +10 -3
- data/bin/{stress → stress_many} +0 -0
- data/bin/stress_one +13 -0
- data/docker-compose.yml +23 -18
- data/lib/karafka/active_job/routing/extensions.rb +1 -1
- data/lib/karafka/app.rb +2 -1
- data/lib/karafka/base_consumer.rb +26 -19
- data/lib/karafka/connection/client.rb +24 -4
- data/lib/karafka/connection/listener.rb +49 -11
- data/lib/karafka/connection/pauses_manager.rb +8 -0
- data/lib/karafka/connection/rebalance_manager.rb +20 -19
- data/lib/karafka/contracts/config.rb +17 -4
- data/lib/karafka/contracts/server_cli_options.rb +1 -1
- data/lib/karafka/errors.rb +3 -0
- data/lib/karafka/pro/active_job/consumer.rb +1 -8
- data/lib/karafka/pro/base_consumer.rb +10 -13
- data/lib/karafka/pro/loader.rb +11 -6
- data/lib/karafka/pro/processing/coordinator.rb +12 -0
- data/lib/karafka/pro/processing/jobs_builder.rb +3 -2
- data/lib/karafka/pro/processing/scheduler.rb +56 -0
- data/lib/karafka/processing/coordinator.rb +84 -0
- data/lib/karafka/processing/coordinators_buffer.rb +58 -0
- data/lib/karafka/processing/executor.rb +6 -16
- data/lib/karafka/processing/executors_buffer.rb +46 -15
- data/lib/karafka/processing/jobs/consume.rb +4 -2
- data/lib/karafka/processing/jobs_builder.rb +3 -2
- data/lib/karafka/processing/result.rb +0 -5
- data/lib/karafka/processing/scheduler.rb +22 -0
- data/lib/karafka/routing/consumer_group.rb +1 -1
- data/lib/karafka/routing/topic.rb +9 -0
- data/lib/karafka/setup/config.rb +18 -10
- data/lib/karafka/version.rb +1 -1
- data.tar.gz.sig +0 -0
- metadata +9 -5
- metadata.gz.sig +4 -1
- data/lib/karafka/pro/scheduler.rb +0 -54
- data/lib/karafka/scheduler.rb +0 -20
@@ -30,12 +30,25 @@ module Karafka
|
|
30
30
|
|
31
31
|
# We validate internals just to be sure, that they are present and working
|
32
32
|
required(:internal).schema do
|
33
|
-
required(:routing_builder)
|
34
|
-
required(:subscription_groups_builder)
|
35
|
-
required(:jobs_builder)
|
36
33
|
required(:status)
|
37
34
|
required(:process)
|
38
|
-
|
35
|
+
|
36
|
+
required(:routing).schema do
|
37
|
+
required(:builder)
|
38
|
+
required(:subscription_groups_builder)
|
39
|
+
end
|
40
|
+
|
41
|
+
required(:processing).schema do
|
42
|
+
required(:jobs_builder)
|
43
|
+
required(:scheduler)
|
44
|
+
required(:coordinator_class)
|
45
|
+
end
|
46
|
+
|
47
|
+
required(:active_job).schema do
|
48
|
+
required(:dispatcher)
|
49
|
+
required(:job_options_contract)
|
50
|
+
required(:consumer_class)
|
51
|
+
end
|
39
52
|
end
|
40
53
|
end
|
41
54
|
|
@@ -12,7 +12,7 @@ module Karafka
|
|
12
12
|
# If there were no consumer_groups declared in the server cli, it means that we will
|
13
13
|
# run all of them and no need to validate them here at all
|
14
14
|
if !value.nil? &&
|
15
|
-
!(value - Karafka::App.config.internal.
|
15
|
+
!(value - Karafka::App.config.internal.routing.builder.map(&:name)).empty?
|
16
16
|
key(:consumer_groups).failure(:consumer_groups_inclusion)
|
17
17
|
end
|
18
18
|
end
|
data/lib/karafka/errors.rb
CHANGED
@@ -47,5 +47,8 @@ module Karafka
|
|
47
47
|
# Used to instrument this error into the error notifications
|
48
48
|
# We do not raise it so we won't crash deployed systems
|
49
49
|
ExpiredLicenseTokenError = Class.new(BaseError)
|
50
|
+
|
51
|
+
# This should never happen. Please open an issue if it does.
|
52
|
+
InvalidCoordinatorState = Class.new(BaseError)
|
50
53
|
end
|
51
54
|
end
|
@@ -26,7 +26,7 @@ module Karafka
|
|
26
26
|
messages.each do |message|
|
27
27
|
# If for any reason we've lost this partition, not worth iterating over new messages
|
28
28
|
# as they are no longer ours
|
29
|
-
|
29
|
+
break if revoked?
|
30
30
|
break if Karafka::App.stopping?
|
31
31
|
|
32
32
|
::ActiveJob::Base.execute(
|
@@ -34,13 +34,6 @@ module Karafka
|
|
34
34
|
)
|
35
35
|
|
36
36
|
mark_as_consumed(message)
|
37
|
-
|
38
|
-
# We check it twice as the job may be long running
|
39
|
-
# If marking fails, it also means it got revoked and we can stop consuming
|
40
|
-
return if revoked?
|
41
|
-
|
42
|
-
# Do not process more if we are shutting down
|
43
|
-
break if Karafka::App.stopping?
|
44
37
|
end
|
45
38
|
end
|
46
39
|
end
|
@@ -39,38 +39,35 @@ module Karafka
|
|
39
39
|
# Nothing to do if we lost the partition
|
40
40
|
return if revoked?
|
41
41
|
|
42
|
-
if @
|
43
|
-
pause_tracker.reset
|
42
|
+
if @coordinator.success?
|
43
|
+
coordinator.pause_tracker.reset
|
44
44
|
|
45
45
|
# We use the non-blocking one here. If someone needs the blocking one, can implement it
|
46
46
|
# with manual offset management
|
47
47
|
# Mark as consumed only if manual offset management is not on
|
48
48
|
mark_as_consumed(messages.last) unless topic.manual_offset_management?
|
49
49
|
|
50
|
+
# We check it twice as marking could change this state
|
51
|
+
return if revoked?
|
52
|
+
|
50
53
|
# If this is not a long running job there is nothing for us to do here
|
51
54
|
return unless topic.long_running_job?
|
52
55
|
|
53
56
|
# Once processing is done, we move to the new offset based on commits
|
54
57
|
# Here, in case manual offset management is off, we have the new proper offset of a
|
55
58
|
# first message from another batch from `@seek_offset`. If manual offset management
|
56
|
-
# is on, we move to place where the user indicated it was finished.
|
59
|
+
# is on, we move to place where the user indicated it was finished. This can create an
|
60
|
+
# interesting (yet valid) corner case, where with manual offset management on and no
|
61
|
+
# marking as consumed, we end up with an infinite loop processing same messages over and
|
62
|
+
# over again
|
57
63
|
seek(@seek_offset || messages.first.offset)
|
64
|
+
|
58
65
|
resume
|
59
66
|
else
|
60
67
|
# If processing failed, we need to pause
|
61
68
|
pause(@seek_offset || messages.first.offset)
|
62
69
|
end
|
63
70
|
end
|
64
|
-
|
65
|
-
# Marks this consumer revoked state as true
|
66
|
-
# This allows us for things like lrj to finish early as this state may change during lrj
|
67
|
-
# execution
|
68
|
-
def on_revoked
|
69
|
-
# @note This may already be set to true if we tried to commit offsets and failed. In case
|
70
|
-
# like this it will automatically be marked as revoked.
|
71
|
-
@revoked = true
|
72
|
-
super
|
73
|
-
end
|
74
71
|
end
|
75
72
|
end
|
76
73
|
end
|
data/lib/karafka/pro/loader.rb
CHANGED
@@ -17,9 +17,10 @@ module Karafka
|
|
17
17
|
COMPONENTS = %w[
|
18
18
|
base_consumer
|
19
19
|
performance_tracker
|
20
|
-
scheduler
|
20
|
+
processing/scheduler
|
21
21
|
processing/jobs/consume_non_blocking
|
22
22
|
processing/jobs_builder
|
23
|
+
processing/coordinator
|
23
24
|
routing/extensions
|
24
25
|
active_job/consumer
|
25
26
|
active_job/dispatcher
|
@@ -35,11 +36,15 @@ module Karafka
|
|
35
36
|
def setup(config)
|
36
37
|
COMPONENTS.each { |component| require_relative(component) }
|
37
38
|
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
39
|
+
icfg = config.internal
|
40
|
+
|
41
|
+
icfg.processing.coordinator_class = Processing::Coordinator
|
42
|
+
icfg.processing.scheduler = Processing::Scheduler.new
|
43
|
+
icfg.processing.jobs_builder = Processing::JobsBuilder.new
|
44
|
+
|
45
|
+
icfg.active_job.consumer_class = ActiveJob::Consumer
|
46
|
+
icfg.active_job.dispatcher = ActiveJob::Dispatcher.new
|
47
|
+
icfg.active_job.job_options_contract = ActiveJob::JobOptionsContract.new
|
43
48
|
|
44
49
|
::Karafka::Routing::Topic.include(Routing::Extensions)
|
45
50
|
|
@@ -0,0 +1,12 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Pro
|
5
|
+
module Processing
|
6
|
+
# Pro coordinator that provides extra orchestration methods useful for parallel processing
|
7
|
+
# within the same partition
|
8
|
+
class Coordinator < ::Karafka::Processing::Coordinator
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
@@ -16,11 +16,12 @@ module Karafka
|
|
16
16
|
class JobsBuilder < ::Karafka::Processing::JobsBuilder
|
17
17
|
# @param executor [Karafka::Processing::Executor]
|
18
18
|
# @param messages [Karafka::Messages::Messages] messages batch to be consumed
|
19
|
+
# @param coordinator [Karafka::Processing::Coordinator]
|
19
20
|
# @return [Karafka::Processing::Jobs::Consume] blocking job
|
20
21
|
# @return [Karafka::Pro::Processing::Jobs::ConsumeNonBlocking] non blocking for lrj
|
21
|
-
def consume(executor, messages)
|
22
|
+
def consume(executor, messages, coordinator)
|
22
23
|
if executor.topic.long_running_job?
|
23
|
-
Jobs::ConsumeNonBlocking.new(executor, messages)
|
24
|
+
Jobs::ConsumeNonBlocking.new(executor, messages, coordinator)
|
24
25
|
else
|
25
26
|
super
|
26
27
|
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# This Karafka component is a Pro component.
|
4
|
+
# All of the commercial components are present in the lib/karafka/pro directory of this
|
5
|
+
# repository and their usage requires commercial license agreement.
|
6
|
+
#
|
7
|
+
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
8
|
+
#
|
9
|
+
# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
|
10
|
+
# your code to Maciej Mensfeld.
|
11
|
+
|
12
|
+
module Karafka
|
13
|
+
module Pro
|
14
|
+
module Processing
|
15
|
+
# Optimizes scheduler that takes into consideration of execution time needed to process
|
16
|
+
# messages from given topics partitions. It uses the non-preemptive LJF algorithm
|
17
|
+
#
|
18
|
+
# This scheduler is designed to optimize execution times on jobs that perform IO operations
|
19
|
+
# as when taking IO into consideration, the can achieve optimized parallel processing.
|
20
|
+
#
|
21
|
+
# This scheduler can also work with virtual partitions.
|
22
|
+
#
|
23
|
+
# Aside from consumption jobs, other jobs do not run often, thus we can leave them with
|
24
|
+
# default FIFO scheduler from the default Karafka scheduler
|
25
|
+
class Scheduler < ::Karafka::Processing::Scheduler
|
26
|
+
# Schedules jobs in the LJF order for consumption
|
27
|
+
#
|
28
|
+
# @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
|
29
|
+
# @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
|
30
|
+
#
|
31
|
+
def schedule_consumption(queue, jobs_array)
|
32
|
+
pt = PerformanceTracker.instance
|
33
|
+
|
34
|
+
ordered = []
|
35
|
+
|
36
|
+
jobs_array.each do |job|
|
37
|
+
messages = job.messages
|
38
|
+
message = messages.first
|
39
|
+
|
40
|
+
cost = pt.processing_time_p95(message.topic, message.partition) * messages.size
|
41
|
+
|
42
|
+
ordered << [job, cost]
|
43
|
+
end
|
44
|
+
|
45
|
+
ordered.sort_by!(&:last)
|
46
|
+
ordered.reverse!
|
47
|
+
ordered.map!(&:first)
|
48
|
+
|
49
|
+
ordered.each do |job|
|
50
|
+
queue << job
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,84 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Processing
|
5
|
+
# Basic coordinator that allows us to provide coordination objects into consumers.
|
6
|
+
#
|
7
|
+
# This is a wrapping layer to simplify management of work to be handled around consumption.
|
8
|
+
#
|
9
|
+
# @note This coordinator needs to be thread safe. Some operations are performed only in the
|
10
|
+
# listener thread, but we go with thread-safe by default for all not to worry about potential
|
11
|
+
# future mistakes.
|
12
|
+
class Coordinator
|
13
|
+
# @return [Karafka::TimeTrackers::Pause]
|
14
|
+
attr_reader :pause_tracker
|
15
|
+
|
16
|
+
# @param pause_tracker [Karafka::TimeTrackers::Pause] pause tracker for given topic partition
|
17
|
+
def initialize(pause_tracker)
|
18
|
+
@pause_tracker = pause_tracker
|
19
|
+
@revoked = false
|
20
|
+
@consumptions = {}
|
21
|
+
@running_jobs = 0
|
22
|
+
@mutex = Mutex.new
|
23
|
+
end
|
24
|
+
|
25
|
+
# Starts the coordinator for given consumption jobs
|
26
|
+
def start
|
27
|
+
@mutex.synchronize do
|
28
|
+
@running_jobs = 0
|
29
|
+
# We need to clear the consumption results hash here, otherwise we could end up storing
|
30
|
+
# consumption results of consumer instances we no longer control
|
31
|
+
@consumptions.clear
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
# Increases number of jobs that we handle with this coordinator
|
36
|
+
def increment
|
37
|
+
@mutex.synchronize { @running_jobs += 1 }
|
38
|
+
end
|
39
|
+
|
40
|
+
# Decrements number of jobs we handle at the moment
|
41
|
+
def decrement
|
42
|
+
@mutex.synchronize do
|
43
|
+
@running_jobs -= 1
|
44
|
+
|
45
|
+
return @running_jobs unless @running_jobs.negative?
|
46
|
+
|
47
|
+
raise Karafka::Errors::InvalidCoordinatorState, @running_jobs
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
# @param consumer [Object] karafka consumer (normal or pro)
|
52
|
+
# @return [Karafka::Processing::Result] result object which we can use to indicate
|
53
|
+
# consumption processing state.
|
54
|
+
def consumption(consumer)
|
55
|
+
@mutex.synchronize do
|
56
|
+
@consumptions[consumer] ||= Processing::Result.new
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
# Is all the consumption done and finished successfully for this coordinator
|
61
|
+
def success?
|
62
|
+
@mutex.synchronize { @running_jobs.zero? && @consumptions.values.all?(&:success?) }
|
63
|
+
end
|
64
|
+
|
65
|
+
# Marks given coordinator for processing group as revoked
|
66
|
+
#
|
67
|
+
# This is invoked in two places:
|
68
|
+
# - from the main listener loop when we detect revoked partitions
|
69
|
+
# - from the consumer in case checkpointing fails
|
70
|
+
#
|
71
|
+
# This means, we can end up having consumer being aware that it was revoked prior to the
|
72
|
+
# listener loop dispatching the revocation job. It is ok, as effectively nothing will be
|
73
|
+
# processed until revocation jobs are done.
|
74
|
+
def revoke
|
75
|
+
@mutex.synchronize { @revoked = true }
|
76
|
+
end
|
77
|
+
|
78
|
+
# @return [Boolean] is the partition we are processing revoked or not
|
79
|
+
def revoked?
|
80
|
+
@revoked
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Processing
|
5
|
+
# Buffer used to build and store coordinators per topic partition
|
6
|
+
#
|
7
|
+
# It provides direct pauses access for revocation
|
8
|
+
#
|
9
|
+
# @note This buffer operates only from the listener loop, thus we do not have to make it
|
10
|
+
# thread-safe.
|
11
|
+
class CoordinatorsBuffer
|
12
|
+
def initialize
|
13
|
+
@pauses_manager = Connection::PausesManager.new
|
14
|
+
@coordinator_class = ::Karafka::App.config.internal.processing.coordinator_class
|
15
|
+
@coordinators = Hash.new { |h, k| h[k] = {} }
|
16
|
+
end
|
17
|
+
|
18
|
+
# @param topic [String] topic name
|
19
|
+
# @param partition [Integer] partition number
|
20
|
+
def find_or_create(topic, partition)
|
21
|
+
@coordinators[topic][partition] ||= @coordinator_class.new(
|
22
|
+
@pauses_manager.fetch(topic, partition)
|
23
|
+
)
|
24
|
+
end
|
25
|
+
|
26
|
+
# Resumes processing of partitions for which pause time has ended.
|
27
|
+
# @param block we want to run for resumed topic partitions
|
28
|
+
# @yieldparam [String] topic name
|
29
|
+
# @yieldparam [Integer] partition number
|
30
|
+
def resume(&block)
|
31
|
+
@pauses_manager.resume(&block)
|
32
|
+
end
|
33
|
+
|
34
|
+
# @param topic [String] topic name
|
35
|
+
# @param partition [Integer] partition number
|
36
|
+
def revoke(topic, partition)
|
37
|
+
@pauses_manager.revoke(topic, partition)
|
38
|
+
|
39
|
+
# The fact that we delete here does not change the fact that the executor still holds the
|
40
|
+
# reference to this coordinator. We delete it here, as we will no longer process any
|
41
|
+
# new stuff with it and we may need a new coordinator if we regain this partition, but the
|
42
|
+
# coordinator may still be in use
|
43
|
+
coordinator = @coordinators[topic].delete(partition)
|
44
|
+
|
45
|
+
return unless coordinator
|
46
|
+
|
47
|
+
coordinator.revoke
|
48
|
+
end
|
49
|
+
|
50
|
+
# Clears coordinators and re-created the pauses manager
|
51
|
+
# This should be used only for critical errors recovery
|
52
|
+
def reset
|
53
|
+
@pauses_manager = Connection::PausesManager.new
|
54
|
+
@coordinators.clear
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -30,13 +30,11 @@ module Karafka
|
|
30
30
|
# @param group_id [String] id of the subscription group to which the executor belongs
|
31
31
|
# @param client [Karafka::Connection::Client] kafka client
|
32
32
|
# @param topic [Karafka::Routing::Topic] topic for which this executor will run
|
33
|
-
|
34
|
-
def initialize(group_id, client, topic, pause_tracker)
|
33
|
+
def initialize(group_id, client, topic)
|
35
34
|
@id = SecureRandom.uuid
|
36
35
|
@group_id = group_id
|
37
36
|
@client = client
|
38
37
|
@topic = topic
|
39
|
-
@pause_tracker = pause_tracker
|
40
38
|
end
|
41
39
|
|
42
40
|
# Builds the consumer instance, builds messages batch and sets all that is needed to run the
|
@@ -45,20 +43,15 @@ module Karafka
|
|
45
43
|
# @param messages [Array<Karafka::Messages::Message>]
|
46
44
|
# @param received_at [Time] the moment we've received the batch (actually the moment we've)
|
47
45
|
# enqueued it, but good enough
|
48
|
-
|
46
|
+
# @param coordinator [Karafka::Processing::Coordinator] coordinator for processing management
|
47
|
+
def before_consume(messages, received_at, coordinator)
|
49
48
|
# Recreate consumer with each batch if persistence is not enabled
|
50
49
|
# We reload the consumers with each batch instead of relying on some external signals
|
51
50
|
# when needed for consistency. That way devs may have it on or off and not in this
|
52
51
|
# middle state, where re-creation of a consumer instance would occur only sometimes
|
53
|
-
@
|
52
|
+
@consumer = nil unless ::Karafka::App.config.consumer_persistence
|
54
53
|
|
55
|
-
|
56
|
-
# a shutdown happened and we need to have a new instance for running another consume for
|
57
|
-
# this topic partition
|
58
|
-
if @recreate
|
59
|
-
@consumer = nil
|
60
|
-
@recreate = false
|
61
|
-
end
|
54
|
+
consumer.coordinator = coordinator
|
62
55
|
|
63
56
|
# First we build messages batch...
|
64
57
|
consumer.messages = Messages::Builders::Messages.call(
|
@@ -95,7 +88,6 @@ module Karafka
|
|
95
88
|
# consumer instance.
|
96
89
|
def revoked
|
97
90
|
consumer.on_revoked if @consumer
|
98
|
-
@recreate = true
|
99
91
|
end
|
100
92
|
|
101
93
|
# Runs the controller `#shutdown` method that should be triggered when a given consumer is
|
@@ -107,7 +99,6 @@ module Karafka
|
|
107
99
|
# There is a case, where the consumer no longer exists because it was revoked, in case like
|
108
100
|
# that we do not build a new instance and shutdown should not be triggered.
|
109
101
|
consumer.on_shutdown if @consumer
|
110
|
-
@recreate = true
|
111
102
|
end
|
112
103
|
|
113
104
|
private
|
@@ -115,10 +106,9 @@ module Karafka
|
|
115
106
|
# @return [Object] cached consumer instance
|
116
107
|
def consumer
|
117
108
|
@consumer ||= begin
|
118
|
-
consumer = @topic.
|
109
|
+
consumer = @topic.consumer_class.new
|
119
110
|
consumer.topic = @topic
|
120
111
|
consumer.client = @client
|
121
|
-
consumer.pause_tracker = @pause_tracker
|
122
112
|
consumer.producer = ::Karafka::App.producer
|
123
113
|
consumer
|
124
114
|
end
|
@@ -11,30 +11,48 @@ module Karafka
|
|
11
11
|
def initialize(client, subscription_group)
|
12
12
|
@subscription_group = subscription_group
|
13
13
|
@client = client
|
14
|
-
|
14
|
+
# We need two layers here to keep track of topics, partitions and processing groups
|
15
|
+
@buffer = Hash.new { |h, k| h[k] = Hash.new { |h2, k2| h2[k2] = {} } }
|
15
16
|
end
|
16
17
|
|
18
|
+
# Finds or creates an executor based on the provided details
|
19
|
+
#
|
17
20
|
# @param topic [String] topic name
|
18
21
|
# @param partition [Integer] partition number
|
19
|
-
# @param
|
22
|
+
# @param parallel_key [String] parallel group key
|
20
23
|
# @return [Executor] consumer executor
|
21
|
-
def
|
22
|
-
topic
|
23
|
-
partition,
|
24
|
-
pause
|
25
|
-
)
|
26
|
-
ktopic = @subscription_group.topics.find(topic)
|
24
|
+
def find_or_create(topic, partition, parallel_key)
|
25
|
+
ktopic = find_topic(topic)
|
27
26
|
|
28
|
-
ktopic
|
29
|
-
|
30
|
-
@buffer[ktopic][partition] ||= Executor.new(
|
27
|
+
@buffer[ktopic][partition][parallel_key] ||= Executor.new(
|
31
28
|
@subscription_group.id,
|
32
29
|
@client,
|
33
|
-
ktopic
|
34
|
-
pause
|
30
|
+
ktopic
|
35
31
|
)
|
36
32
|
end
|
37
33
|
|
34
|
+
# Revokes executors of a given topic partition, so they won't be used anymore for incoming
|
35
|
+
# messages
|
36
|
+
#
|
37
|
+
# @param topic [String] topic name
|
38
|
+
# @param partition [Integer] partition number
|
39
|
+
def revoke(topic, partition)
|
40
|
+
ktopic = find_topic(topic)
|
41
|
+
|
42
|
+
@buffer[ktopic][partition].clear
|
43
|
+
end
|
44
|
+
|
45
|
+
# Finds all the executors available for a given topic partition
|
46
|
+
#
|
47
|
+
# @param topic [String] topic name
|
48
|
+
# @param partition [Integer] partition number
|
49
|
+
# @return [Array<Executor>] executors in use for this topic + partition
|
50
|
+
def find_all(topic, partition)
|
51
|
+
ktopic = find_topic(topic)
|
52
|
+
|
53
|
+
@buffer[ktopic][partition].values
|
54
|
+
end
|
55
|
+
|
38
56
|
# Iterates over all available executors and yields them together with topic and partition
|
39
57
|
# info
|
40
58
|
# @yieldparam [Routing::Topic] karafka routing topic object
|
@@ -42,8 +60,11 @@ module Karafka
|
|
42
60
|
# @yieldparam [Executor] given executor
|
43
61
|
def each
|
44
62
|
@buffer.each do |ktopic, partitions|
|
45
|
-
partitions.each do |partition,
|
46
|
-
|
63
|
+
partitions.each do |partition, executors|
|
64
|
+
executors.each do |_parallel_key, executor|
|
65
|
+
# We skip the parallel key here as it does not serve any value when iterating
|
66
|
+
yield(ktopic, partition, executor)
|
67
|
+
end
|
47
68
|
end
|
48
69
|
end
|
49
70
|
end
|
@@ -52,6 +73,16 @@ module Karafka
|
|
52
73
|
def clear
|
53
74
|
@buffer.clear
|
54
75
|
end
|
76
|
+
|
77
|
+
private
|
78
|
+
|
79
|
+
# Finds topic based on its name
|
80
|
+
#
|
81
|
+
# @param topic [String] topic we're looking for
|
82
|
+
# @return [Karafka::Routing::Topic] topic we're interested in
|
83
|
+
def find_topic(topic)
|
84
|
+
@subscription_group.topics.find(topic) || raise(Errors::TopicNotFoundError, topic)
|
85
|
+
end
|
55
86
|
end
|
56
87
|
end
|
57
88
|
end
|
@@ -12,17 +12,19 @@ module Karafka
|
|
12
12
|
# @param executor [Karafka::Processing::Executor] executor that is suppose to run a given
|
13
13
|
# job
|
14
14
|
# @param messages [Karafka::Messages::Messages] karafka messages batch
|
15
|
+
# @param coordinator [Karafka::Processing::Coordinator] processing coordinator
|
15
16
|
# @return [Consume]
|
16
|
-
def initialize(executor, messages)
|
17
|
+
def initialize(executor, messages, coordinator)
|
17
18
|
@executor = executor
|
18
19
|
@messages = messages
|
20
|
+
@coordinator = coordinator
|
19
21
|
@created_at = Time.now
|
20
22
|
super()
|
21
23
|
end
|
22
24
|
|
23
25
|
# Runs the before consumption preparations on the executor
|
24
26
|
def before_call
|
25
|
-
executor.before_consume(@messages, @created_at)
|
27
|
+
executor.before_consume(@messages, @created_at, @coordinator)
|
26
28
|
end
|
27
29
|
|
28
30
|
# Runs the given executor
|
@@ -7,9 +7,10 @@ module Karafka
|
|
7
7
|
class JobsBuilder
|
8
8
|
# @param executor [Karafka::Processing::Executor]
|
9
9
|
# @param messages [Karafka::Messages::Messages] messages batch to be consumed
|
10
|
+
# @param coordinator [Karafka::Processing::Coordinator]
|
10
11
|
# @return [Karafka::Processing::Jobs::Consume] consumption job
|
11
|
-
def consume(executor, messages)
|
12
|
-
Jobs::Consume.new(executor, messages)
|
12
|
+
def consume(executor, messages, coordinator)
|
13
|
+
Jobs::Consume.new(executor, messages, coordinator)
|
13
14
|
end
|
14
15
|
|
15
16
|
# @param executor [Karafka::Processing::Executor]
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Processing
|
5
|
+
# FIFO scheduler for messages coming from various topics and partitions
|
6
|
+
class Scheduler
|
7
|
+
# Schedules jobs in the fifo order
|
8
|
+
#
|
9
|
+
# @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
|
10
|
+
# @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
|
11
|
+
def schedule_consumption(queue, jobs_array)
|
12
|
+
jobs_array.each do |job|
|
13
|
+
queue << job
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
# Both revocation and shutdown jobs can also run in fifo by default
|
18
|
+
alias schedule_revocation schedule_consumption
|
19
|
+
alias schedule_shutdown schedule_consumption
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -38,7 +38,7 @@ module Karafka
|
|
38
38
|
# @return [Array<Routing::SubscriptionGroup>] all the subscription groups build based on
|
39
39
|
# the consumer group topics
|
40
40
|
def subscription_groups
|
41
|
-
App.config.internal.subscription_groups_builder.call(topics)
|
41
|
+
App.config.internal.routing.subscription_groups_builder.call(topics)
|
42
42
|
end
|
43
43
|
|
44
44
|
# Hashed version of consumer group that can be used for validation purposes
|
@@ -66,6 +66,15 @@ module Karafka
|
|
66
66
|
end
|
67
67
|
end
|
68
68
|
|
69
|
+
# @return [Class] consumer class that we should use
|
70
|
+
# @note This is just an alias to the `#consumer` method. We however want to use it internally
|
71
|
+
# instead of referencing the `#consumer`. We use this to indicate that this method returns
|
72
|
+
# class and not an instance. In the routing we want to keep the `#consumer Consumer`
|
73
|
+
# routing syntax, but for references outside, we should use this one.
|
74
|
+
def consumer_class
|
75
|
+
consumer
|
76
|
+
end
|
77
|
+
|
69
78
|
# @return [Boolean] true if this topic offset is handled by the end user
|
70
79
|
def manual_offset_management?
|
71
80
|
manual_offset_management
|