karafka 2.0.0.beta2 → 2.0.0.beta5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.github/workflows/ci.yml +18 -15
- data/CHANGELOG.md +49 -0
- data/Gemfile.lock +8 -8
- data/bin/benchmarks +2 -2
- data/bin/integrations +44 -15
- data/bin/scenario +29 -0
- data/bin/{stress → stress_many} +0 -0
- data/bin/stress_one +13 -0
- data/bin/wait_for_kafka +20 -0
- data/docker-compose.yml +28 -11
- data/karafka.gemspec +2 -2
- data/lib/karafka/active_job/routing/extensions.rb +12 -2
- data/lib/karafka/app.rb +2 -1
- data/lib/karafka/base_consumer.rb +75 -45
- data/lib/karafka/connection/client.rb +88 -22
- data/lib/karafka/connection/listener.rb +60 -18
- data/lib/karafka/connection/pauses_manager.rb +8 -0
- data/lib/karafka/connection/rebalance_manager.rb +20 -19
- data/lib/karafka/contracts/config.rb +17 -3
- data/lib/karafka/contracts/server_cli_options.rb +1 -1
- data/lib/karafka/errors.rb +3 -0
- data/lib/karafka/instrumentation/logger_listener.rb +34 -10
- data/lib/karafka/instrumentation/monitor.rb +3 -1
- data/lib/karafka/licenser.rb +26 -7
- data/lib/karafka/pro/active_job/consumer.rb +30 -9
- data/lib/karafka/pro/active_job/dispatcher.rb +9 -9
- data/lib/karafka/pro/active_job/job_options_contract.rb +9 -9
- data/lib/karafka/pro/base_consumer.rb +73 -0
- data/lib/karafka/pro/loader.rb +38 -20
- data/lib/karafka/pro/performance_tracker.rb +9 -9
- data/lib/karafka/pro/processing/coordinator.rb +12 -0
- data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb +10 -11
- data/lib/karafka/pro/processing/jobs_builder.rb +32 -0
- data/lib/karafka/pro/processing/scheduler.rb +56 -0
- data/lib/karafka/pro/routing/extensions.rb +32 -0
- data/lib/karafka/processing/coordinator.rb +84 -0
- data/lib/karafka/processing/coordinators_buffer.rb +58 -0
- data/lib/karafka/processing/executor.rb +23 -9
- data/lib/karafka/processing/executors_buffer.rb +46 -15
- data/lib/karafka/processing/jobs/base.rb +8 -3
- data/lib/karafka/processing/jobs/consume.rb +11 -4
- data/lib/karafka/processing/jobs_builder.rb +29 -0
- data/lib/karafka/processing/result.rb +29 -0
- data/lib/karafka/processing/scheduler.rb +22 -0
- data/lib/karafka/processing/worker.rb +17 -9
- data/lib/karafka/routing/consumer_group.rb +1 -1
- data/lib/karafka/routing/subscription_group.rb +1 -1
- data/lib/karafka/routing/topic.rb +14 -0
- data/lib/karafka/setup/config.rb +19 -9
- data/lib/karafka/status.rb +1 -3
- data/lib/karafka/version.rb +1 -1
- data.tar.gz.sig +0 -0
- metadata +19 -7
- metadata.gz.sig +0 -0
- data/lib/karafka/pro/scheduler.rb +0 -54
- data/lib/karafka/scheduler.rb +0 -20
@@ -0,0 +1,84 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Processing
|
5
|
+
# Basic coordinator that allows us to provide coordination objects into consumers.
|
6
|
+
#
|
7
|
+
# This is a wrapping layer to simplify management of work to be handled around consumption.
|
8
|
+
#
|
9
|
+
# @note This coordinator needs to be thread safe. Some operations are performed only in the
|
10
|
+
# listener thread, but we go with thread-safe by default for all not to worry about potential
|
11
|
+
# future mistakes.
|
12
|
+
class Coordinator
|
13
|
+
# @return [Karafka::TimeTrackers::Pause]
|
14
|
+
attr_reader :pause_tracker
|
15
|
+
|
16
|
+
# @param pause_tracker [Karafka::TimeTrackers::Pause] pause tracker for given topic partition
|
17
|
+
def initialize(pause_tracker)
|
18
|
+
@pause_tracker = pause_tracker
|
19
|
+
@revoked = false
|
20
|
+
@consumptions = {}
|
21
|
+
@running_jobs = 0
|
22
|
+
@mutex = Mutex.new
|
23
|
+
end
|
24
|
+
|
25
|
+
# Starts the coordinator for given consumption jobs
|
26
|
+
def start
|
27
|
+
@mutex.synchronize do
|
28
|
+
@running_jobs = 0
|
29
|
+
# We need to clear the consumption results hash here, otherwise we could end up storing
|
30
|
+
# consumption results of consumer instances we no longer control
|
31
|
+
@consumptions.clear
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
# Increases number of jobs that we handle with this coordinator
|
36
|
+
def increment
|
37
|
+
@mutex.synchronize { @running_jobs += 1 }
|
38
|
+
end
|
39
|
+
|
40
|
+
# Decrements number of jobs we handle at the moment
|
41
|
+
def decrement
|
42
|
+
@mutex.synchronize do
|
43
|
+
@running_jobs -= 1
|
44
|
+
|
45
|
+
return @running_jobs unless @running_jobs.negative?
|
46
|
+
|
47
|
+
raise Karafka::Errors::InvalidCoordinatorState, @running_jobs
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
# @param consumer [Object] karafka consumer (normal or pro)
|
52
|
+
# @return [Karafka::Processing::Result] result object which we can use to indicate
|
53
|
+
# consumption processing state.
|
54
|
+
def consumption(consumer)
|
55
|
+
@mutex.synchronize do
|
56
|
+
@consumptions[consumer] ||= Processing::Result.new
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
# Is all the consumption done and finished successfully for this coordinator
|
61
|
+
def success?
|
62
|
+
@mutex.synchronize { @running_jobs.zero? && @consumptions.values.all?(&:success?) }
|
63
|
+
end
|
64
|
+
|
65
|
+
# Marks given coordinator for processing group as revoked
|
66
|
+
#
|
67
|
+
# This is invoked in two places:
|
68
|
+
# - from the main listener loop when we detect revoked partitions
|
69
|
+
# - from the consumer in case checkpointing fails
|
70
|
+
#
|
71
|
+
# This means, we can end up having consumer being aware that it was revoked prior to the
|
72
|
+
# listener loop dispatching the revocation job. It is ok, as effectively nothing will be
|
73
|
+
# processed until revocation jobs are done.
|
74
|
+
def revoke
|
75
|
+
@mutex.synchronize { @revoked = true }
|
76
|
+
end
|
77
|
+
|
78
|
+
# @return [Boolean] is the partition we are processing revoked or not
|
79
|
+
def revoked?
|
80
|
+
@revoked
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Processing
|
5
|
+
# Buffer used to build and store coordinators per topic partition
|
6
|
+
#
|
7
|
+
# It provides direct pauses access for revocation
|
8
|
+
#
|
9
|
+
# @note This buffer operates only from the listener loop, thus we do not have to make it
|
10
|
+
# thread-safe.
|
11
|
+
class CoordinatorsBuffer
|
12
|
+
def initialize
|
13
|
+
@pauses_manager = Connection::PausesManager.new
|
14
|
+
@coordinator_class = ::Karafka::App.config.internal.processing.coordinator_class
|
15
|
+
@coordinators = Hash.new { |h, k| h[k] = {} }
|
16
|
+
end
|
17
|
+
|
18
|
+
# @param topic [String] topic name
|
19
|
+
# @param partition [Integer] partition number
|
20
|
+
def find_or_create(topic, partition)
|
21
|
+
@coordinators[topic][partition] ||= @coordinator_class.new(
|
22
|
+
@pauses_manager.fetch(topic, partition)
|
23
|
+
)
|
24
|
+
end
|
25
|
+
|
26
|
+
# Resumes processing of partitions for which pause time has ended.
|
27
|
+
# @param block we want to run for resumed topic partitions
|
28
|
+
# @yieldparam [String] topic name
|
29
|
+
# @yieldparam [Integer] partition number
|
30
|
+
def resume(&block)
|
31
|
+
@pauses_manager.resume(&block)
|
32
|
+
end
|
33
|
+
|
34
|
+
# @param topic [String] topic name
|
35
|
+
# @param partition [Integer] partition number
|
36
|
+
def revoke(topic, partition)
|
37
|
+
@pauses_manager.revoke(topic, partition)
|
38
|
+
|
39
|
+
# The fact that we delete here does not change the fact that the executor still holds the
|
40
|
+
# reference to this coordinator. We delete it here, as we will no longer process any
|
41
|
+
# new stuff with it and we may need a new coordinator if we regain this partition, but the
|
42
|
+
# coordinator may still be in use
|
43
|
+
coordinator = @coordinators[topic].delete(partition)
|
44
|
+
|
45
|
+
return unless coordinator
|
46
|
+
|
47
|
+
coordinator.revoke
|
48
|
+
end
|
49
|
+
|
50
|
+
# Clears coordinators and re-created the pauses manager
|
51
|
+
# This should be used only for critical errors recovery
|
52
|
+
def reset
|
53
|
+
@pauses_manager = Connection::PausesManager.new
|
54
|
+
@coordinators.clear
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -18,18 +18,23 @@ module Karafka
|
|
18
18
|
# @return [String] subscription group id to which a given executor belongs
|
19
19
|
attr_reader :group_id
|
20
20
|
|
21
|
+
# @return [Karafka::Messages::Messages] messages batch
|
21
22
|
attr_reader :messages
|
22
23
|
|
24
|
+
# Topic accessibility may be needed for the jobs builder to be able to build a proper job
|
25
|
+
# based on the topic settings defined by the end user
|
26
|
+
#
|
27
|
+
# @return [Karafka::Routing::Topic] topic of this executor
|
28
|
+
attr_reader :topic
|
29
|
+
|
23
30
|
# @param group_id [String] id of the subscription group to which the executor belongs
|
24
31
|
# @param client [Karafka::Connection::Client] kafka client
|
25
32
|
# @param topic [Karafka::Routing::Topic] topic for which this executor will run
|
26
|
-
|
27
|
-
def initialize(group_id, client, topic, pause_tracker)
|
33
|
+
def initialize(group_id, client, topic)
|
28
34
|
@id = SecureRandom.uuid
|
29
35
|
@group_id = group_id
|
30
36
|
@client = client
|
31
37
|
@topic = topic
|
32
|
-
@pause_tracker = pause_tracker
|
33
38
|
end
|
34
39
|
|
35
40
|
# Builds the consumer instance, builds messages batch and sets all that is needed to run the
|
@@ -38,13 +43,16 @@ module Karafka
|
|
38
43
|
# @param messages [Array<Karafka::Messages::Message>]
|
39
44
|
# @param received_at [Time] the moment we've received the batch (actually the moment we've)
|
40
45
|
# enqueued it, but good enough
|
41
|
-
|
46
|
+
# @param coordinator [Karafka::Processing::Coordinator] coordinator for processing management
|
47
|
+
def before_consume(messages, received_at, coordinator)
|
42
48
|
# Recreate consumer with each batch if persistence is not enabled
|
43
49
|
# We reload the consumers with each batch instead of relying on some external signals
|
44
50
|
# when needed for consistency. That way devs may have it on or off and not in this
|
45
51
|
# middle state, where re-creation of a consumer instance would occur only sometimes
|
46
52
|
@consumer = nil unless ::Karafka::App.config.consumer_persistence
|
47
53
|
|
54
|
+
consumer.coordinator = coordinator
|
55
|
+
|
48
56
|
# First we build messages batch...
|
49
57
|
consumer.messages = Messages::Builders::Messages.call(
|
50
58
|
messages,
|
@@ -52,7 +60,7 @@ module Karafka
|
|
52
60
|
received_at
|
53
61
|
)
|
54
62
|
|
55
|
-
consumer.
|
63
|
+
consumer.on_before_consume
|
56
64
|
end
|
57
65
|
|
58
66
|
# Runs consumer data processing against given batch and handles failures and errors.
|
@@ -61,6 +69,11 @@ module Karafka
|
|
61
69
|
consumer.on_consume
|
62
70
|
end
|
63
71
|
|
72
|
+
# Runs consumer after consumption code
|
73
|
+
def after_consume
|
74
|
+
consumer.on_after_consume if @consumer
|
75
|
+
end
|
76
|
+
|
64
77
|
# Runs the controller `#revoked` method that should be triggered when a given consumer is
|
65
78
|
# no longer needed due to partitions reassignment.
|
66
79
|
#
|
@@ -69,9 +82,12 @@ module Karafka
|
|
69
82
|
#
|
70
83
|
# @note We run it only when consumer was present, because presence indicates, that at least
|
71
84
|
# a single message has been consumed.
|
85
|
+
#
|
86
|
+
# @note We do not reset the consumer but we indicate need for recreation instead, because
|
87
|
+
# after the revocation, there still may be `#after_consume` running that needs a given
|
88
|
+
# consumer instance.
|
72
89
|
def revoked
|
73
90
|
consumer.on_revoked if @consumer
|
74
|
-
@consumer = nil
|
75
91
|
end
|
76
92
|
|
77
93
|
# Runs the controller `#shutdown` method that should be triggered when a given consumer is
|
@@ -83,7 +99,6 @@ module Karafka
|
|
83
99
|
# There is a case, where the consumer no longer exists because it was revoked, in case like
|
84
100
|
# that we do not build a new instance and shutdown should not be triggered.
|
85
101
|
consumer.on_shutdown if @consumer
|
86
|
-
@consumer = nil
|
87
102
|
end
|
88
103
|
|
89
104
|
private
|
@@ -91,10 +106,9 @@ module Karafka
|
|
91
106
|
# @return [Object] cached consumer instance
|
92
107
|
def consumer
|
93
108
|
@consumer ||= begin
|
94
|
-
consumer = @topic.
|
109
|
+
consumer = @topic.consumer_class.new
|
95
110
|
consumer.topic = @topic
|
96
111
|
consumer.client = @client
|
97
|
-
consumer.pause_tracker = @pause_tracker
|
98
112
|
consumer.producer = ::Karafka::App.producer
|
99
113
|
consumer
|
100
114
|
end
|
@@ -11,30 +11,48 @@ module Karafka
|
|
11
11
|
def initialize(client, subscription_group)
|
12
12
|
@subscription_group = subscription_group
|
13
13
|
@client = client
|
14
|
-
|
14
|
+
# We need two layers here to keep track of topics, partitions and processing groups
|
15
|
+
@buffer = Hash.new { |h, k| h[k] = Hash.new { |h2, k2| h2[k2] = {} } }
|
15
16
|
end
|
16
17
|
|
18
|
+
# Finds or creates an executor based on the provided details
|
19
|
+
#
|
17
20
|
# @param topic [String] topic name
|
18
21
|
# @param partition [Integer] partition number
|
19
|
-
# @param
|
22
|
+
# @param parallel_key [String] parallel group key
|
20
23
|
# @return [Executor] consumer executor
|
21
|
-
def
|
22
|
-
topic
|
23
|
-
partition,
|
24
|
-
pause
|
25
|
-
)
|
26
|
-
ktopic = @subscription_group.topics.find(topic)
|
24
|
+
def find_or_create(topic, partition, parallel_key)
|
25
|
+
ktopic = find_topic(topic)
|
27
26
|
|
28
|
-
ktopic
|
29
|
-
|
30
|
-
@buffer[ktopic][partition] ||= Executor.new(
|
27
|
+
@buffer[ktopic][partition][parallel_key] ||= Executor.new(
|
31
28
|
@subscription_group.id,
|
32
29
|
@client,
|
33
|
-
ktopic
|
34
|
-
pause
|
30
|
+
ktopic
|
35
31
|
)
|
36
32
|
end
|
37
33
|
|
34
|
+
# Revokes executors of a given topic partition, so they won't be used anymore for incoming
|
35
|
+
# messages
|
36
|
+
#
|
37
|
+
# @param topic [String] topic name
|
38
|
+
# @param partition [Integer] partition number
|
39
|
+
def revoke(topic, partition)
|
40
|
+
ktopic = find_topic(topic)
|
41
|
+
|
42
|
+
@buffer[ktopic][partition].clear
|
43
|
+
end
|
44
|
+
|
45
|
+
# Finds all the executors available for a given topic partition
|
46
|
+
#
|
47
|
+
# @param topic [String] topic name
|
48
|
+
# @param partition [Integer] partition number
|
49
|
+
# @return [Array<Executor>] executors in use for this topic + partition
|
50
|
+
def find_all(topic, partition)
|
51
|
+
ktopic = find_topic(topic)
|
52
|
+
|
53
|
+
@buffer[ktopic][partition].values
|
54
|
+
end
|
55
|
+
|
38
56
|
# Iterates over all available executors and yields them together with topic and partition
|
39
57
|
# info
|
40
58
|
# @yieldparam [Routing::Topic] karafka routing topic object
|
@@ -42,8 +60,11 @@ module Karafka
|
|
42
60
|
# @yieldparam [Executor] given executor
|
43
61
|
def each
|
44
62
|
@buffer.each do |ktopic, partitions|
|
45
|
-
partitions.each do |partition,
|
46
|
-
|
63
|
+
partitions.each do |partition, executors|
|
64
|
+
executors.each do |_parallel_key, executor|
|
65
|
+
# We skip the parallel key here as it does not serve any value when iterating
|
66
|
+
yield(ktopic, partition, executor)
|
67
|
+
end
|
47
68
|
end
|
48
69
|
end
|
49
70
|
end
|
@@ -52,6 +73,16 @@ module Karafka
|
|
52
73
|
def clear
|
53
74
|
@buffer.clear
|
54
75
|
end
|
76
|
+
|
77
|
+
private
|
78
|
+
|
79
|
+
# Finds topic based on its name
|
80
|
+
#
|
81
|
+
# @param topic [String] topic we're looking for
|
82
|
+
# @return [Karafka::Routing::Topic] topic we're interested in
|
83
|
+
def find_topic(topic)
|
84
|
+
@subscription_group.topics.find(topic) || raise(Errors::TopicNotFoundError, topic)
|
85
|
+
end
|
55
86
|
end
|
56
87
|
end
|
57
88
|
end
|
@@ -5,7 +5,7 @@ module Karafka
|
|
5
5
|
# Namespace for all the jobs that are suppose to run in workers.
|
6
6
|
module Jobs
|
7
7
|
# Base class for all the jobs types that are suppose to run in workers threads.
|
8
|
-
# Each job can have 3 main entry-points: `#
|
8
|
+
# Each job can have 3 main entry-points: `#before_call`, `#call` and `#after_call`
|
9
9
|
# Only `#call` is required.
|
10
10
|
class Base
|
11
11
|
extend Forwardable
|
@@ -23,10 +23,15 @@ module Karafka
|
|
23
23
|
end
|
24
24
|
|
25
25
|
# When redefined can run any code that should run before executing the proper code
|
26
|
-
def
|
26
|
+
def before_call; end
|
27
|
+
|
28
|
+
# The main entry-point of a job
|
29
|
+
def call
|
30
|
+
raise NotImplementedError, 'Please implement in a subclass'
|
31
|
+
end
|
27
32
|
|
28
33
|
# When redefined can run any code that should run after executing the proper code
|
29
|
-
def
|
34
|
+
def after_call; end
|
30
35
|
|
31
36
|
# @return [Boolean] is this a non-blocking job
|
32
37
|
#
|
@@ -12,23 +12,30 @@ module Karafka
|
|
12
12
|
# @param executor [Karafka::Processing::Executor] executor that is suppose to run a given
|
13
13
|
# job
|
14
14
|
# @param messages [Karafka::Messages::Messages] karafka messages batch
|
15
|
+
# @param coordinator [Karafka::Processing::Coordinator] processing coordinator
|
15
16
|
# @return [Consume]
|
16
|
-
def initialize(executor, messages)
|
17
|
+
def initialize(executor, messages, coordinator)
|
17
18
|
@executor = executor
|
18
19
|
@messages = messages
|
20
|
+
@coordinator = coordinator
|
19
21
|
@created_at = Time.now
|
20
22
|
super()
|
21
23
|
end
|
22
24
|
|
23
|
-
# Runs the preparations on the executor
|
24
|
-
def
|
25
|
-
executor.
|
25
|
+
# Runs the before consumption preparations on the executor
|
26
|
+
def before_call
|
27
|
+
executor.before_consume(@messages, @created_at, @coordinator)
|
26
28
|
end
|
27
29
|
|
28
30
|
# Runs the given executor
|
29
31
|
def call
|
30
32
|
executor.consume
|
31
33
|
end
|
34
|
+
|
35
|
+
# Runs any error handling and other post-consumption stuff on the executor
|
36
|
+
def after_call
|
37
|
+
executor.after_consume
|
38
|
+
end
|
32
39
|
end
|
33
40
|
end
|
34
41
|
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Processing
|
5
|
+
# Class responsible for deciding what type of job should we build to run a given command and
|
6
|
+
# for building a proper job for it.
|
7
|
+
class JobsBuilder
|
8
|
+
# @param executor [Karafka::Processing::Executor]
|
9
|
+
# @param messages [Karafka::Messages::Messages] messages batch to be consumed
|
10
|
+
# @param coordinator [Karafka::Processing::Coordinator]
|
11
|
+
# @return [Karafka::Processing::Jobs::Consume] consumption job
|
12
|
+
def consume(executor, messages, coordinator)
|
13
|
+
Jobs::Consume.new(executor, messages, coordinator)
|
14
|
+
end
|
15
|
+
|
16
|
+
# @param executor [Karafka::Processing::Executor]
|
17
|
+
# @return [Karafka::Processing::Jobs::Revoked] revocation job
|
18
|
+
def revoked(executor)
|
19
|
+
Jobs::Revoked.new(executor)
|
20
|
+
end
|
21
|
+
|
22
|
+
# @param executor [Karafka::Processing::Executor]
|
23
|
+
# @return [Karafka::Processing::Jobs::Shutdown] shutdown job
|
24
|
+
def shutdown(executor)
|
25
|
+
Jobs::Shutdown.new(executor)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Processing
|
5
|
+
# A simple object that allows us to keep track of processing state.
|
6
|
+
# It allows to indicate if given thing moved from success to a failure or the other way around
|
7
|
+
# Useful for tracking consumption state
|
8
|
+
class Result
|
9
|
+
def initialize
|
10
|
+
@success = true
|
11
|
+
end
|
12
|
+
|
13
|
+
# @return [Boolean]
|
14
|
+
def success?
|
15
|
+
@success
|
16
|
+
end
|
17
|
+
|
18
|
+
# Marks state as successful
|
19
|
+
def success!
|
20
|
+
@success = true
|
21
|
+
end
|
22
|
+
|
23
|
+
# Marks state as failure
|
24
|
+
def failure!
|
25
|
+
@success = false
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Processing
|
5
|
+
# FIFO scheduler for messages coming from various topics and partitions
|
6
|
+
class Scheduler
|
7
|
+
# Schedules jobs in the fifo order
|
8
|
+
#
|
9
|
+
# @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
|
10
|
+
# @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
|
11
|
+
def schedule_consumption(queue, jobs_array)
|
12
|
+
jobs_array.each do |job|
|
13
|
+
queue << job
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
# Both revocation and shutdown jobs can also run in fifo by default
|
18
|
+
alias schedule_revocation schedule_consumption
|
19
|
+
alias schedule_shutdown schedule_consumption
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -19,9 +19,13 @@ module Karafka
|
|
19
19
|
class Worker
|
20
20
|
include Helpers::Async
|
21
21
|
|
22
|
+
# @return [String] id of this worker
|
23
|
+
attr_reader :id
|
24
|
+
|
22
25
|
# @param jobs_queue [JobsQueue]
|
23
26
|
# @return [Worker]
|
24
27
|
def initialize(jobs_queue)
|
28
|
+
@id = SecureRandom.uuid
|
25
29
|
@jobs_queue = jobs_queue
|
26
30
|
end
|
27
31
|
|
@@ -43,19 +47,23 @@ module Karafka
|
|
43
47
|
job = @jobs_queue.pop
|
44
48
|
|
45
49
|
if job
|
46
|
-
job
|
50
|
+
Karafka.monitor.instrument('worker.process', caller: self, job: job)
|
51
|
+
|
52
|
+
Karafka.monitor.instrument('worker.processed', caller: self, job: job) do
|
53
|
+
job.before_call
|
47
54
|
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
55
|
+
# If a job is marked as non blocking, we can run a tick in the job queue and if there
|
56
|
+
# are no other blocking factors, the job queue will be unlocked.
|
57
|
+
# If this does not run, all the things will be blocking and job queue won't allow to
|
58
|
+
# pass it until done.
|
59
|
+
@jobs_queue.tick(job.group_id) if job.non_blocking?
|
53
60
|
|
54
|
-
|
61
|
+
job.call
|
55
62
|
|
56
|
-
|
63
|
+
job.after_call
|
57
64
|
|
58
|
-
|
65
|
+
true
|
66
|
+
end
|
59
67
|
else
|
60
68
|
false
|
61
69
|
end
|
@@ -38,7 +38,7 @@ module Karafka
|
|
38
38
|
# @return [Array<Routing::SubscriptionGroup>] all the subscription groups build based on
|
39
39
|
# the consumer group topics
|
40
40
|
def subscription_groups
|
41
|
-
App.config.internal.subscription_groups_builder.call(topics)
|
41
|
+
App.config.internal.routing.subscription_groups_builder.call(topics)
|
42
42
|
end
|
43
43
|
|
44
44
|
# Hashed version of consumer group that can be used for validation purposes
|
@@ -44,7 +44,7 @@ module Karafka
|
|
44
44
|
kafka[:'auto.offset.reset'] ||= @topics.first.initial_offset
|
45
45
|
# Karafka manages the offsets based on the processing state, thus we do not rely on the
|
46
46
|
# rdkafka offset auto-storing
|
47
|
-
kafka[:'enable.auto.offset.store'] =
|
47
|
+
kafka[:'enable.auto.offset.store'] = false
|
48
48
|
kafka.freeze
|
49
49
|
kafka
|
50
50
|
end
|
@@ -66,6 +66,20 @@ module Karafka
|
|
66
66
|
end
|
67
67
|
end
|
68
68
|
|
69
|
+
# @return [Class] consumer class that we should use
|
70
|
+
# @note This is just an alias to the `#consumer` method. We however want to use it internally
|
71
|
+
# instead of referencing the `#consumer`. We use this to indicate that this method returns
|
72
|
+
# class and not an instance. In the routing we want to keep the `#consumer Consumer`
|
73
|
+
# routing syntax, but for references outside, we should use this one.
|
74
|
+
def consumer_class
|
75
|
+
consumer
|
76
|
+
end
|
77
|
+
|
78
|
+
# @return [Boolean] true if this topic offset is handled by the end user
|
79
|
+
def manual_offset_management?
|
80
|
+
manual_offset_management
|
81
|
+
end
|
82
|
+
|
69
83
|
# @return [Hash] hash with all the topic attributes
|
70
84
|
# @note This is being used when we validate the consumer_group and its topics
|
71
85
|
def to_h
|
data/lib/karafka/setup/config.rb
CHANGED
@@ -83,21 +83,31 @@ module Karafka
|
|
83
83
|
# @see https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
|
84
84
|
setting :kafka, default: {}
|
85
85
|
|
86
|
-
# Namespace for internal settings that should not be modified
|
86
|
+
# Namespace for internal settings that should not be modified directly
|
87
87
|
setting :internal do
|
88
|
-
# option routing_builder [Karafka::Routing::Builder] builder instance
|
89
|
-
setting :routing_builder, default: Routing::Builder.new
|
90
88
|
# option status [Karafka::Status] app status
|
91
89
|
setting :status, default: Status.new
|
92
90
|
# option process [Karafka::Process] process status
|
93
91
|
# @note In the future, we need to have a single process representation for all the karafka
|
94
92
|
# instances
|
95
93
|
setting :process, default: Process.new
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
94
|
+
|
95
|
+
setting :routing do
|
96
|
+
# option builder [Karafka::Routing::Builder] builder instance
|
97
|
+
setting :builder, default: Routing::Builder.new
|
98
|
+
# option subscription_groups_builder [Routing::SubscriptionGroupsBuilder] subscription
|
99
|
+
# group builder
|
100
|
+
setting :subscription_groups_builder, default: Routing::SubscriptionGroupsBuilder.new
|
101
|
+
end
|
102
|
+
|
103
|
+
setting :processing do
|
104
|
+
# option scheduler [Object] scheduler we will be using
|
105
|
+
setting :scheduler, default: Processing::Scheduler.new
|
106
|
+
# option jobs_builder [Object] jobs builder we want to use
|
107
|
+
setting :jobs_builder, default: Processing::JobsBuilder.new
|
108
|
+
# option coordinator [Class] work coordinator we want to user for processing coordination
|
109
|
+
setting :coordinator_class, default: Processing::Coordinator
|
110
|
+
end
|
101
111
|
|
102
112
|
# Karafka components for ActiveJob
|
103
113
|
setting :active_job do
|
@@ -107,7 +117,7 @@ module Karafka
|
|
107
117
|
# ensuring, that extra job options defined are valid
|
108
118
|
setting :job_options_contract, default: ActiveJob::JobOptionsContract.new
|
109
119
|
# option consumer [Class] consumer class that should be used to consume ActiveJob data
|
110
|
-
setting :
|
120
|
+
setting :consumer_class, default: ActiveJob::Consumer
|
111
121
|
end
|
112
122
|
end
|
113
123
|
|
data/lib/karafka/status.rb
CHANGED
@@ -31,9 +31,7 @@ module Karafka
|
|
31
31
|
# We skip as during this state we do not have yet a monitor
|
32
32
|
return if initializing?
|
33
33
|
|
34
|
-
|
35
|
-
# so the state changes are executed from a separate thread
|
36
|
-
Thread.new { Karafka.monitor.instrument("app.#{state}") }.join
|
34
|
+
Karafka.monitor.instrument("app.#{state}")
|
37
35
|
end
|
38
36
|
end
|
39
37
|
end
|
data/lib/karafka/version.rb
CHANGED
data.tar.gz.sig
CHANGED
Binary file
|