karafka 2.0.0.beta1 → 2.0.0.beta4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.github/workflows/ci.yml +9 -23
- data/CHANGELOG.md +47 -0
- data/Gemfile.lock +8 -8
- data/bin/integrations +36 -14
- data/bin/scenario +29 -0
- data/bin/wait_for_kafka +20 -0
- data/config/errors.yml +1 -0
- data/docker-compose.yml +12 -0
- data/karafka.gemspec +2 -2
- data/lib/active_job/karafka.rb +2 -2
- data/lib/karafka/active_job/routing/extensions.rb +31 -0
- data/lib/karafka/base_consumer.rb +65 -42
- data/lib/karafka/connection/client.rb +65 -19
- data/lib/karafka/connection/listener.rb +99 -34
- data/lib/karafka/connection/listeners_batch.rb +24 -0
- data/lib/karafka/connection/messages_buffer.rb +50 -54
- data/lib/karafka/connection/raw_messages_buffer.rb +101 -0
- data/lib/karafka/contracts/config.rb +9 -1
- data/lib/karafka/helpers/async.rb +33 -0
- data/lib/karafka/instrumentation/logger_listener.rb +34 -10
- data/lib/karafka/instrumentation/monitor.rb +3 -1
- data/lib/karafka/licenser.rb +26 -7
- data/lib/karafka/messages/batch_metadata.rb +26 -3
- data/lib/karafka/messages/builders/batch_metadata.rb +17 -29
- data/lib/karafka/messages/builders/message.rb +1 -0
- data/lib/karafka/messages/builders/messages.rb +4 -12
- data/lib/karafka/pro/active_job/consumer.rb +49 -0
- data/lib/karafka/pro/active_job/dispatcher.rb +10 -10
- data/lib/karafka/pro/active_job/job_options_contract.rb +9 -9
- data/lib/karafka/pro/base_consumer.rb +76 -0
- data/lib/karafka/pro/loader.rb +30 -13
- data/lib/karafka/pro/performance_tracker.rb +9 -9
- data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb +37 -0
- data/lib/karafka/pro/processing/jobs_builder.rb +31 -0
- data/lib/karafka/pro/routing/extensions.rb +32 -0
- data/lib/karafka/pro/scheduler.rb +54 -0
- data/lib/karafka/processing/executor.rb +34 -7
- data/lib/karafka/processing/executors_buffer.rb +15 -7
- data/lib/karafka/processing/jobs/base.rb +21 -4
- data/lib/karafka/processing/jobs/consume.rb +12 -5
- data/lib/karafka/processing/jobs_builder.rb +28 -0
- data/lib/karafka/processing/jobs_queue.rb +15 -12
- data/lib/karafka/processing/result.rb +34 -0
- data/lib/karafka/processing/worker.rb +23 -17
- data/lib/karafka/processing/workers_batch.rb +5 -0
- data/lib/karafka/routing/consumer_group.rb +1 -1
- data/lib/karafka/routing/subscription_group.rb +2 -2
- data/lib/karafka/routing/subscription_groups_builder.rb +3 -2
- data/lib/karafka/routing/topic.rb +5 -0
- data/lib/karafka/routing/topics.rb +38 -0
- data/lib/karafka/runner.rb +19 -27
- data/lib/karafka/scheduler.rb +10 -11
- data/lib/karafka/server.rb +24 -23
- data/lib/karafka/setup/config.rb +4 -1
- data/lib/karafka/status.rb +1 -3
- data/lib/karafka/version.rb +1 -1
- data.tar.gz.sig +0 -0
- metadata +20 -5
- metadata.gz.sig +0 -0
- data/lib/karafka/active_job/routing_extensions.rb +0 -18
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Helpers
|
5
|
+
# Allows a given class to run async in a separate thread. Provides also few methods we may
|
6
|
+
# want to use to control the underlying thread
|
7
|
+
#
|
8
|
+
# @note Thread running code needs to manage it's own exceptions. If they leak out, they will
|
9
|
+
# abort thread on exception.
|
10
|
+
module Async
|
11
|
+
class << self
|
12
|
+
# Adds forwardable to redirect thread-based control methods to the underlying thread that
|
13
|
+
# runs the async operations
|
14
|
+
#
|
15
|
+
# @param base [Class] class we're including this module in
|
16
|
+
def included(base)
|
17
|
+
base.extend ::Forwardable
|
18
|
+
|
19
|
+
base.def_delegators :@thread, :join, :terminate, :alive?
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
# Runs the `#call` method in a new thread
|
24
|
+
def async_call
|
25
|
+
@thread = Thread.new do
|
26
|
+
Thread.current.abort_on_exception = true
|
27
|
+
|
28
|
+
call
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -15,16 +15,43 @@ module Karafka
|
|
15
15
|
|
16
16
|
# Logs each messages fetching attempt
|
17
17
|
#
|
18
|
-
# @param
|
19
|
-
def on_connection_listener_fetch_loop(
|
20
|
-
|
18
|
+
# @param event [Dry::Events::Event] event details including payload
|
19
|
+
def on_connection_listener_fetch_loop(event)
|
20
|
+
listener = event[:caller]
|
21
|
+
info "[#{listener.id}] Polling messages..."
|
21
22
|
end
|
22
23
|
|
23
24
|
# Logs about messages that we've received from Kafka
|
24
25
|
#
|
25
26
|
# @param event [Dry::Events::Event] event details including payload
|
26
27
|
def on_connection_listener_fetch_loop_received(event)
|
27
|
-
|
28
|
+
listener = event[:caller]
|
29
|
+
time = event[:time]
|
30
|
+
messages_count = event[:messages_buffer].size
|
31
|
+
info "[#{listener.id}] Polled #{messages_count} messages in #{time}ms"
|
32
|
+
end
|
33
|
+
|
34
|
+
# Prints info about the fact that a given job has started
|
35
|
+
#
|
36
|
+
# @param event [Dry::Events::Event] event details including payload
|
37
|
+
def on_worker_process(event)
|
38
|
+
job = event[:job]
|
39
|
+
job_type = job.class.to_s.split('::').last
|
40
|
+
consumer = job.executor.topic.consumer
|
41
|
+
topic = job.executor.topic.name
|
42
|
+
info "[#{job.id}] #{job_type} job for #{consumer} on #{topic} started"
|
43
|
+
end
|
44
|
+
|
45
|
+
# Prints info about the fact that a given job has finished
|
46
|
+
#
|
47
|
+
# @param event [Dry::Events::Event] event details including payload
|
48
|
+
def on_worker_processed(event)
|
49
|
+
job = event[:job]
|
50
|
+
time = event[:time]
|
51
|
+
job_type = job.class.to_s.split('::').last
|
52
|
+
consumer = job.executor.topic.consumer
|
53
|
+
topic = job.executor.topic.name
|
54
|
+
info "[#{job.id}] #{job_type} job for #{consumer} on #{topic} finished in #{time}ms"
|
28
55
|
end
|
29
56
|
|
30
57
|
# Logs info about system signals that Karafka received.
|
@@ -52,16 +79,14 @@ module Karafka
|
|
52
79
|
#
|
53
80
|
# @param _event [Dry::Events::Event] event details including payload
|
54
81
|
def on_app_stopping(_event)
|
55
|
-
|
56
|
-
Thread.new { info 'Stopping Karafka server' }
|
82
|
+
info 'Stopping Karafka server'
|
57
83
|
end
|
58
84
|
|
59
85
|
# Logs info that we stopped the Karafka server.
|
60
86
|
#
|
61
87
|
# @param _event [Dry::Events::Event] event details including payload
|
62
88
|
def on_app_stopped(_event)
|
63
|
-
|
64
|
-
Thread.new { info 'Stopped Karafka server' }
|
89
|
+
info 'Stopped Karafka server'
|
65
90
|
end
|
66
91
|
|
67
92
|
# There are many types of errors that can occur in many places, but we provide a single
|
@@ -95,8 +120,7 @@ module Karafka
|
|
95
120
|
fatal "Runner crashed due to an error: #{error}"
|
96
121
|
fatal details
|
97
122
|
when 'app.stopping.error'
|
98
|
-
|
99
|
-
Thread.new { error 'Forceful Karafka server stop' }
|
123
|
+
error 'Forceful Karafka server stop'
|
100
124
|
when 'librdkafka.error'
|
101
125
|
error "librdkafka internal error occurred: #{error}"
|
102
126
|
error details
|
@@ -22,7 +22,6 @@ module Karafka
|
|
22
22
|
app.stopping
|
23
23
|
app.stopped
|
24
24
|
|
25
|
-
consumer.prepared
|
26
25
|
consumer.consumed
|
27
26
|
consumer.revoked
|
28
27
|
consumer.shutdown
|
@@ -33,6 +32,9 @@ module Karafka
|
|
33
32
|
connection.listener.fetch_loop
|
34
33
|
connection.listener.fetch_loop.received
|
35
34
|
|
35
|
+
worker.process
|
36
|
+
worker.processed
|
37
|
+
|
36
38
|
statistics.emitted
|
37
39
|
|
38
40
|
error.occurred
|
data/lib/karafka/licenser.rb
CHANGED
@@ -33,6 +33,8 @@ module Karafka
|
|
33
33
|
|
34
34
|
return if license_config.expires_on > Date.today
|
35
35
|
|
36
|
+
raise_expired_license_token_in_dev(license_config.expires_on)
|
37
|
+
|
36
38
|
notify_if_license_expired(license_config.expires_on)
|
37
39
|
end
|
38
40
|
|
@@ -53,24 +55,41 @@ module Karafka
|
|
53
55
|
)
|
54
56
|
end
|
55
57
|
|
58
|
+
# Raises an error for test and dev environments if running pro with expired license
|
59
|
+
# We never want to cause any non-dev problems and we should never crash anything else than
|
60
|
+
# tests and development envs.
|
61
|
+
#
|
62
|
+
# @param expires_on [Date] when the license expires
|
63
|
+
def raise_expired_license_token_in_dev(expires_on)
|
64
|
+
env = Karafka::App.env
|
65
|
+
|
66
|
+
return unless env.development? || env.test?
|
67
|
+
|
68
|
+
raise Errors::ExpiredLicenseTokenError.new, expired_message(expires_on)
|
69
|
+
end
|
70
|
+
|
56
71
|
# We do not raise an error here as we don't want to cause any problems to someone that runs
|
57
72
|
# Karafka on production. Error message is enough.
|
58
73
|
#
|
59
74
|
# @param expires_on [Date] when the license expires
|
60
75
|
def notify_if_license_expired(expires_on)
|
61
|
-
|
62
|
-
Your license expired on #{expires_on}.
|
63
|
-
Please reach us at contact@karafka.io or visit https://karafka.io to obtain a valid one.
|
64
|
-
MSG
|
65
|
-
|
66
|
-
Karafka.logger.error(message)
|
76
|
+
Karafka.logger.error(expired_message(expires_on))
|
67
77
|
|
68
78
|
Karafka.monitor.instrument(
|
69
79
|
'error.occurred',
|
70
80
|
caller: self,
|
71
|
-
error: Errors::ExpiredLicenseTokenError.new(
|
81
|
+
error: Errors::ExpiredLicenseTokenError.new(expired_message(expires_on)),
|
72
82
|
type: 'licenser.expired'
|
73
83
|
)
|
74
84
|
end
|
85
|
+
|
86
|
+
# @param expires_on [Date] when the license expires
|
87
|
+
# @return [String] expired message
|
88
|
+
def expired_message(expires_on)
|
89
|
+
<<~MSG.tr("\n", ' ')
|
90
|
+
Your license expired on #{expires_on}.
|
91
|
+
Please reach us at contact@karafka.io or visit https://karafka.io to obtain a valid one.
|
92
|
+
MSG
|
93
|
+
end
|
75
94
|
end
|
76
95
|
end
|
@@ -13,10 +13,33 @@ module Karafka
|
|
13
13
|
:deserializer,
|
14
14
|
:partition,
|
15
15
|
:topic,
|
16
|
+
:created_at,
|
16
17
|
:scheduled_at,
|
17
|
-
:
|
18
|
-
:processing_lag,
|
18
|
+
:processed_at,
|
19
19
|
keyword_init: true
|
20
|
-
)
|
20
|
+
) do
|
21
|
+
# This lag describes how long did it take for a message to be consumed from the moment it was
|
22
|
+
# created
|
23
|
+
def consumption_lag
|
24
|
+
time_distance_in_ms(processed_at, created_at)
|
25
|
+
end
|
26
|
+
|
27
|
+
# This lag describes how long did a batch have to wait before it was picked up by one of the
|
28
|
+
# workers
|
29
|
+
def processing_lag
|
30
|
+
time_distance_in_ms(processed_at, scheduled_at)
|
31
|
+
end
|
32
|
+
|
33
|
+
private
|
34
|
+
|
35
|
+
# Computes time distance in between two times in ms
|
36
|
+
#
|
37
|
+
# @param time1 [Time]
|
38
|
+
# @param time2 [Time]
|
39
|
+
# @return [Integer] distance in between two times in ms
|
40
|
+
def time_distance_in_ms(time1, time2)
|
41
|
+
((time1 - time2) * 1_000).round
|
42
|
+
end
|
43
|
+
end
|
21
44
|
end
|
22
45
|
end
|
@@ -8,42 +8,30 @@ module Karafka
|
|
8
8
|
class << self
|
9
9
|
# Creates metadata based on the kafka batch data.
|
10
10
|
#
|
11
|
-
# @param
|
11
|
+
# @param messages [Array<Karafka::Messages::Message>] messages array
|
12
12
|
# @param topic [Karafka::Routing::Topic] topic for which we've fetched the batch
|
13
13
|
# @param scheduled_at [Time] moment when the batch was scheduled for processing
|
14
14
|
# @return [Karafka::Messages::BatchMetadata] batch metadata object
|
15
|
-
#
|
16
|
-
#
|
17
|
-
#
|
18
|
-
def call(
|
19
|
-
now = Time.now
|
20
|
-
|
15
|
+
#
|
16
|
+
# @note We do not set `processed_at` as this needs to be assigned when the batch is
|
17
|
+
# picked up for processing.
|
18
|
+
def call(messages, topic, scheduled_at)
|
21
19
|
Karafka::Messages::BatchMetadata.new(
|
22
|
-
size:
|
23
|
-
first_offset:
|
24
|
-
last_offset:
|
20
|
+
size: messages.count,
|
21
|
+
first_offset: messages.first.offset,
|
22
|
+
last_offset: messages.last.offset,
|
25
23
|
deserializer: topic.deserializer,
|
26
|
-
partition:
|
24
|
+
partition: messages.first.partition,
|
27
25
|
topic: topic.name,
|
26
|
+
# We go with the assumption that the creation of the whole batch is the last message
|
27
|
+
# creation time
|
28
|
+
created_at: messages.last.timestamp,
|
29
|
+
# When this batch was built and scheduled for execution
|
28
30
|
scheduled_at: scheduled_at,
|
29
|
-
#
|
30
|
-
#
|
31
|
-
|
32
|
-
|
33
|
-
# one of the workers
|
34
|
-
processing_lag: time_distance_in_ms(now, scheduled_at)
|
35
|
-
).freeze
|
36
|
-
end
|
37
|
-
|
38
|
-
private
|
39
|
-
|
40
|
-
# Computes time distance in between two times in ms
|
41
|
-
#
|
42
|
-
# @param time1 [Time]
|
43
|
-
# @param time2 [Time]
|
44
|
-
# @return [Integer] distance in between two times in ms
|
45
|
-
def time_distance_in_ms(time1, time2)
|
46
|
-
((time1 - time2) * 1_000).round
|
31
|
+
# We build the batch metadata when we pick up the job in the worker, thus we can use
|
32
|
+
# current time here
|
33
|
+
processed_at: Time.now
|
34
|
+
)
|
47
35
|
end
|
48
36
|
end
|
49
37
|
end
|
@@ -9,27 +9,19 @@ module Karafka
|
|
9
9
|
# Creates messages batch with messages inside based on the incoming messages and the
|
10
10
|
# topic from which it comes.
|
11
11
|
#
|
12
|
-
# @param
|
12
|
+
# @param messages [Array<Karafka::Messages::Message>] karafka messages array
|
13
13
|
# @param topic [Karafka::Routing::Topic] topic for which we're received messages
|
14
14
|
# @param received_at [Time] moment in time when the messages were received
|
15
15
|
# @return [Karafka::Messages::Messages] messages batch object
|
16
|
-
def call(
|
17
|
-
messages_array = kafka_messages.map do |message|
|
18
|
-
Karafka::Messages::Builders::Message.call(
|
19
|
-
message,
|
20
|
-
topic,
|
21
|
-
received_at
|
22
|
-
)
|
23
|
-
end
|
24
|
-
|
16
|
+
def call(messages, topic, received_at)
|
25
17
|
metadata = BatchMetadata.call(
|
26
|
-
|
18
|
+
messages,
|
27
19
|
topic,
|
28
20
|
received_at
|
29
21
|
).freeze
|
30
22
|
|
31
23
|
Karafka::Messages::Messages.new(
|
32
|
-
|
24
|
+
messages,
|
33
25
|
metadata
|
34
26
|
).freeze
|
35
27
|
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# This Karafka component is a Pro component.
|
4
|
+
# All of the commercial components are present in the lib/karafka/pro directory of this
|
5
|
+
# repository and their usage requires commercial license agreement.
|
6
|
+
#
|
7
|
+
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
8
|
+
#
|
9
|
+
# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
|
10
|
+
# your code to Maciej Mensfeld.
|
11
|
+
|
12
|
+
module Karafka
|
13
|
+
module Pro
|
14
|
+
module ActiveJob
|
15
|
+
# Pro ActiveJob consumer that is suppose to handle long-running jobs as well as short
|
16
|
+
# running jobs
|
17
|
+
#
|
18
|
+
# When in LRJ, it will pause a given partition forever and will resume its processing only
|
19
|
+
# when all the jobs are done processing.
|
20
|
+
#
|
21
|
+
# It contains slightly better revocation warranties than the regular blocking consumer as
|
22
|
+
# it can stop processing batch of jobs in the middle after the revocation.
|
23
|
+
class Consumer < Karafka::Pro::BaseConsumer
|
24
|
+
# Runs ActiveJob jobs processing and handles lrj if needed
|
25
|
+
def consume
|
26
|
+
messages.each do |message|
|
27
|
+
# If for any reason we've lost this partition, not worth iterating over new messages
|
28
|
+
# as they are no longer ours
|
29
|
+
return if revoked?
|
30
|
+
break if Karafka::App.stopping?
|
31
|
+
|
32
|
+
::ActiveJob::Base.execute(
|
33
|
+
::ActiveSupport::JSON.decode(message.raw_payload)
|
34
|
+
)
|
35
|
+
|
36
|
+
mark_as_consumed(message)
|
37
|
+
|
38
|
+
# We check it twice as the job may be long running
|
39
|
+
# If marking fails, it also means it got revoked and we can stop consuming
|
40
|
+
return if revoked?
|
41
|
+
|
42
|
+
# Do not process more if we are shutting down
|
43
|
+
break if Karafka::App.stopping?
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
@@ -1,24 +1,24 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
# This Karafka component is a Pro component.
|
4
|
+
# All of the commercial components are present in the lib/karafka/pro directory of this
|
5
|
+
# repository and their usage requires commercial license agreement.
|
6
|
+
#
|
7
|
+
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
8
|
+
#
|
9
|
+
# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
|
10
|
+
# your code to Maciej Mensfeld.
|
11
|
+
|
3
12
|
module Karafka
|
4
13
|
module Pro
|
5
14
|
# Karafka Pro ActiveJob components
|
6
15
|
module ActiveJob
|
7
|
-
# This Karafka component is a Pro component.
|
8
|
-
# All of the commercial components are present in the lib/karafka/pro directory of this
|
9
|
-
# repository and their usage requires commercial license agreement.
|
10
|
-
#
|
11
|
-
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
12
|
-
#
|
13
|
-
# By sending a pull request to the pro components, you are agreeing to transfer the copyright
|
14
|
-
# of your code to Maciej Mensfeld.
|
15
|
-
|
16
16
|
# Pro dispatcher that sends the ActiveJob job to a proper topic based on the queue name
|
17
17
|
# and that allows to inject additional options into the producer, effectively allowing for a
|
18
18
|
# much better and more granular control over the dispatch and consumption process.
|
19
19
|
class Dispatcher < ::Karafka::ActiveJob::Dispatcher
|
20
20
|
# Defaults for dispatching
|
21
|
-
#
|
21
|
+
# They can be updated by using `#karafka_options` on the job
|
22
22
|
DEFAULTS = {
|
23
23
|
dispatch_method: :produce_async,
|
24
24
|
# We don't create a dummy proc based partitioner as we would have to evaluate it with
|
@@ -1,17 +1,17 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
# This Karafka component is a Pro component.
|
4
|
+
# All of the commercial components are present in the lib/karafka/pro directory of this
|
5
|
+
# repository and their usage requires commercial license agreement.
|
6
|
+
#
|
7
|
+
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
8
|
+
#
|
9
|
+
# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
|
10
|
+
# your code to Maciej Mensfeld.
|
11
|
+
|
3
12
|
module Karafka
|
4
13
|
module Pro
|
5
14
|
module ActiveJob
|
6
|
-
# This Karafka component is a Pro component.
|
7
|
-
# All of the commercial components are present in the lib/karafka/pro directory of this
|
8
|
-
# repository and their usage requires commercial license agreement.
|
9
|
-
#
|
10
|
-
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
11
|
-
#
|
12
|
-
# By sending a pull request to the pro components, you are agreeing to transfer the copyright
|
13
|
-
# of your code to Maciej Mensfeld.
|
14
|
-
|
15
15
|
# Contract for validating the options that can be altered with `#karafka_options` per job
|
16
16
|
# class that works with Pro features.
|
17
17
|
class JobOptionsContract < ::Karafka::ActiveJob::JobOptionsContract
|
@@ -0,0 +1,76 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# This Karafka component is a Pro component.
|
4
|
+
# All of the commercial components are present in the lib/karafka/pro directory of this
|
5
|
+
# repository and their usage requires commercial license agreement.
|
6
|
+
#
|
7
|
+
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
8
|
+
#
|
9
|
+
# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
|
10
|
+
# your code to Maciej Mensfeld.
|
11
|
+
|
12
|
+
module Karafka
|
13
|
+
module Pro
|
14
|
+
# Karafka PRO consumer.
|
15
|
+
#
|
16
|
+
# If you use PRO, all your consumers should inherit (indirectly) from it.
|
17
|
+
#
|
18
|
+
# @note In case of using lrj, manual pausing may not be the best idea as resume needs to happen
|
19
|
+
# after each batch is processed.
|
20
|
+
class BaseConsumer < Karafka::BaseConsumer
|
21
|
+
# Pause for tops 31 years
|
22
|
+
MAX_PAUSE_TIME = 1_000_000_000_000
|
23
|
+
|
24
|
+
private_constant :MAX_PAUSE_TIME
|
25
|
+
|
26
|
+
# Pauses processing of a given partition until we're done with the processing
|
27
|
+
# This ensures, that we can easily poll not reaching the `max.poll.interval`
|
28
|
+
def on_before_consume
|
29
|
+
# Pause at the first message in a batch. That way in case of a crash, we will not loose
|
30
|
+
# any messages
|
31
|
+
return unless topic.long_running_job?
|
32
|
+
|
33
|
+
pause(messages.first.offset, MAX_PAUSE_TIME)
|
34
|
+
end
|
35
|
+
|
36
|
+
# Runs extra logic after consumption that is related to handling long running jobs
|
37
|
+
# @note This overwrites the '#on_after_consume' from the base consumer
|
38
|
+
def on_after_consume
|
39
|
+
# Nothing to do if we lost the partition
|
40
|
+
return if revoked?
|
41
|
+
|
42
|
+
if @consumption.success?
|
43
|
+
pause_tracker.reset
|
44
|
+
|
45
|
+
# We use the non-blocking one here. If someone needs the blocking one, can implement it
|
46
|
+
# with manual offset management
|
47
|
+
# Mark as consumed only if manual offset management is not on
|
48
|
+
mark_as_consumed(messages.last) unless topic.manual_offset_management?
|
49
|
+
|
50
|
+
# If this is not a long running job there is nothing for us to do here
|
51
|
+
return unless topic.long_running_job?
|
52
|
+
|
53
|
+
# Once processing is done, we move to the new offset based on commits
|
54
|
+
# Here, in case manual offset management is off, we have the new proper offset of a
|
55
|
+
# first message from another batch from `@seek_offset`. If manual offset management
|
56
|
+
# is on, we move to place where the user indicated it was finished.
|
57
|
+
seek(@seek_offset || messages.first.offset)
|
58
|
+
resume
|
59
|
+
else
|
60
|
+
# If processing failed, we need to pause
|
61
|
+
pause(@seek_offset || messages.first.offset)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
# Marks this consumer revoked state as true
|
66
|
+
# This allows us for things like lrj to finish early as this state may change during lrj
|
67
|
+
# execution
|
68
|
+
def on_revoked
|
69
|
+
# @note This may already be set to true if we tried to commit offsets and failed. In case
|
70
|
+
# like this it will automatically be marked as revoked.
|
71
|
+
@revoked = true
|
72
|
+
super
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
data/lib/karafka/pro/loader.rb
CHANGED
@@ -1,31 +1,48 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
# This Karafka component is a Pro component.
|
4
|
+
# All of the commercial components are present in the lib/karafka/pro directory of this
|
5
|
+
# repository and their usage requires commercial license agreement.
|
6
|
+
#
|
7
|
+
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
8
|
+
#
|
9
|
+
# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
|
10
|
+
# your code to Maciej Mensfeld.
|
11
|
+
|
3
12
|
module Karafka
|
4
13
|
module Pro
|
5
|
-
# This Karafka component is a Pro component.
|
6
|
-
# All of the commercial components are present in the lib/karafka/pro directory of this
|
7
|
-
# repository and their usage requires commercial license agreement.
|
8
|
-
#
|
9
|
-
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
10
|
-
#
|
11
|
-
# By sending a pull request to the pro components, you are agreeing to transfer the copyright
|
12
|
-
# of your code to Maciej Mensfeld.
|
13
|
-
|
14
14
|
# Loader requires and loads all the pro components only when they are needed
|
15
15
|
class Loader
|
16
|
+
# All the pro components that need to be loaded
|
17
|
+
COMPONENTS = %w[
|
18
|
+
base_consumer
|
19
|
+
performance_tracker
|
20
|
+
scheduler
|
21
|
+
processing/jobs/consume_non_blocking
|
22
|
+
processing/jobs_builder
|
23
|
+
routing/extensions
|
24
|
+
active_job/consumer
|
25
|
+
active_job/dispatcher
|
26
|
+
active_job/job_options_contract
|
27
|
+
].freeze
|
28
|
+
|
29
|
+
private_constant :COMPONENTS
|
30
|
+
|
16
31
|
class << self
|
17
32
|
# Loads all the pro components and configures them wherever it is expected
|
18
33
|
# @param config [Dry::Configurable::Config] whole app config that we can alter with pro
|
19
34
|
# components
|
20
35
|
def setup(config)
|
21
|
-
require_relative
|
22
|
-
require_relative 'active_job/dispatcher'
|
23
|
-
require_relative 'active_job/job_options_contract'
|
36
|
+
COMPONENTS.each { |component| require_relative(component) }
|
24
37
|
|
38
|
+
config.internal.scheduler = Scheduler.new
|
39
|
+
config.internal.jobs_builder = Processing::JobsBuilder.new
|
40
|
+
config.internal.active_job.consumer = ActiveJob::Consumer
|
25
41
|
config.internal.active_job.dispatcher = ActiveJob::Dispatcher.new
|
26
42
|
config.internal.active_job.job_options_contract = ActiveJob::JobOptionsContract.new
|
27
43
|
|
28
|
-
|
44
|
+
::Karafka::Routing::Topic.include(Routing::Extensions)
|
45
|
+
|
29
46
|
config.monitor.subscribe(PerformanceTracker.instance)
|
30
47
|
end
|
31
48
|
end
|
@@ -1,16 +1,16 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
# This Karafka component is a Pro component.
|
4
|
+
# All of the commercial components are present in the lib/karafka/pro directory of this
|
5
|
+
# repository and their usage requires commercial license agreement.
|
6
|
+
#
|
7
|
+
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
8
|
+
#
|
9
|
+
# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
|
10
|
+
# your code to Maciej Mensfeld.
|
11
|
+
|
3
12
|
module Karafka
|
4
13
|
module Pro
|
5
|
-
# This Karafka component is a Pro component.
|
6
|
-
# All of the commercial components are present in the lib/karafka/pro directory of this
|
7
|
-
# repository and their usage requires commercial license agreement.
|
8
|
-
#
|
9
|
-
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
10
|
-
#
|
11
|
-
# By sending a pull request to the pro components, you are agreeing to transfer the copyright
|
12
|
-
# of your code to Maciej Mensfeld.
|
13
|
-
|
14
14
|
# Tracker used to keep track of performance metrics
|
15
15
|
# It provides insights that can be used to optimize processing flow
|
16
16
|
class PerformanceTracker
|
@@ -0,0 +1,37 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# This Karafka component is a Pro component.
|
4
|
+
# All of the commercial components are present in the lib/karafka/pro directory of this
|
5
|
+
# repository and their usage requires commercial license agreement.
|
6
|
+
#
|
7
|
+
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
8
|
+
#
|
9
|
+
# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
|
10
|
+
# your code to Maciej Mensfeld.
|
11
|
+
|
12
|
+
module Karafka
|
13
|
+
module Pro
|
14
|
+
# Pro components related to processing part of Karafka
|
15
|
+
module Processing
|
16
|
+
# Pro jobs
|
17
|
+
module Jobs
|
18
|
+
# The main job type in a non-blocking variant.
|
19
|
+
# This variant works "like" the regular consumption but pauses the partition for as long
|
20
|
+
# as it is needed until a job is done.
|
21
|
+
#
|
22
|
+
# It can be useful when having long lasting jobs that would exceed `max.poll.interval`
|
23
|
+
# if would block.
|
24
|
+
#
|
25
|
+
# @note It needs to be working with a proper consumer that will handle the partition
|
26
|
+
# management. This layer of the framework knows nothing about Kafka messages consumption.
|
27
|
+
class ConsumeNonBlocking < ::Karafka::Processing::Jobs::Consume
|
28
|
+
# Releases the blocking lock after it is done with the preparation phase for this job
|
29
|
+
def before_call
|
30
|
+
super
|
31
|
+
@non_blocking = true
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|