karafka 2.0.0.beta1 → 2.0.0.beta2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +13 -0
- data/Gemfile.lock +1 -1
- data/config/errors.yml +1 -0
- data/lib/active_job/karafka.rb +2 -2
- data/lib/karafka/active_job/routing/extensions.rb +21 -0
- data/lib/karafka/base_consumer.rb +1 -1
- data/lib/karafka/connection/client.rb +1 -1
- data/lib/karafka/connection/listener.rb +88 -27
- data/lib/karafka/connection/listeners_batch.rb +24 -0
- data/lib/karafka/connection/messages_buffer.rb +50 -54
- data/lib/karafka/connection/raw_messages_buffer.rb +101 -0
- data/lib/karafka/contracts/config.rb +7 -0
- data/lib/karafka/helpers/async.rb +33 -0
- data/lib/karafka/messages/batch_metadata.rb +26 -3
- data/lib/karafka/messages/builders/batch_metadata.rb +17 -29
- data/lib/karafka/messages/builders/message.rb +1 -0
- data/lib/karafka/messages/builders/messages.rb +4 -12
- data/lib/karafka/pro/active_job/consumer.rb +21 -0
- data/lib/karafka/pro/active_job/dispatcher.rb +1 -1
- data/lib/karafka/pro/loader.rb +5 -1
- data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb +38 -0
- data/lib/karafka/pro/scheduler.rb +54 -0
- data/lib/karafka/processing/executor.rb +5 -2
- data/lib/karafka/processing/executors_buffer.rb +15 -7
- data/lib/karafka/processing/jobs/base.rb +13 -1
- data/lib/karafka/processing/jobs/consume.rb +4 -2
- data/lib/karafka/processing/jobs_queue.rb +15 -12
- data/lib/karafka/processing/worker.rb +7 -9
- data/lib/karafka/processing/workers_batch.rb +5 -0
- data/lib/karafka/routing/consumer_group.rb +1 -1
- data/lib/karafka/routing/subscription_group.rb +1 -1
- data/lib/karafka/routing/subscription_groups_builder.rb +3 -2
- data/lib/karafka/routing/topics.rb +38 -0
- data/lib/karafka/runner.rb +19 -27
- data/lib/karafka/scheduler.rb +10 -11
- data/lib/karafka/server.rb +24 -23
- data/lib/karafka/setup/config.rb +1 -0
- data/lib/karafka/version.rb +1 -1
- data.tar.gz.sig +1 -3
- metadata +10 -3
- metadata.gz.sig +0 -0
- data/lib/karafka/active_job/routing_extensions.rb +0 -18
data/lib/karafka/server.rb
CHANGED
@@ -15,7 +15,7 @@ module Karafka
|
|
15
15
|
|
16
16
|
class << self
|
17
17
|
# Set of consuming threads. Each consumer thread contains a single consumer
|
18
|
-
attr_accessor :
|
18
|
+
attr_accessor :listeners
|
19
19
|
|
20
20
|
# Set of workers
|
21
21
|
attr_accessor :workers
|
@@ -25,9 +25,12 @@ module Karafka
|
|
25
25
|
|
26
26
|
# Method which runs app
|
27
27
|
def run
|
28
|
-
|
29
|
-
|
30
|
-
|
28
|
+
# Since we do a lot of threading and queuing, we don't want to stop from the trap context
|
29
|
+
# as some things may not work there as expected, that is why we spawn a separate thread to
|
30
|
+
# handle the stopping process
|
31
|
+
process.on_sigint { Thread.new { stop } }
|
32
|
+
process.on_sigquit { Thread.new { stop } }
|
33
|
+
process.on_sigterm { Thread.new { stop } }
|
31
34
|
|
32
35
|
# Start is blocking until stop is called and when we stop, it will wait until
|
33
36
|
# all of the things are ready to stop
|
@@ -35,6 +38,8 @@ module Karafka
|
|
35
38
|
|
36
39
|
# We always need to wait for Karafka to stop here since we should wait for the stop running
|
37
40
|
# in a separate thread (or trap context) to indicate everything is closed
|
41
|
+
# Since `#start` is blocking, we were get here only after the runner is done. This will
|
42
|
+
# not add any performance degradation because of that.
|
38
43
|
Thread.pass until Karafka::App.stopped?
|
39
44
|
# Try its best to shutdown underlying components before re-raising
|
40
45
|
# rubocop:disable Lint/RescueException
|
@@ -70,16 +75,16 @@ module Karafka
|
|
70
75
|
def stop
|
71
76
|
Karafka::App.stop!
|
72
77
|
|
73
|
-
timeout =
|
78
|
+
timeout = Karafka::App.config.shutdown_timeout
|
74
79
|
|
75
80
|
# We check from time to time (for the timeout period) if all the threads finished
|
76
81
|
# their work and if so, we can just return and normal shutdown process will take place
|
77
82
|
# We divide it by 1000 because we use time in ms.
|
78
83
|
((timeout / 1_000) * SUPERVISION_CHECK_FACTOR).to_i.times do
|
79
|
-
if
|
84
|
+
if listeners.count(&:alive?).zero? &&
|
80
85
|
workers.count(&:alive?).zero?
|
81
86
|
|
82
|
-
|
87
|
+
Karafka::App.producer.close
|
83
88
|
|
84
89
|
return
|
85
90
|
end
|
@@ -89,22 +94,18 @@ module Karafka
|
|
89
94
|
|
90
95
|
raise Errors::ForcefulShutdownError
|
91
96
|
rescue Errors::ForcefulShutdownError => e
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
Karafka::App.producer.close
|
105
|
-
end
|
106
|
-
|
107
|
-
thread.join
|
97
|
+
Karafka.monitor.instrument(
|
98
|
+
'error.occurred',
|
99
|
+
caller: self,
|
100
|
+
error: e,
|
101
|
+
type: 'app.stopping.error'
|
102
|
+
)
|
103
|
+
|
104
|
+
# We're done waiting, lets kill them!
|
105
|
+
workers.each(&:terminate)
|
106
|
+
listeners.each(&:terminate)
|
107
|
+
|
108
|
+
Karafka::App.producer.close
|
108
109
|
|
109
110
|
# exit! is not within the instrumentation as it would not trigger due to exit
|
110
111
|
Kernel.exit! FORCEFUL_EXIT_CODE
|
data/lib/karafka/setup/config.rb
CHANGED
data/lib/karafka/version.rb
CHANGED
data.tar.gz.sig
CHANGED
@@ -1,3 +1 @@
|
|
1
|
-
|
2
|
-
?��@�Ԕ��DA��+�[cT�"_t�J�M�aT0ʳ��F���U��]���oIP�C������勳������/,��R�NES-�����]R���
|
3
|
-
��k`�gf��=���}�5Jk�y�-f�'��HQ.�C$�L��_B�A#^5ZA��⤋r����K���1)���2���h�
|
1
|
+
S*^�w`��&qU��Lrӵ�w��� �"�q��U�W�QCˊ1a�����>��S��K����F��0���%=�7��Q����x�{��*p��z\�d����� �VjL!;�b!��rzk�68Q�����fp&?8G' .�_h{v�^`�.Y�Bl�C�t��fkJ��EGn�~��\pP����J�����Aߎ���<bC�䲼�f�UEB:�CD]'?�ɡ#E>"��1��mq�ʟl{gp�p��e}�i�Q�ё�O���f�Ao�פ����:��]�C��x�m��=$f7�g.>/��j���?�rG��#lώ��$�Jh��c�no/��#eB%Ғԇ���`,8֙�Ts�9|��O�p�~�P�� �6㌐��
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.0.
|
4
|
+
version: 2.0.0.beta2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Maciej Mensfeld
|
@@ -34,7 +34,7 @@ cert_chain:
|
|
34
34
|
R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
|
35
35
|
pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
|
36
36
|
-----END CERTIFICATE-----
|
37
|
-
date: 2022-
|
37
|
+
date: 2022-06-07 00:00:00.000000000 Z
|
38
38
|
dependencies:
|
39
39
|
- !ruby/object:Gem::Dependency
|
40
40
|
name: dry-configurable
|
@@ -184,7 +184,7 @@ files:
|
|
184
184
|
- lib/karafka/active_job/dispatcher.rb
|
185
185
|
- lib/karafka/active_job/job_extensions.rb
|
186
186
|
- lib/karafka/active_job/job_options_contract.rb
|
187
|
-
- lib/karafka/active_job/
|
187
|
+
- lib/karafka/active_job/routing/extensions.rb
|
188
188
|
- lib/karafka/app.rb
|
189
189
|
- lib/karafka/base_consumer.rb
|
190
190
|
- lib/karafka/cli.rb
|
@@ -195,8 +195,10 @@ files:
|
|
195
195
|
- lib/karafka/cli/server.rb
|
196
196
|
- lib/karafka/connection/client.rb
|
197
197
|
- lib/karafka/connection/listener.rb
|
198
|
+
- lib/karafka/connection/listeners_batch.rb
|
198
199
|
- lib/karafka/connection/messages_buffer.rb
|
199
200
|
- lib/karafka/connection/pauses_manager.rb
|
201
|
+
- lib/karafka/connection/raw_messages_buffer.rb
|
200
202
|
- lib/karafka/connection/rebalance_manager.rb
|
201
203
|
- lib/karafka/contracts.rb
|
202
204
|
- lib/karafka/contracts/base.rb
|
@@ -206,6 +208,7 @@ files:
|
|
206
208
|
- lib/karafka/contracts/server_cli_options.rb
|
207
209
|
- lib/karafka/env.rb
|
208
210
|
- lib/karafka/errors.rb
|
211
|
+
- lib/karafka/helpers/async.rb
|
209
212
|
- lib/karafka/helpers/multi_delegator.rb
|
210
213
|
- lib/karafka/instrumentation.rb
|
211
214
|
- lib/karafka/instrumentation/callbacks/error.rb
|
@@ -225,10 +228,13 @@ files:
|
|
225
228
|
- lib/karafka/messages/seek.rb
|
226
229
|
- lib/karafka/patches/rdkafka/consumer.rb
|
227
230
|
- lib/karafka/pro.rb
|
231
|
+
- lib/karafka/pro/active_job/consumer.rb
|
228
232
|
- lib/karafka/pro/active_job/dispatcher.rb
|
229
233
|
- lib/karafka/pro/active_job/job_options_contract.rb
|
230
234
|
- lib/karafka/pro/loader.rb
|
231
235
|
- lib/karafka/pro/performance_tracker.rb
|
236
|
+
- lib/karafka/pro/processing/jobs/consume_non_blocking.rb
|
237
|
+
- lib/karafka/pro/scheduler.rb
|
232
238
|
- lib/karafka/process.rb
|
233
239
|
- lib/karafka/processing/executor.rb
|
234
240
|
- lib/karafka/processing/executors_buffer.rb
|
@@ -248,6 +254,7 @@ files:
|
|
248
254
|
- lib/karafka/routing/subscription_group.rb
|
249
255
|
- lib/karafka/routing/subscription_groups_builder.rb
|
250
256
|
- lib/karafka/routing/topic.rb
|
257
|
+
- lib/karafka/routing/topics.rb
|
251
258
|
- lib/karafka/runner.rb
|
252
259
|
- lib/karafka/scheduler.rb
|
253
260
|
- lib/karafka/serialization/json/deserializer.rb
|
metadata.gz.sig
CHANGED
Binary file
|
@@ -1,18 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Karafka
|
4
|
-
# ActiveJob related Karafka stuff
|
5
|
-
module ActiveJob
|
6
|
-
# Routing extensions for ActiveJob
|
7
|
-
module RoutingExtensions
|
8
|
-
# This method simplifies routes definition for ActiveJob topics / queues by auto-injecting
|
9
|
-
# the consumer class
|
10
|
-
# @param name [String, Symbol] name of the topic where ActiveJobs jobs should go
|
11
|
-
def active_job_topic(name)
|
12
|
-
topic(name) do
|
13
|
-
consumer App.config.internal.active_job.consumer
|
14
|
-
end
|
15
|
-
end
|
16
|
-
end
|
17
|
-
end
|
18
|
-
end
|