karafka 2.0.0.beta1 → 2.0.0.beta2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +13 -0
- data/Gemfile.lock +1 -1
- data/config/errors.yml +1 -0
- data/lib/active_job/karafka.rb +2 -2
- data/lib/karafka/active_job/routing/extensions.rb +21 -0
- data/lib/karafka/base_consumer.rb +1 -1
- data/lib/karafka/connection/client.rb +1 -1
- data/lib/karafka/connection/listener.rb +88 -27
- data/lib/karafka/connection/listeners_batch.rb +24 -0
- data/lib/karafka/connection/messages_buffer.rb +50 -54
- data/lib/karafka/connection/raw_messages_buffer.rb +101 -0
- data/lib/karafka/contracts/config.rb +7 -0
- data/lib/karafka/helpers/async.rb +33 -0
- data/lib/karafka/messages/batch_metadata.rb +26 -3
- data/lib/karafka/messages/builders/batch_metadata.rb +17 -29
- data/lib/karafka/messages/builders/message.rb +1 -0
- data/lib/karafka/messages/builders/messages.rb +4 -12
- data/lib/karafka/pro/active_job/consumer.rb +21 -0
- data/lib/karafka/pro/active_job/dispatcher.rb +1 -1
- data/lib/karafka/pro/loader.rb +5 -1
- data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb +38 -0
- data/lib/karafka/pro/scheduler.rb +54 -0
- data/lib/karafka/processing/executor.rb +5 -2
- data/lib/karafka/processing/executors_buffer.rb +15 -7
- data/lib/karafka/processing/jobs/base.rb +13 -1
- data/lib/karafka/processing/jobs/consume.rb +4 -2
- data/lib/karafka/processing/jobs_queue.rb +15 -12
- data/lib/karafka/processing/worker.rb +7 -9
- data/lib/karafka/processing/workers_batch.rb +5 -0
- data/lib/karafka/routing/consumer_group.rb +1 -1
- data/lib/karafka/routing/subscription_group.rb +1 -1
- data/lib/karafka/routing/subscription_groups_builder.rb +3 -2
- data/lib/karafka/routing/topics.rb +38 -0
- data/lib/karafka/runner.rb +19 -27
- data/lib/karafka/scheduler.rb +10 -11
- data/lib/karafka/server.rb +24 -23
- data/lib/karafka/setup/config.rb +1 -0
- data/lib/karafka/version.rb +1 -1
- data.tar.gz.sig +1 -3
- metadata +10 -3
- metadata.gz.sig +0 -0
- data/lib/karafka/active_job/routing_extensions.rb +0 -18
data/lib/karafka/server.rb
CHANGED
@@ -15,7 +15,7 @@ module Karafka
|
|
15
15
|
|
16
16
|
class << self
|
17
17
|
# Set of consuming threads. Each consumer thread contains a single consumer
|
18
|
-
attr_accessor :
|
18
|
+
attr_accessor :listeners
|
19
19
|
|
20
20
|
# Set of workers
|
21
21
|
attr_accessor :workers
|
@@ -25,9 +25,12 @@ module Karafka
|
|
25
25
|
|
26
26
|
# Method which runs app
|
27
27
|
def run
|
28
|
-
|
29
|
-
|
30
|
-
|
28
|
+
# Since we do a lot of threading and queuing, we don't want to stop from the trap context
|
29
|
+
# as some things may not work there as expected, that is why we spawn a separate thread to
|
30
|
+
# handle the stopping process
|
31
|
+
process.on_sigint { Thread.new { stop } }
|
32
|
+
process.on_sigquit { Thread.new { stop } }
|
33
|
+
process.on_sigterm { Thread.new { stop } }
|
31
34
|
|
32
35
|
# Start is blocking until stop is called and when we stop, it will wait until
|
33
36
|
# all of the things are ready to stop
|
@@ -35,6 +38,8 @@ module Karafka
|
|
35
38
|
|
36
39
|
# We always need to wait for Karafka to stop here since we should wait for the stop running
|
37
40
|
# in a separate thread (or trap context) to indicate everything is closed
|
41
|
+
# Since `#start` is blocking, we were get here only after the runner is done. This will
|
42
|
+
# not add any performance degradation because of that.
|
38
43
|
Thread.pass until Karafka::App.stopped?
|
39
44
|
# Try its best to shutdown underlying components before re-raising
|
40
45
|
# rubocop:disable Lint/RescueException
|
@@ -70,16 +75,16 @@ module Karafka
|
|
70
75
|
def stop
|
71
76
|
Karafka::App.stop!
|
72
77
|
|
73
|
-
timeout =
|
78
|
+
timeout = Karafka::App.config.shutdown_timeout
|
74
79
|
|
75
80
|
# We check from time to time (for the timeout period) if all the threads finished
|
76
81
|
# their work and if so, we can just return and normal shutdown process will take place
|
77
82
|
# We divide it by 1000 because we use time in ms.
|
78
83
|
((timeout / 1_000) * SUPERVISION_CHECK_FACTOR).to_i.times do
|
79
|
-
if
|
84
|
+
if listeners.count(&:alive?).zero? &&
|
80
85
|
workers.count(&:alive?).zero?
|
81
86
|
|
82
|
-
|
87
|
+
Karafka::App.producer.close
|
83
88
|
|
84
89
|
return
|
85
90
|
end
|
@@ -89,22 +94,18 @@ module Karafka
|
|
89
94
|
|
90
95
|
raise Errors::ForcefulShutdownError
|
91
96
|
rescue Errors::ForcefulShutdownError => e
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
Karafka::App.producer.close
|
105
|
-
end
|
106
|
-
|
107
|
-
thread.join
|
97
|
+
Karafka.monitor.instrument(
|
98
|
+
'error.occurred',
|
99
|
+
caller: self,
|
100
|
+
error: e,
|
101
|
+
type: 'app.stopping.error'
|
102
|
+
)
|
103
|
+
|
104
|
+
# We're done waiting, lets kill them!
|
105
|
+
workers.each(&:terminate)
|
106
|
+
listeners.each(&:terminate)
|
107
|
+
|
108
|
+
Karafka::App.producer.close
|
108
109
|
|
109
110
|
# exit! is not within the instrumentation as it would not trigger due to exit
|
110
111
|
Kernel.exit! FORCEFUL_EXIT_CODE
|
data/lib/karafka/setup/config.rb
CHANGED
data/lib/karafka/version.rb
CHANGED
data.tar.gz.sig
CHANGED
@@ -1,3 +1 @@
|
|
1
|
-
|
2
|
-
?��@�Ԕ��DA��+�[cT�"_t�J�M�aT0ʳ��F���U��]���oIP�C������勳������/,��R�NES-�����]R���
|
3
|
-
��k`�gf��=���}�5Jk�y�-f�'��HQ.�C$�L��_B�A#^5ZA��⤋r����K���1)���2���h�
|
1
|
+
S*^�w`��&qU��Lrӵ�w��� �"�q��U�W�QCˊ1a�����>��S��K����F��0���%=�7��Q����x�{��*p��z\�d����� �VjL!;�b!��rzk�68Q�����fp&?8G' .�_h{v�^`�.Y�Bl�C�t��fkJ��EGn�~��\pP����J�����Aߎ���<bC�䲼�f�UEB:�CD]'?�ɡ#E>"��1��mq�ʟl{gp�p��e}�i�Q�ё�O���f�Ao�פ����:��]�C��x�m��=$f7�g.>/��j���?�rG��#lώ��$�Jh��c�no/��#eB%Ғԇ���`,8֙�Ts�9|��O�p�~�P�� �6㌐��
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.0.
|
4
|
+
version: 2.0.0.beta2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Maciej Mensfeld
|
@@ -34,7 +34,7 @@ cert_chain:
|
|
34
34
|
R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
|
35
35
|
pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
|
36
36
|
-----END CERTIFICATE-----
|
37
|
-
date: 2022-
|
37
|
+
date: 2022-06-07 00:00:00.000000000 Z
|
38
38
|
dependencies:
|
39
39
|
- !ruby/object:Gem::Dependency
|
40
40
|
name: dry-configurable
|
@@ -184,7 +184,7 @@ files:
|
|
184
184
|
- lib/karafka/active_job/dispatcher.rb
|
185
185
|
- lib/karafka/active_job/job_extensions.rb
|
186
186
|
- lib/karafka/active_job/job_options_contract.rb
|
187
|
-
- lib/karafka/active_job/
|
187
|
+
- lib/karafka/active_job/routing/extensions.rb
|
188
188
|
- lib/karafka/app.rb
|
189
189
|
- lib/karafka/base_consumer.rb
|
190
190
|
- lib/karafka/cli.rb
|
@@ -195,8 +195,10 @@ files:
|
|
195
195
|
- lib/karafka/cli/server.rb
|
196
196
|
- lib/karafka/connection/client.rb
|
197
197
|
- lib/karafka/connection/listener.rb
|
198
|
+
- lib/karafka/connection/listeners_batch.rb
|
198
199
|
- lib/karafka/connection/messages_buffer.rb
|
199
200
|
- lib/karafka/connection/pauses_manager.rb
|
201
|
+
- lib/karafka/connection/raw_messages_buffer.rb
|
200
202
|
- lib/karafka/connection/rebalance_manager.rb
|
201
203
|
- lib/karafka/contracts.rb
|
202
204
|
- lib/karafka/contracts/base.rb
|
@@ -206,6 +208,7 @@ files:
|
|
206
208
|
- lib/karafka/contracts/server_cli_options.rb
|
207
209
|
- lib/karafka/env.rb
|
208
210
|
- lib/karafka/errors.rb
|
211
|
+
- lib/karafka/helpers/async.rb
|
209
212
|
- lib/karafka/helpers/multi_delegator.rb
|
210
213
|
- lib/karafka/instrumentation.rb
|
211
214
|
- lib/karafka/instrumentation/callbacks/error.rb
|
@@ -225,10 +228,13 @@ files:
|
|
225
228
|
- lib/karafka/messages/seek.rb
|
226
229
|
- lib/karafka/patches/rdkafka/consumer.rb
|
227
230
|
- lib/karafka/pro.rb
|
231
|
+
- lib/karafka/pro/active_job/consumer.rb
|
228
232
|
- lib/karafka/pro/active_job/dispatcher.rb
|
229
233
|
- lib/karafka/pro/active_job/job_options_contract.rb
|
230
234
|
- lib/karafka/pro/loader.rb
|
231
235
|
- lib/karafka/pro/performance_tracker.rb
|
236
|
+
- lib/karafka/pro/processing/jobs/consume_non_blocking.rb
|
237
|
+
- lib/karafka/pro/scheduler.rb
|
232
238
|
- lib/karafka/process.rb
|
233
239
|
- lib/karafka/processing/executor.rb
|
234
240
|
- lib/karafka/processing/executors_buffer.rb
|
@@ -248,6 +254,7 @@ files:
|
|
248
254
|
- lib/karafka/routing/subscription_group.rb
|
249
255
|
- lib/karafka/routing/subscription_groups_builder.rb
|
250
256
|
- lib/karafka/routing/topic.rb
|
257
|
+
- lib/karafka/routing/topics.rb
|
251
258
|
- lib/karafka/runner.rb
|
252
259
|
- lib/karafka/scheduler.rb
|
253
260
|
- lib/karafka/serialization/json/deserializer.rb
|
metadata.gz.sig
CHANGED
Binary file
|
@@ -1,18 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Karafka
|
4
|
-
# ActiveJob related Karafka stuff
|
5
|
-
module ActiveJob
|
6
|
-
# Routing extensions for ActiveJob
|
7
|
-
module RoutingExtensions
|
8
|
-
# This method simplifies routes definition for ActiveJob topics / queues by auto-injecting
|
9
|
-
# the consumer class
|
10
|
-
# @param name [String, Symbol] name of the topic where ActiveJobs jobs should go
|
11
|
-
def active_job_topic(name)
|
12
|
-
topic(name) do
|
13
|
-
consumer App.config.internal.active_job.consumer
|
14
|
-
end
|
15
|
-
end
|
16
|
-
end
|
17
|
-
end
|
18
|
-
end
|