karafka 2.0.0.beta1 → 2.0.0.beta2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +13 -0
  4. data/Gemfile.lock +1 -1
  5. data/config/errors.yml +1 -0
  6. data/lib/active_job/karafka.rb +2 -2
  7. data/lib/karafka/active_job/routing/extensions.rb +21 -0
  8. data/lib/karafka/base_consumer.rb +1 -1
  9. data/lib/karafka/connection/client.rb +1 -1
  10. data/lib/karafka/connection/listener.rb +88 -27
  11. data/lib/karafka/connection/listeners_batch.rb +24 -0
  12. data/lib/karafka/connection/messages_buffer.rb +50 -54
  13. data/lib/karafka/connection/raw_messages_buffer.rb +101 -0
  14. data/lib/karafka/contracts/config.rb +7 -0
  15. data/lib/karafka/helpers/async.rb +33 -0
  16. data/lib/karafka/messages/batch_metadata.rb +26 -3
  17. data/lib/karafka/messages/builders/batch_metadata.rb +17 -29
  18. data/lib/karafka/messages/builders/message.rb +1 -0
  19. data/lib/karafka/messages/builders/messages.rb +4 -12
  20. data/lib/karafka/pro/active_job/consumer.rb +21 -0
  21. data/lib/karafka/pro/active_job/dispatcher.rb +1 -1
  22. data/lib/karafka/pro/loader.rb +5 -1
  23. data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb +38 -0
  24. data/lib/karafka/pro/scheduler.rb +54 -0
  25. data/lib/karafka/processing/executor.rb +5 -2
  26. data/lib/karafka/processing/executors_buffer.rb +15 -7
  27. data/lib/karafka/processing/jobs/base.rb +13 -1
  28. data/lib/karafka/processing/jobs/consume.rb +4 -2
  29. data/lib/karafka/processing/jobs_queue.rb +15 -12
  30. data/lib/karafka/processing/worker.rb +7 -9
  31. data/lib/karafka/processing/workers_batch.rb +5 -0
  32. data/lib/karafka/routing/consumer_group.rb +1 -1
  33. data/lib/karafka/routing/subscription_group.rb +1 -1
  34. data/lib/karafka/routing/subscription_groups_builder.rb +3 -2
  35. data/lib/karafka/routing/topics.rb +38 -0
  36. data/lib/karafka/runner.rb +19 -27
  37. data/lib/karafka/scheduler.rb +10 -11
  38. data/lib/karafka/server.rb +24 -23
  39. data/lib/karafka/setup/config.rb +1 -0
  40. data/lib/karafka/version.rb +1 -1
  41. data.tar.gz.sig +1 -3
  42. metadata +10 -3
  43. metadata.gz.sig +0 -0
  44. data/lib/karafka/active_job/routing_extensions.rb +0 -18
@@ -15,7 +15,7 @@ module Karafka
15
15
 
16
16
  class << self
17
17
  # Set of consuming threads. Each consumer thread contains a single consumer
18
- attr_accessor :consumer_threads
18
+ attr_accessor :listeners
19
19
 
20
20
  # Set of workers
21
21
  attr_accessor :workers
@@ -25,9 +25,12 @@ module Karafka
25
25
 
26
26
  # Method which runs app
27
27
  def run
28
- process.on_sigint { stop }
29
- process.on_sigquit { stop }
30
- process.on_sigterm { stop }
28
+ # Since we do a lot of threading and queuing, we don't want to stop from the trap context
29
+ # as some things may not work there as expected, that is why we spawn a separate thread to
30
+ # handle the stopping process
31
+ process.on_sigint { Thread.new { stop } }
32
+ process.on_sigquit { Thread.new { stop } }
33
+ process.on_sigterm { Thread.new { stop } }
31
34
 
32
35
  # Start is blocking until stop is called and when we stop, it will wait until
33
36
  # all of the things are ready to stop
@@ -35,6 +38,8 @@ module Karafka
35
38
 
36
39
  # We always need to wait for Karafka to stop here since we should wait for the stop running
37
40
  # in a separate thread (or trap context) to indicate everything is closed
41
+ # Since `#start` is blocking, we were get here only after the runner is done. This will
42
+ # not add any performance degradation because of that.
38
43
  Thread.pass until Karafka::App.stopped?
39
44
  # Try its best to shutdown underlying components before re-raising
40
45
  # rubocop:disable Lint/RescueException
@@ -70,16 +75,16 @@ module Karafka
70
75
  def stop
71
76
  Karafka::App.stop!
72
77
 
73
- timeout = Thread.new { Karafka::App.config.shutdown_timeout }.join.value
78
+ timeout = Karafka::App.config.shutdown_timeout
74
79
 
75
80
  # We check from time to time (for the timeout period) if all the threads finished
76
81
  # their work and if so, we can just return and normal shutdown process will take place
77
82
  # We divide it by 1000 because we use time in ms.
78
83
  ((timeout / 1_000) * SUPERVISION_CHECK_FACTOR).to_i.times do
79
- if consumer_threads.count(&:alive?).zero? &&
84
+ if listeners.count(&:alive?).zero? &&
80
85
  workers.count(&:alive?).zero?
81
86
 
82
- Thread.new { Karafka::App.producer.close }.join
87
+ Karafka::App.producer.close
83
88
 
84
89
  return
85
90
  end
@@ -89,22 +94,18 @@ module Karafka
89
94
 
90
95
  raise Errors::ForcefulShutdownError
91
96
  rescue Errors::ForcefulShutdownError => e
92
- thread = Thread.new do
93
- Karafka.monitor.instrument(
94
- 'error.occurred',
95
- caller: self,
96
- error: e,
97
- type: 'app.stopping.error'
98
- )
99
-
100
- # We're done waiting, lets kill them!
101
- workers.each(&:terminate)
102
- consumer_threads.each(&:terminate)
103
-
104
- Karafka::App.producer.close
105
- end
106
-
107
- thread.join
97
+ Karafka.monitor.instrument(
98
+ 'error.occurred',
99
+ caller: self,
100
+ error: e,
101
+ type: 'app.stopping.error'
102
+ )
103
+
104
+ # We're done waiting, lets kill them!
105
+ workers.each(&:terminate)
106
+ listeners.each(&:terminate)
107
+
108
+ Karafka::App.producer.close
108
109
 
109
110
  # exit! is not within the instrumentation as it would not trigger due to exit
110
111
  Kernel.exit! FORCEFUL_EXIT_CODE
@@ -117,6 +117,7 @@ module Karafka
117
117
  def setup(&block)
118
118
  configure(&block)
119
119
  merge_kafka_defaults!(config)
120
+
120
121
  Contracts::Config.new.validate!(config.to_h)
121
122
 
122
123
  # Check the license presence (if needed) and
@@ -3,5 +3,5 @@
3
3
  # Main module namespace
4
4
  module Karafka
5
5
  # Current Karafka version
6
- VERSION = '2.0.0.beta1'
6
+ VERSION = '2.0.0.beta2'
7
7
  end
data.tar.gz.sig CHANGED
@@ -1,3 +1 @@
1
- Ό��vڹ��4�nS��[U+,��u�6bC*@+���[�:$֒؞K�kw��(gl��G K����=Iҳ�N2�OAoHP����Pmt�@�-$��ͧ {�Ӷ\ gB����@��A�w 6j�agL�=;;84����$@{��BS�%�+��o��Z*���*��Оנ1Ie���ZQOL�Ӆ��bx`
2
- ?��@�Ԕ��DA��+�[cT�"_t�J�M�aT0ʳ��F���U��]���oIP�C������勳������/,��R�NES-�����]R���
3
- ��k`�gf��=���}�5Jk�y�-f�'��HQ.�C$�L��_B�A#^5ZA��⤋r����K���1)���2���h�
1
+ S*^�w` ��&qU��Lrӵ�w޺��� �"�q��U�W�QCˊ1a�����>��S��K����F��0���%=�7��Q����x{��*p��z\�d����� VjL!;�b!��rzk68Q�����fp&?8G' .�_h{v�^`�.YBl�C�t��fkJ��EGn�~��\pP����J�����Aߎ���<bC�䲼�fUEB:�CD]'?�ɡ#E>"��1��mq�ʟl{gpp��e}i�Q�ё�O���f�Ao�פ����:��]�C��x�m��=$f7�g.>/��j���?�rG��#lώ��$�Jh��cno/��#eB%Ғԇ���`,8֙�Ts�9|��O�p�~�P�� �6㌐��
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: karafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.0.beta1
4
+ version: 2.0.0.beta2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Maciej Mensfeld
@@ -34,7 +34,7 @@ cert_chain:
34
34
  R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
35
35
  pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
36
36
  -----END CERTIFICATE-----
37
- date: 2022-05-22 00:00:00.000000000 Z
37
+ date: 2022-06-07 00:00:00.000000000 Z
38
38
  dependencies:
39
39
  - !ruby/object:Gem::Dependency
40
40
  name: dry-configurable
@@ -184,7 +184,7 @@ files:
184
184
  - lib/karafka/active_job/dispatcher.rb
185
185
  - lib/karafka/active_job/job_extensions.rb
186
186
  - lib/karafka/active_job/job_options_contract.rb
187
- - lib/karafka/active_job/routing_extensions.rb
187
+ - lib/karafka/active_job/routing/extensions.rb
188
188
  - lib/karafka/app.rb
189
189
  - lib/karafka/base_consumer.rb
190
190
  - lib/karafka/cli.rb
@@ -195,8 +195,10 @@ files:
195
195
  - lib/karafka/cli/server.rb
196
196
  - lib/karafka/connection/client.rb
197
197
  - lib/karafka/connection/listener.rb
198
+ - lib/karafka/connection/listeners_batch.rb
198
199
  - lib/karafka/connection/messages_buffer.rb
199
200
  - lib/karafka/connection/pauses_manager.rb
201
+ - lib/karafka/connection/raw_messages_buffer.rb
200
202
  - lib/karafka/connection/rebalance_manager.rb
201
203
  - lib/karafka/contracts.rb
202
204
  - lib/karafka/contracts/base.rb
@@ -206,6 +208,7 @@ files:
206
208
  - lib/karafka/contracts/server_cli_options.rb
207
209
  - lib/karafka/env.rb
208
210
  - lib/karafka/errors.rb
211
+ - lib/karafka/helpers/async.rb
209
212
  - lib/karafka/helpers/multi_delegator.rb
210
213
  - lib/karafka/instrumentation.rb
211
214
  - lib/karafka/instrumentation/callbacks/error.rb
@@ -225,10 +228,13 @@ files:
225
228
  - lib/karafka/messages/seek.rb
226
229
  - lib/karafka/patches/rdkafka/consumer.rb
227
230
  - lib/karafka/pro.rb
231
+ - lib/karafka/pro/active_job/consumer.rb
228
232
  - lib/karafka/pro/active_job/dispatcher.rb
229
233
  - lib/karafka/pro/active_job/job_options_contract.rb
230
234
  - lib/karafka/pro/loader.rb
231
235
  - lib/karafka/pro/performance_tracker.rb
236
+ - lib/karafka/pro/processing/jobs/consume_non_blocking.rb
237
+ - lib/karafka/pro/scheduler.rb
232
238
  - lib/karafka/process.rb
233
239
  - lib/karafka/processing/executor.rb
234
240
  - lib/karafka/processing/executors_buffer.rb
@@ -248,6 +254,7 @@ files:
248
254
  - lib/karafka/routing/subscription_group.rb
249
255
  - lib/karafka/routing/subscription_groups_builder.rb
250
256
  - lib/karafka/routing/topic.rb
257
+ - lib/karafka/routing/topics.rb
251
258
  - lib/karafka/runner.rb
252
259
  - lib/karafka/scheduler.rb
253
260
  - lib/karafka/serialization/json/deserializer.rb
metadata.gz.sig CHANGED
Binary file
@@ -1,18 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- # ActiveJob related Karafka stuff
5
- module ActiveJob
6
- # Routing extensions for ActiveJob
7
- module RoutingExtensions
8
- # This method simplifies routes definition for ActiveJob topics / queues by auto-injecting
9
- # the consumer class
10
- # @param name [String, Symbol] name of the topic where ActiveJobs jobs should go
11
- def active_job_topic(name)
12
- topic(name) do
13
- consumer App.config.internal.active_job.consumer
14
- end
15
- end
16
- end
17
- end
18
- end