karafka 2.0.0.beta4 → 2.0.0.beta5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/.github/workflows/ci.yml +18 -1
  4. data/CHANGELOG.md +15 -0
  5. data/Gemfile.lock +1 -1
  6. data/bin/benchmarks +2 -2
  7. data/bin/integrations +10 -3
  8. data/bin/{stress → stress_many} +0 -0
  9. data/bin/stress_one +13 -0
  10. data/docker-compose.yml +23 -18
  11. data/lib/karafka/active_job/routing/extensions.rb +1 -1
  12. data/lib/karafka/app.rb +2 -1
  13. data/lib/karafka/base_consumer.rb +26 -19
  14. data/lib/karafka/connection/client.rb +24 -4
  15. data/lib/karafka/connection/listener.rb +49 -11
  16. data/lib/karafka/connection/pauses_manager.rb +8 -0
  17. data/lib/karafka/connection/rebalance_manager.rb +20 -19
  18. data/lib/karafka/contracts/config.rb +17 -4
  19. data/lib/karafka/contracts/server_cli_options.rb +1 -1
  20. data/lib/karafka/errors.rb +3 -0
  21. data/lib/karafka/pro/active_job/consumer.rb +1 -8
  22. data/lib/karafka/pro/base_consumer.rb +10 -13
  23. data/lib/karafka/pro/loader.rb +11 -6
  24. data/lib/karafka/pro/processing/coordinator.rb +12 -0
  25. data/lib/karafka/pro/processing/jobs_builder.rb +3 -2
  26. data/lib/karafka/pro/processing/scheduler.rb +56 -0
  27. data/lib/karafka/processing/coordinator.rb +84 -0
  28. data/lib/karafka/processing/coordinators_buffer.rb +58 -0
  29. data/lib/karafka/processing/executor.rb +6 -16
  30. data/lib/karafka/processing/executors_buffer.rb +46 -15
  31. data/lib/karafka/processing/jobs/consume.rb +4 -2
  32. data/lib/karafka/processing/jobs_builder.rb +3 -2
  33. data/lib/karafka/processing/result.rb +0 -5
  34. data/lib/karafka/processing/scheduler.rb +22 -0
  35. data/lib/karafka/routing/consumer_group.rb +1 -1
  36. data/lib/karafka/routing/topic.rb +9 -0
  37. data/lib/karafka/setup/config.rb +18 -10
  38. data/lib/karafka/version.rb +1 -1
  39. data.tar.gz.sig +0 -0
  40. metadata +9 -5
  41. metadata.gz.sig +4 -1
  42. data/lib/karafka/pro/scheduler.rb +0 -54
  43. data/lib/karafka/scheduler.rb +0 -20
@@ -85,21 +85,29 @@ module Karafka
85
85
 
86
86
  # Namespace for internal settings that should not be modified directly
87
87
  setting :internal do
88
- # option routing_builder [Karafka::Routing::Builder] builder instance
89
- setting :routing_builder, default: Routing::Builder.new
90
88
  # option status [Karafka::Status] app status
91
89
  setting :status, default: Status.new
92
90
  # option process [Karafka::Process] process status
93
91
  # @note In the future, we need to have a single process representation for all the karafka
94
92
  # instances
95
93
  setting :process, default: Process.new
96
- # option subscription_groups_builder [Routing::SubscriptionGroupsBuilder] subscription
97
- # group builder
98
- setting :subscription_groups_builder, default: Routing::SubscriptionGroupsBuilder.new
99
- # option scheduler [Class] scheduler we will be using
100
- setting :scheduler, default: Scheduler.new
101
- # option jobs_builder [Class] jobs builder we want to use
102
- setting :jobs_builder, default: Processing::JobsBuilder.new
94
+
95
+ setting :routing do
96
+ # option builder [Karafka::Routing::Builder] builder instance
97
+ setting :builder, default: Routing::Builder.new
98
+ # option subscription_groups_builder [Routing::SubscriptionGroupsBuilder] subscription
99
+ # group builder
100
+ setting :subscription_groups_builder, default: Routing::SubscriptionGroupsBuilder.new
101
+ end
102
+
103
+ setting :processing do
104
+ # option scheduler [Object] scheduler we will be using
105
+ setting :scheduler, default: Processing::Scheduler.new
106
+ # option jobs_builder [Object] jobs builder we want to use
107
+ setting :jobs_builder, default: Processing::JobsBuilder.new
108
+ # option coordinator [Class] work coordinator we want to user for processing coordination
109
+ setting :coordinator_class, default: Processing::Coordinator
110
+ end
103
111
 
104
112
  # Karafka components for ActiveJob
105
113
  setting :active_job do
@@ -109,7 +117,7 @@ module Karafka
109
117
  # ensuring, that extra job options defined are valid
110
118
  setting :job_options_contract, default: ActiveJob::JobOptionsContract.new
111
119
  # option consumer [Class] consumer class that should be used to consume ActiveJob data
112
- setting :consumer, default: ActiveJob::Consumer
120
+ setting :consumer_class, default: ActiveJob::Consumer
113
121
  end
114
122
  end
115
123
 
@@ -3,5 +3,5 @@
3
3
  # Main module namespace
4
4
  module Karafka
5
5
  # Current Karafka version
6
- VERSION = '2.0.0.beta4'
6
+ VERSION = '2.0.0.beta5'
7
7
  end
data.tar.gz.sig CHANGED
Binary file
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: karafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.0.beta4
4
+ version: 2.0.0.beta5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Maciej Mensfeld
@@ -34,7 +34,7 @@ cert_chain:
34
34
  R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
35
35
  pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
36
36
  -----END CERTIFICATE-----
37
- date: 2022-06-20 00:00:00.000000000 Z
37
+ date: 2022-07-05 00:00:00.000000000 Z
38
38
  dependencies:
39
39
  - !ruby/object:Gem::Dependency
40
40
  name: dry-configurable
@@ -172,7 +172,8 @@ files:
172
172
  - bin/integrations
173
173
  - bin/karafka
174
174
  - bin/scenario
175
- - bin/stress
175
+ - bin/stress_many
176
+ - bin/stress_one
176
177
  - bin/wait_for_kafka
177
178
  - certs/karafka-pro.pem
178
179
  - certs/mensfeld.pem
@@ -236,11 +237,14 @@ files:
236
237
  - lib/karafka/pro/base_consumer.rb
237
238
  - lib/karafka/pro/loader.rb
238
239
  - lib/karafka/pro/performance_tracker.rb
240
+ - lib/karafka/pro/processing/coordinator.rb
239
241
  - lib/karafka/pro/processing/jobs/consume_non_blocking.rb
240
242
  - lib/karafka/pro/processing/jobs_builder.rb
243
+ - lib/karafka/pro/processing/scheduler.rb
241
244
  - lib/karafka/pro/routing/extensions.rb
242
- - lib/karafka/pro/scheduler.rb
243
245
  - lib/karafka/process.rb
246
+ - lib/karafka/processing/coordinator.rb
247
+ - lib/karafka/processing/coordinators_buffer.rb
244
248
  - lib/karafka/processing/executor.rb
245
249
  - lib/karafka/processing/executors_buffer.rb
246
250
  - lib/karafka/processing/jobs/base.rb
@@ -250,6 +254,7 @@ files:
250
254
  - lib/karafka/processing/jobs_builder.rb
251
255
  - lib/karafka/processing/jobs_queue.rb
252
256
  - lib/karafka/processing/result.rb
257
+ - lib/karafka/processing/scheduler.rb
253
258
  - lib/karafka/processing/worker.rb
254
259
  - lib/karafka/processing/workers_batch.rb
255
260
  - lib/karafka/railtie.rb
@@ -263,7 +268,6 @@ files:
263
268
  - lib/karafka/routing/topic.rb
264
269
  - lib/karafka/routing/topics.rb
265
270
  - lib/karafka/runner.rb
266
- - lib/karafka/scheduler.rb
267
271
  - lib/karafka/serialization/json/deserializer.rb
268
272
  - lib/karafka/server.rb
269
273
  - lib/karafka/setup/config.rb
metadata.gz.sig CHANGED
@@ -1 +1,4 @@
1
- "k (w��r�&L�.3!E긽w3m46��7�c�>zSL�:w1�g`���(��r�U0�����SϹ�b�ű�A��5M�%�EL-v���bw�Ah�B���>�P{amO�u�$7@��3�c�����ڐ��P��Hu:��͸��K�h�E"��EZ1�)h�i��wk?'�Jճ����w�Uu����}F(5zv��O֯ћ��L�^�J�6����B| ���|ujiG �W/ ��A��;}U�9%��۬�r4�Af�>���LϞ"�����+��E'�N��Y?ӂ����c�Z꺼ڭH}�QĪ�*��|��*���93��BX��쎾c�K�y]�DB�W�Y�])��ߣ�I+k���F�쇔RJ��
1
+ 뙮��F��|-aa�}:�g*Z��b�eM~hv��bX ;���2Խ�a�`/��̊��r!WY�-h΂�Μ����]\t��fذ��M%5N���YDnu6�ի7�O`�rP���<�����#Ӊ�؞0����Db��ȭ�E (��T}��\���2\=�TfΠ
2
+ 
3
+ ��e >�
4
+ z:O��N�1�g�i(Y�%$�w*�[-f��/�
@@ -1,54 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # This Karafka component is a Pro component.
4
- # All of the commercial components are present in the lib/karafka/pro directory of this
5
- # repository and their usage requires commercial license agreement.
6
- #
7
- # Karafka has also commercial-friendly license, commercial support and commercial components.
8
- #
9
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
10
- # your code to Maciej Mensfeld.
11
-
12
- module Karafka
13
- module Pro
14
- # Optimizes scheduler that takes into consideration of execution time needed to process
15
- # messages from given topics partitions. It uses the non-preemptive LJF algorithm
16
- #
17
- # This scheduler is designed to optimize execution times on jobs that perform IO operations as
18
- # when taking IO into consideration, the can achieve optimized parallel processing.
19
- #
20
- # This scheduler can also work with virtual partitions.
21
- #
22
- # Aside from consumption jobs, other jobs do not run often, thus we can leave them with
23
- # default FIFO scheduler from the default Karafka scheduler
24
- class Scheduler < ::Karafka::Scheduler
25
- # Schedules jobs in the LJF order for consumption
26
- #
27
- # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
28
- # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
29
- #
30
- def schedule_consumption(queue, jobs_array)
31
- pt = PerformanceTracker.instance
32
-
33
- ordered = []
34
-
35
- jobs_array.each do |job|
36
- messages = job.messages
37
- message = messages.first
38
-
39
- cost = pt.processing_time_p95(message.topic, message.partition) * messages.size
40
-
41
- ordered << [job, cost]
42
- end
43
-
44
- ordered.sort_by!(&:last)
45
- ordered.reverse!
46
- ordered.map!(&:first)
47
-
48
- ordered.each do |job|
49
- queue << job
50
- end
51
- end
52
- end
53
- end
54
- end
@@ -1,20 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- # FIFO scheduler for messages coming from various topics and partitions
5
- class Scheduler
6
- # Schedules jobs in the fifo order
7
- #
8
- # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
9
- # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
10
- def schedule_consumption(queue, jobs_array)
11
- jobs_array.each do |job|
12
- queue << job
13
- end
14
- end
15
-
16
- # Both revocation and shutdown jobs can also run in fifo by default
17
- alias schedule_revocation schedule_consumption
18
- alias schedule_shutdown schedule_consumption
19
- end
20
- end