karafka 2.0.0.beta4 → 2.0.0.beta5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.github/workflows/ci.yml +18 -1
- data/CHANGELOG.md +15 -0
- data/Gemfile.lock +1 -1
- data/bin/benchmarks +2 -2
- data/bin/integrations +10 -3
- data/bin/{stress → stress_many} +0 -0
- data/bin/stress_one +13 -0
- data/docker-compose.yml +23 -18
- data/lib/karafka/active_job/routing/extensions.rb +1 -1
- data/lib/karafka/app.rb +2 -1
- data/lib/karafka/base_consumer.rb +26 -19
- data/lib/karafka/connection/client.rb +24 -4
- data/lib/karafka/connection/listener.rb +49 -11
- data/lib/karafka/connection/pauses_manager.rb +8 -0
- data/lib/karafka/connection/rebalance_manager.rb +20 -19
- data/lib/karafka/contracts/config.rb +17 -4
- data/lib/karafka/contracts/server_cli_options.rb +1 -1
- data/lib/karafka/errors.rb +3 -0
- data/lib/karafka/pro/active_job/consumer.rb +1 -8
- data/lib/karafka/pro/base_consumer.rb +10 -13
- data/lib/karafka/pro/loader.rb +11 -6
- data/lib/karafka/pro/processing/coordinator.rb +12 -0
- data/lib/karafka/pro/processing/jobs_builder.rb +3 -2
- data/lib/karafka/pro/processing/scheduler.rb +56 -0
- data/lib/karafka/processing/coordinator.rb +84 -0
- data/lib/karafka/processing/coordinators_buffer.rb +58 -0
- data/lib/karafka/processing/executor.rb +6 -16
- data/lib/karafka/processing/executors_buffer.rb +46 -15
- data/lib/karafka/processing/jobs/consume.rb +4 -2
- data/lib/karafka/processing/jobs_builder.rb +3 -2
- data/lib/karafka/processing/result.rb +0 -5
- data/lib/karafka/processing/scheduler.rb +22 -0
- data/lib/karafka/routing/consumer_group.rb +1 -1
- data/lib/karafka/routing/topic.rb +9 -0
- data/lib/karafka/setup/config.rb +18 -10
- data/lib/karafka/version.rb +1 -1
- data.tar.gz.sig +0 -0
- metadata +9 -5
- metadata.gz.sig +4 -1
- data/lib/karafka/pro/scheduler.rb +0 -54
- data/lib/karafka/scheduler.rb +0 -20
data/lib/karafka/setup/config.rb
CHANGED
@@ -85,21 +85,29 @@ module Karafka
|
|
85
85
|
|
86
86
|
# Namespace for internal settings that should not be modified directly
|
87
87
|
setting :internal do
|
88
|
-
# option routing_builder [Karafka::Routing::Builder] builder instance
|
89
|
-
setting :routing_builder, default: Routing::Builder.new
|
90
88
|
# option status [Karafka::Status] app status
|
91
89
|
setting :status, default: Status.new
|
92
90
|
# option process [Karafka::Process] process status
|
93
91
|
# @note In the future, we need to have a single process representation for all the karafka
|
94
92
|
# instances
|
95
93
|
setting :process, default: Process.new
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
94
|
+
|
95
|
+
setting :routing do
|
96
|
+
# option builder [Karafka::Routing::Builder] builder instance
|
97
|
+
setting :builder, default: Routing::Builder.new
|
98
|
+
# option subscription_groups_builder [Routing::SubscriptionGroupsBuilder] subscription
|
99
|
+
# group builder
|
100
|
+
setting :subscription_groups_builder, default: Routing::SubscriptionGroupsBuilder.new
|
101
|
+
end
|
102
|
+
|
103
|
+
setting :processing do
|
104
|
+
# option scheduler [Object] scheduler we will be using
|
105
|
+
setting :scheduler, default: Processing::Scheduler.new
|
106
|
+
# option jobs_builder [Object] jobs builder we want to use
|
107
|
+
setting :jobs_builder, default: Processing::JobsBuilder.new
|
108
|
+
# option coordinator [Class] work coordinator we want to user for processing coordination
|
109
|
+
setting :coordinator_class, default: Processing::Coordinator
|
110
|
+
end
|
103
111
|
|
104
112
|
# Karafka components for ActiveJob
|
105
113
|
setting :active_job do
|
@@ -109,7 +117,7 @@ module Karafka
|
|
109
117
|
# ensuring, that extra job options defined are valid
|
110
118
|
setting :job_options_contract, default: ActiveJob::JobOptionsContract.new
|
111
119
|
# option consumer [Class] consumer class that should be used to consume ActiveJob data
|
112
|
-
setting :
|
120
|
+
setting :consumer_class, default: ActiveJob::Consumer
|
113
121
|
end
|
114
122
|
end
|
115
123
|
|
data/lib/karafka/version.rb
CHANGED
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.0.
|
4
|
+
version: 2.0.0.beta5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Maciej Mensfeld
|
@@ -34,7 +34,7 @@ cert_chain:
|
|
34
34
|
R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
|
35
35
|
pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
|
36
36
|
-----END CERTIFICATE-----
|
37
|
-
date: 2022-
|
37
|
+
date: 2022-07-05 00:00:00.000000000 Z
|
38
38
|
dependencies:
|
39
39
|
- !ruby/object:Gem::Dependency
|
40
40
|
name: dry-configurable
|
@@ -172,7 +172,8 @@ files:
|
|
172
172
|
- bin/integrations
|
173
173
|
- bin/karafka
|
174
174
|
- bin/scenario
|
175
|
-
- bin/
|
175
|
+
- bin/stress_many
|
176
|
+
- bin/stress_one
|
176
177
|
- bin/wait_for_kafka
|
177
178
|
- certs/karafka-pro.pem
|
178
179
|
- certs/mensfeld.pem
|
@@ -236,11 +237,14 @@ files:
|
|
236
237
|
- lib/karafka/pro/base_consumer.rb
|
237
238
|
- lib/karafka/pro/loader.rb
|
238
239
|
- lib/karafka/pro/performance_tracker.rb
|
240
|
+
- lib/karafka/pro/processing/coordinator.rb
|
239
241
|
- lib/karafka/pro/processing/jobs/consume_non_blocking.rb
|
240
242
|
- lib/karafka/pro/processing/jobs_builder.rb
|
243
|
+
- lib/karafka/pro/processing/scheduler.rb
|
241
244
|
- lib/karafka/pro/routing/extensions.rb
|
242
|
-
- lib/karafka/pro/scheduler.rb
|
243
245
|
- lib/karafka/process.rb
|
246
|
+
- lib/karafka/processing/coordinator.rb
|
247
|
+
- lib/karafka/processing/coordinators_buffer.rb
|
244
248
|
- lib/karafka/processing/executor.rb
|
245
249
|
- lib/karafka/processing/executors_buffer.rb
|
246
250
|
- lib/karafka/processing/jobs/base.rb
|
@@ -250,6 +254,7 @@ files:
|
|
250
254
|
- lib/karafka/processing/jobs_builder.rb
|
251
255
|
- lib/karafka/processing/jobs_queue.rb
|
252
256
|
- lib/karafka/processing/result.rb
|
257
|
+
- lib/karafka/processing/scheduler.rb
|
253
258
|
- lib/karafka/processing/worker.rb
|
254
259
|
- lib/karafka/processing/workers_batch.rb
|
255
260
|
- lib/karafka/railtie.rb
|
@@ -263,7 +268,6 @@ files:
|
|
263
268
|
- lib/karafka/routing/topic.rb
|
264
269
|
- lib/karafka/routing/topics.rb
|
265
270
|
- lib/karafka/runner.rb
|
266
|
-
- lib/karafka/scheduler.rb
|
267
271
|
- lib/karafka/serialization/json/deserializer.rb
|
268
272
|
- lib/karafka/server.rb
|
269
273
|
- lib/karafka/setup/config.rb
|
metadata.gz.sig
CHANGED
@@ -1 +1,4 @@
|
|
1
|
-
|
1
|
+
뙮��F��|-aa�}:�g*Z��b�eM~hv��bX;���2Խ�a�`/��̊��r!WY�-h�Μ����]\t��fذ��M%5N���YDnu6�ի7�O`�rP���<�����#Ӊ�؞0����Db��ȭ�E (��T}��\���2\=�TfΠ
|
2
|
+
|
3
|
+
��e >�
|
4
|
+
z:O��N�1�g�i(Y�%$�w*�[-f��/�
|
@@ -1,54 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
# This Karafka component is a Pro component.
|
4
|
-
# All of the commercial components are present in the lib/karafka/pro directory of this
|
5
|
-
# repository and their usage requires commercial license agreement.
|
6
|
-
#
|
7
|
-
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
8
|
-
#
|
9
|
-
# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
|
10
|
-
# your code to Maciej Mensfeld.
|
11
|
-
|
12
|
-
module Karafka
|
13
|
-
module Pro
|
14
|
-
# Optimizes scheduler that takes into consideration of execution time needed to process
|
15
|
-
# messages from given topics partitions. It uses the non-preemptive LJF algorithm
|
16
|
-
#
|
17
|
-
# This scheduler is designed to optimize execution times on jobs that perform IO operations as
|
18
|
-
# when taking IO into consideration, the can achieve optimized parallel processing.
|
19
|
-
#
|
20
|
-
# This scheduler can also work with virtual partitions.
|
21
|
-
#
|
22
|
-
# Aside from consumption jobs, other jobs do not run often, thus we can leave them with
|
23
|
-
# default FIFO scheduler from the default Karafka scheduler
|
24
|
-
class Scheduler < ::Karafka::Scheduler
|
25
|
-
# Schedules jobs in the LJF order for consumption
|
26
|
-
#
|
27
|
-
# @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
|
28
|
-
# @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
|
29
|
-
#
|
30
|
-
def schedule_consumption(queue, jobs_array)
|
31
|
-
pt = PerformanceTracker.instance
|
32
|
-
|
33
|
-
ordered = []
|
34
|
-
|
35
|
-
jobs_array.each do |job|
|
36
|
-
messages = job.messages
|
37
|
-
message = messages.first
|
38
|
-
|
39
|
-
cost = pt.processing_time_p95(message.topic, message.partition) * messages.size
|
40
|
-
|
41
|
-
ordered << [job, cost]
|
42
|
-
end
|
43
|
-
|
44
|
-
ordered.sort_by!(&:last)
|
45
|
-
ordered.reverse!
|
46
|
-
ordered.map!(&:first)
|
47
|
-
|
48
|
-
ordered.each do |job|
|
49
|
-
queue << job
|
50
|
-
end
|
51
|
-
end
|
52
|
-
end
|
53
|
-
end
|
54
|
-
end
|
data/lib/karafka/scheduler.rb
DELETED
@@ -1,20 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Karafka
|
4
|
-
# FIFO scheduler for messages coming from various topics and partitions
|
5
|
-
class Scheduler
|
6
|
-
# Schedules jobs in the fifo order
|
7
|
-
#
|
8
|
-
# @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
|
9
|
-
# @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
|
10
|
-
def schedule_consumption(queue, jobs_array)
|
11
|
-
jobs_array.each do |job|
|
12
|
-
queue << job
|
13
|
-
end
|
14
|
-
end
|
15
|
-
|
16
|
-
# Both revocation and shutdown jobs can also run in fifo by default
|
17
|
-
alias schedule_revocation schedule_consumption
|
18
|
-
alias schedule_shutdown schedule_consumption
|
19
|
-
end
|
20
|
-
end
|