karafka 2.0.0.alpha6 → 2.0.0.beta3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.ruby-version +1 -1
- data/CHANGELOG.md +42 -2
- data/Gemfile.lock +9 -9
- data/bin/integrations +36 -14
- data/bin/scenario +29 -0
- data/config/errors.yml +1 -0
- data/docker-compose.yml +3 -0
- data/karafka.gemspec +1 -1
- data/lib/active_job/karafka.rb +2 -2
- data/lib/karafka/active_job/routing/extensions.rb +31 -0
- data/lib/karafka/base_consumer.rb +74 -6
- data/lib/karafka/connection/client.rb +39 -16
- data/lib/karafka/connection/listener.rb +103 -34
- data/lib/karafka/connection/listeners_batch.rb +24 -0
- data/lib/karafka/connection/messages_buffer.rb +48 -61
- data/lib/karafka/connection/pauses_manager.rb +2 -2
- data/lib/karafka/connection/raw_messages_buffer.rb +101 -0
- data/lib/karafka/contracts/config.rb +10 -1
- data/lib/karafka/helpers/async.rb +33 -0
- data/lib/karafka/instrumentation/logger_listener.rb +37 -10
- data/lib/karafka/instrumentation/monitor.rb +4 -0
- data/lib/karafka/licenser.rb +26 -7
- data/lib/karafka/messages/batch_metadata.rb +26 -3
- data/lib/karafka/messages/builders/batch_metadata.rb +17 -29
- data/lib/karafka/messages/builders/message.rb +1 -0
- data/lib/karafka/messages/builders/messages.rb +4 -12
- data/lib/karafka/pro/active_job/consumer.rb +48 -0
- data/lib/karafka/pro/active_job/dispatcher.rb +3 -3
- data/lib/karafka/pro/active_job/job_options_contract.rb +2 -2
- data/lib/karafka/pro/base_consumer_extensions.rb +66 -0
- data/lib/karafka/pro/loader.rb +27 -4
- data/lib/karafka/pro/performance_tracker.rb +80 -0
- data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb +37 -0
- data/lib/karafka/pro/processing/jobs_builder.rb +31 -0
- data/lib/karafka/pro/routing/extensions.rb +32 -0
- data/lib/karafka/pro/scheduler.rb +54 -0
- data/lib/karafka/processing/executor.rb +26 -11
- data/lib/karafka/processing/executors_buffer.rb +15 -7
- data/lib/karafka/processing/jobs/base.rb +28 -0
- data/lib/karafka/processing/jobs/consume.rb +11 -4
- data/lib/karafka/processing/jobs_builder.rb +28 -0
- data/lib/karafka/processing/jobs_queue.rb +28 -16
- data/lib/karafka/processing/worker.rb +39 -10
- data/lib/karafka/processing/workers_batch.rb +5 -0
- data/lib/karafka/routing/consumer_group.rb +1 -1
- data/lib/karafka/routing/subscription_group.rb +2 -2
- data/lib/karafka/routing/subscription_groups_builder.rb +3 -2
- data/lib/karafka/routing/topics.rb +38 -0
- data/lib/karafka/runner.rb +19 -27
- data/lib/karafka/scheduler.rb +20 -0
- data/lib/karafka/server.rb +24 -23
- data/lib/karafka/setup/config.rb +6 -1
- data/lib/karafka/status.rb +1 -3
- data/lib/karafka/time_trackers/pause.rb +10 -2
- data/lib/karafka/version.rb +1 -1
- data.tar.gz.sig +0 -0
- metadata +19 -4
- metadata.gz.sig +0 -0
- data/lib/karafka/active_job/routing_extensions.rb +0 -18
data/lib/karafka/setup/config.rb
CHANGED
@@ -83,7 +83,7 @@ module Karafka
|
|
83
83
|
# @see https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
|
84
84
|
setting :kafka, default: {}
|
85
85
|
|
86
|
-
# Namespace for internal settings that should not be modified
|
86
|
+
# Namespace for internal settings that should not be modified directly
|
87
87
|
setting :internal do
|
88
88
|
# option routing_builder [Karafka::Routing::Builder] builder instance
|
89
89
|
setting :routing_builder, default: Routing::Builder.new
|
@@ -96,6 +96,10 @@ module Karafka
|
|
96
96
|
# option subscription_groups_builder [Routing::SubscriptionGroupsBuilder] subscription
|
97
97
|
# group builder
|
98
98
|
setting :subscription_groups_builder, default: Routing::SubscriptionGroupsBuilder.new
|
99
|
+
# option scheduler [Class] scheduler we will be using
|
100
|
+
setting :scheduler, default: Scheduler.new
|
101
|
+
# option jobs_builder [Class] jobs builder we want to use
|
102
|
+
setting :jobs_builder, default: Processing::JobsBuilder.new
|
99
103
|
|
100
104
|
# Karafka components for ActiveJob
|
101
105
|
setting :active_job do
|
@@ -115,6 +119,7 @@ module Karafka
|
|
115
119
|
def setup(&block)
|
116
120
|
configure(&block)
|
117
121
|
merge_kafka_defaults!(config)
|
122
|
+
|
118
123
|
Contracts::Config.new.validate!(config.to_h)
|
119
124
|
|
120
125
|
# Check the license presence (if needed) and
|
data/lib/karafka/status.rb
CHANGED
@@ -31,9 +31,7 @@ module Karafka
|
|
31
31
|
# We skip as during this state we do not have yet a monitor
|
32
32
|
return if initializing?
|
33
33
|
|
34
|
-
|
35
|
-
# so the state changes are executed from a separate thread
|
36
|
-
Thread.new { Karafka.monitor.instrument("app.#{state}") }.join
|
34
|
+
Karafka.monitor.instrument("app.#{state}")
|
37
35
|
end
|
38
36
|
end
|
39
37
|
end
|
@@ -41,9 +41,12 @@ module Karafka
|
|
41
41
|
|
42
42
|
# Pauses the processing from now till the end of the interval (backoff or non-backoff)
|
43
43
|
# and records the count.
|
44
|
-
|
44
|
+
# @param timeout [Integer] timeout value in milliseconds that overwrites the default timeout
|
45
|
+
# @note Providing this value can be useful when we explicitly want to pause for a certain
|
46
|
+
# period of time, outside of any regular pausing logic
|
47
|
+
def pause(timeout = backoff_interval)
|
45
48
|
@started_at = now
|
46
|
-
@ends_at = @started_at +
|
49
|
+
@ends_at = @started_at + timeout
|
47
50
|
@count += 1
|
48
51
|
end
|
49
52
|
|
@@ -53,6 +56,11 @@ module Karafka
|
|
53
56
|
@ends_at = nil
|
54
57
|
end
|
55
58
|
|
59
|
+
# Expires the pause, so it can be considered expired
|
60
|
+
def expire
|
61
|
+
@ends_at = nil
|
62
|
+
end
|
63
|
+
|
56
64
|
# @return [Boolean] are we paused from processing
|
57
65
|
def paused?
|
58
66
|
!@started_at.nil?
|
data/lib/karafka/version.rb
CHANGED
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.0.
|
4
|
+
version: 2.0.0.beta3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Maciej Mensfeld
|
@@ -34,7 +34,7 @@ cert_chain:
|
|
34
34
|
R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
|
35
35
|
pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
|
36
36
|
-----END CERTIFICATE-----
|
37
|
-
date: 2022-
|
37
|
+
date: 2022-06-14 00:00:00.000000000 Z
|
38
38
|
dependencies:
|
39
39
|
- !ruby/object:Gem::Dependency
|
40
40
|
name: dry-configurable
|
@@ -171,6 +171,7 @@ files:
|
|
171
171
|
- bin/create_token
|
172
172
|
- bin/integrations
|
173
173
|
- bin/karafka
|
174
|
+
- bin/scenario
|
174
175
|
- bin/stress
|
175
176
|
- certs/karafka-pro.pem
|
176
177
|
- certs/mensfeld.pem
|
@@ -184,7 +185,7 @@ files:
|
|
184
185
|
- lib/karafka/active_job/dispatcher.rb
|
185
186
|
- lib/karafka/active_job/job_extensions.rb
|
186
187
|
- lib/karafka/active_job/job_options_contract.rb
|
187
|
-
- lib/karafka/active_job/
|
188
|
+
- lib/karafka/active_job/routing/extensions.rb
|
188
189
|
- lib/karafka/app.rb
|
189
190
|
- lib/karafka/base_consumer.rb
|
190
191
|
- lib/karafka/cli.rb
|
@@ -195,8 +196,10 @@ files:
|
|
195
196
|
- lib/karafka/cli/server.rb
|
196
197
|
- lib/karafka/connection/client.rb
|
197
198
|
- lib/karafka/connection/listener.rb
|
199
|
+
- lib/karafka/connection/listeners_batch.rb
|
198
200
|
- lib/karafka/connection/messages_buffer.rb
|
199
201
|
- lib/karafka/connection/pauses_manager.rb
|
202
|
+
- lib/karafka/connection/raw_messages_buffer.rb
|
200
203
|
- lib/karafka/connection/rebalance_manager.rb
|
201
204
|
- lib/karafka/contracts.rb
|
202
205
|
- lib/karafka/contracts/base.rb
|
@@ -206,6 +209,7 @@ files:
|
|
206
209
|
- lib/karafka/contracts/server_cli_options.rb
|
207
210
|
- lib/karafka/env.rb
|
208
211
|
- lib/karafka/errors.rb
|
212
|
+
- lib/karafka/helpers/async.rb
|
209
213
|
- lib/karafka/helpers/multi_delegator.rb
|
210
214
|
- lib/karafka/instrumentation.rb
|
211
215
|
- lib/karafka/instrumentation/callbacks/error.rb
|
@@ -225,9 +229,16 @@ files:
|
|
225
229
|
- lib/karafka/messages/seek.rb
|
226
230
|
- lib/karafka/patches/rdkafka/consumer.rb
|
227
231
|
- lib/karafka/pro.rb
|
232
|
+
- lib/karafka/pro/active_job/consumer.rb
|
228
233
|
- lib/karafka/pro/active_job/dispatcher.rb
|
229
234
|
- lib/karafka/pro/active_job/job_options_contract.rb
|
235
|
+
- lib/karafka/pro/base_consumer_extensions.rb
|
230
236
|
- lib/karafka/pro/loader.rb
|
237
|
+
- lib/karafka/pro/performance_tracker.rb
|
238
|
+
- lib/karafka/pro/processing/jobs/consume_non_blocking.rb
|
239
|
+
- lib/karafka/pro/processing/jobs_builder.rb
|
240
|
+
- lib/karafka/pro/routing/extensions.rb
|
241
|
+
- lib/karafka/pro/scheduler.rb
|
231
242
|
- lib/karafka/process.rb
|
232
243
|
- lib/karafka/processing/executor.rb
|
233
244
|
- lib/karafka/processing/executors_buffer.rb
|
@@ -235,6 +246,7 @@ files:
|
|
235
246
|
- lib/karafka/processing/jobs/consume.rb
|
236
247
|
- lib/karafka/processing/jobs/revoked.rb
|
237
248
|
- lib/karafka/processing/jobs/shutdown.rb
|
249
|
+
- lib/karafka/processing/jobs_builder.rb
|
238
250
|
- lib/karafka/processing/jobs_queue.rb
|
239
251
|
- lib/karafka/processing/worker.rb
|
240
252
|
- lib/karafka/processing/workers_batch.rb
|
@@ -247,7 +259,9 @@ files:
|
|
247
259
|
- lib/karafka/routing/subscription_group.rb
|
248
260
|
- lib/karafka/routing/subscription_groups_builder.rb
|
249
261
|
- lib/karafka/routing/topic.rb
|
262
|
+
- lib/karafka/routing/topics.rb
|
250
263
|
- lib/karafka/runner.rb
|
264
|
+
- lib/karafka/scheduler.rb
|
251
265
|
- lib/karafka/serialization/json/deserializer.rb
|
252
266
|
- lib/karafka/server.rb
|
253
267
|
- lib/karafka/setup/config.rb
|
@@ -264,6 +278,7 @@ files:
|
|
264
278
|
homepage: https://karafka.io
|
265
279
|
licenses:
|
266
280
|
- LGPL-3.0
|
281
|
+
- Commercial
|
267
282
|
metadata:
|
268
283
|
source_code_uri: https://github.com/karafka/karafka
|
269
284
|
rubygems_mfa_required: 'true'
|
@@ -282,7 +297,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
282
297
|
- !ruby/object:Gem::Version
|
283
298
|
version: 1.3.1
|
284
299
|
requirements: []
|
285
|
-
rubygems_version: 3.3.
|
300
|
+
rubygems_version: 3.3.7
|
286
301
|
signing_key:
|
287
302
|
specification_version: 4
|
288
303
|
summary: Ruby based framework for working with Apache Kafka
|
metadata.gz.sig
CHANGED
Binary file
|
@@ -1,18 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Karafka
|
4
|
-
# ActiveJob related Karafka stuff
|
5
|
-
module ActiveJob
|
6
|
-
# Routing extensions for ActiveJob
|
7
|
-
module RoutingExtensions
|
8
|
-
# This method simplifies routes definition for ActiveJob topics / queues by auto-injecting
|
9
|
-
# the consumer class
|
10
|
-
# @param name [String, Symbol] name of the topic where ActiveJobs jobs should go
|
11
|
-
def active_job_topic(name)
|
12
|
-
topic(name) do
|
13
|
-
consumer App.config.internal.active_job.consumer
|
14
|
-
end
|
15
|
-
end
|
16
|
-
end
|
17
|
-
end
|
18
|
-
end
|