karafka 2.0.0.alpha6 → 2.0.0.beta3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/.ruby-version +1 -1
  4. data/CHANGELOG.md +42 -2
  5. data/Gemfile.lock +9 -9
  6. data/bin/integrations +36 -14
  7. data/bin/scenario +29 -0
  8. data/config/errors.yml +1 -0
  9. data/docker-compose.yml +3 -0
  10. data/karafka.gemspec +1 -1
  11. data/lib/active_job/karafka.rb +2 -2
  12. data/lib/karafka/active_job/routing/extensions.rb +31 -0
  13. data/lib/karafka/base_consumer.rb +74 -6
  14. data/lib/karafka/connection/client.rb +39 -16
  15. data/lib/karafka/connection/listener.rb +103 -34
  16. data/lib/karafka/connection/listeners_batch.rb +24 -0
  17. data/lib/karafka/connection/messages_buffer.rb +48 -61
  18. data/lib/karafka/connection/pauses_manager.rb +2 -2
  19. data/lib/karafka/connection/raw_messages_buffer.rb +101 -0
  20. data/lib/karafka/contracts/config.rb +10 -1
  21. data/lib/karafka/helpers/async.rb +33 -0
  22. data/lib/karafka/instrumentation/logger_listener.rb +37 -10
  23. data/lib/karafka/instrumentation/monitor.rb +4 -0
  24. data/lib/karafka/licenser.rb +26 -7
  25. data/lib/karafka/messages/batch_metadata.rb +26 -3
  26. data/lib/karafka/messages/builders/batch_metadata.rb +17 -29
  27. data/lib/karafka/messages/builders/message.rb +1 -0
  28. data/lib/karafka/messages/builders/messages.rb +4 -12
  29. data/lib/karafka/pro/active_job/consumer.rb +48 -0
  30. data/lib/karafka/pro/active_job/dispatcher.rb +3 -3
  31. data/lib/karafka/pro/active_job/job_options_contract.rb +2 -2
  32. data/lib/karafka/pro/base_consumer_extensions.rb +66 -0
  33. data/lib/karafka/pro/loader.rb +27 -4
  34. data/lib/karafka/pro/performance_tracker.rb +80 -0
  35. data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb +37 -0
  36. data/lib/karafka/pro/processing/jobs_builder.rb +31 -0
  37. data/lib/karafka/pro/routing/extensions.rb +32 -0
  38. data/lib/karafka/pro/scheduler.rb +54 -0
  39. data/lib/karafka/processing/executor.rb +26 -11
  40. data/lib/karafka/processing/executors_buffer.rb +15 -7
  41. data/lib/karafka/processing/jobs/base.rb +28 -0
  42. data/lib/karafka/processing/jobs/consume.rb +11 -4
  43. data/lib/karafka/processing/jobs_builder.rb +28 -0
  44. data/lib/karafka/processing/jobs_queue.rb +28 -16
  45. data/lib/karafka/processing/worker.rb +39 -10
  46. data/lib/karafka/processing/workers_batch.rb +5 -0
  47. data/lib/karafka/routing/consumer_group.rb +1 -1
  48. data/lib/karafka/routing/subscription_group.rb +2 -2
  49. data/lib/karafka/routing/subscription_groups_builder.rb +3 -2
  50. data/lib/karafka/routing/topics.rb +38 -0
  51. data/lib/karafka/runner.rb +19 -27
  52. data/lib/karafka/scheduler.rb +20 -0
  53. data/lib/karafka/server.rb +24 -23
  54. data/lib/karafka/setup/config.rb +6 -1
  55. data/lib/karafka/status.rb +1 -3
  56. data/lib/karafka/time_trackers/pause.rb +10 -2
  57. data/lib/karafka/version.rb +1 -1
  58. data.tar.gz.sig +0 -0
  59. metadata +19 -4
  60. metadata.gz.sig +0 -0
  61. data/lib/karafka/active_job/routing_extensions.rb +0 -18
@@ -83,7 +83,7 @@ module Karafka
83
83
  # @see https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
84
84
  setting :kafka, default: {}
85
85
 
86
- # Namespace for internal settings that should not be modified
86
+ # Namespace for internal settings that should not be modified directly
87
87
  setting :internal do
88
88
  # option routing_builder [Karafka::Routing::Builder] builder instance
89
89
  setting :routing_builder, default: Routing::Builder.new
@@ -96,6 +96,10 @@ module Karafka
96
96
  # option subscription_groups_builder [Routing::SubscriptionGroupsBuilder] subscription
97
97
  # group builder
98
98
  setting :subscription_groups_builder, default: Routing::SubscriptionGroupsBuilder.new
99
+ # option scheduler [Class] scheduler we will be using
100
+ setting :scheduler, default: Scheduler.new
101
+ # option jobs_builder [Class] jobs builder we want to use
102
+ setting :jobs_builder, default: Processing::JobsBuilder.new
99
103
 
100
104
  # Karafka components for ActiveJob
101
105
  setting :active_job do
@@ -115,6 +119,7 @@ module Karafka
115
119
  def setup(&block)
116
120
  configure(&block)
117
121
  merge_kafka_defaults!(config)
122
+
118
123
  Contracts::Config.new.validate!(config.to_h)
119
124
 
120
125
  # Check the license presence (if needed) and
@@ -31,9 +31,7 @@ module Karafka
31
31
  # We skip as during this state we do not have yet a monitor
32
32
  return if initializing?
33
33
 
34
- # Trap context disallows to run certain things that we instrument
35
- # so the state changes are executed from a separate thread
36
- Thread.new { Karafka.monitor.instrument("app.#{state}") }.join
34
+ Karafka.monitor.instrument("app.#{state}")
37
35
  end
38
36
  end
39
37
  end
@@ -41,9 +41,12 @@ module Karafka
41
41
 
42
42
  # Pauses the processing from now till the end of the interval (backoff or non-backoff)
43
43
  # and records the count.
44
- def pause
44
+ # @param timeout [Integer] timeout value in milliseconds that overwrites the default timeout
45
+ # @note Providing this value can be useful when we explicitly want to pause for a certain
46
+ # period of time, outside of any regular pausing logic
47
+ def pause(timeout = backoff_interval)
45
48
  @started_at = now
46
- @ends_at = @started_at + backoff_interval
49
+ @ends_at = @started_at + timeout
47
50
  @count += 1
48
51
  end
49
52
 
@@ -53,6 +56,11 @@ module Karafka
53
56
  @ends_at = nil
54
57
  end
55
58
 
59
+ # Expires the pause, so it can be considered expired
60
+ def expire
61
+ @ends_at = nil
62
+ end
63
+
56
64
  # @return [Boolean] are we paused from processing
57
65
  def paused?
58
66
  !@started_at.nil?
@@ -3,5 +3,5 @@
3
3
  # Main module namespace
4
4
  module Karafka
5
5
  # Current Karafka version
6
- VERSION = '2.0.0.alpha6'
6
+ VERSION = '2.0.0.beta3'
7
7
  end
data.tar.gz.sig CHANGED
Binary file
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: karafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.0.alpha6
4
+ version: 2.0.0.beta3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Maciej Mensfeld
@@ -34,7 +34,7 @@ cert_chain:
34
34
  R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
35
35
  pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
36
36
  -----END CERTIFICATE-----
37
- date: 2022-04-17 00:00:00.000000000 Z
37
+ date: 2022-06-14 00:00:00.000000000 Z
38
38
  dependencies:
39
39
  - !ruby/object:Gem::Dependency
40
40
  name: dry-configurable
@@ -171,6 +171,7 @@ files:
171
171
  - bin/create_token
172
172
  - bin/integrations
173
173
  - bin/karafka
174
+ - bin/scenario
174
175
  - bin/stress
175
176
  - certs/karafka-pro.pem
176
177
  - certs/mensfeld.pem
@@ -184,7 +185,7 @@ files:
184
185
  - lib/karafka/active_job/dispatcher.rb
185
186
  - lib/karafka/active_job/job_extensions.rb
186
187
  - lib/karafka/active_job/job_options_contract.rb
187
- - lib/karafka/active_job/routing_extensions.rb
188
+ - lib/karafka/active_job/routing/extensions.rb
188
189
  - lib/karafka/app.rb
189
190
  - lib/karafka/base_consumer.rb
190
191
  - lib/karafka/cli.rb
@@ -195,8 +196,10 @@ files:
195
196
  - lib/karafka/cli/server.rb
196
197
  - lib/karafka/connection/client.rb
197
198
  - lib/karafka/connection/listener.rb
199
+ - lib/karafka/connection/listeners_batch.rb
198
200
  - lib/karafka/connection/messages_buffer.rb
199
201
  - lib/karafka/connection/pauses_manager.rb
202
+ - lib/karafka/connection/raw_messages_buffer.rb
200
203
  - lib/karafka/connection/rebalance_manager.rb
201
204
  - lib/karafka/contracts.rb
202
205
  - lib/karafka/contracts/base.rb
@@ -206,6 +209,7 @@ files:
206
209
  - lib/karafka/contracts/server_cli_options.rb
207
210
  - lib/karafka/env.rb
208
211
  - lib/karafka/errors.rb
212
+ - lib/karafka/helpers/async.rb
209
213
  - lib/karafka/helpers/multi_delegator.rb
210
214
  - lib/karafka/instrumentation.rb
211
215
  - lib/karafka/instrumentation/callbacks/error.rb
@@ -225,9 +229,16 @@ files:
225
229
  - lib/karafka/messages/seek.rb
226
230
  - lib/karafka/patches/rdkafka/consumer.rb
227
231
  - lib/karafka/pro.rb
232
+ - lib/karafka/pro/active_job/consumer.rb
228
233
  - lib/karafka/pro/active_job/dispatcher.rb
229
234
  - lib/karafka/pro/active_job/job_options_contract.rb
235
+ - lib/karafka/pro/base_consumer_extensions.rb
230
236
  - lib/karafka/pro/loader.rb
237
+ - lib/karafka/pro/performance_tracker.rb
238
+ - lib/karafka/pro/processing/jobs/consume_non_blocking.rb
239
+ - lib/karafka/pro/processing/jobs_builder.rb
240
+ - lib/karafka/pro/routing/extensions.rb
241
+ - lib/karafka/pro/scheduler.rb
231
242
  - lib/karafka/process.rb
232
243
  - lib/karafka/processing/executor.rb
233
244
  - lib/karafka/processing/executors_buffer.rb
@@ -235,6 +246,7 @@ files:
235
246
  - lib/karafka/processing/jobs/consume.rb
236
247
  - lib/karafka/processing/jobs/revoked.rb
237
248
  - lib/karafka/processing/jobs/shutdown.rb
249
+ - lib/karafka/processing/jobs_builder.rb
238
250
  - lib/karafka/processing/jobs_queue.rb
239
251
  - lib/karafka/processing/worker.rb
240
252
  - lib/karafka/processing/workers_batch.rb
@@ -247,7 +259,9 @@ files:
247
259
  - lib/karafka/routing/subscription_group.rb
248
260
  - lib/karafka/routing/subscription_groups_builder.rb
249
261
  - lib/karafka/routing/topic.rb
262
+ - lib/karafka/routing/topics.rb
250
263
  - lib/karafka/runner.rb
264
+ - lib/karafka/scheduler.rb
251
265
  - lib/karafka/serialization/json/deserializer.rb
252
266
  - lib/karafka/server.rb
253
267
  - lib/karafka/setup/config.rb
@@ -264,6 +278,7 @@ files:
264
278
  homepage: https://karafka.io
265
279
  licenses:
266
280
  - LGPL-3.0
281
+ - Commercial
267
282
  metadata:
268
283
  source_code_uri: https://github.com/karafka/karafka
269
284
  rubygems_mfa_required: 'true'
@@ -282,7 +297,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
282
297
  - !ruby/object:Gem::Version
283
298
  version: 1.3.1
284
299
  requirements: []
285
- rubygems_version: 3.3.3
300
+ rubygems_version: 3.3.7
286
301
  signing_key:
287
302
  specification_version: 4
288
303
  summary: Ruby based framework for working with Apache Kafka
metadata.gz.sig CHANGED
Binary file
@@ -1,18 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- # ActiveJob related Karafka stuff
5
- module ActiveJob
6
- # Routing extensions for ActiveJob
7
- module RoutingExtensions
8
- # This method simplifies routes definition for ActiveJob topics / queues by auto-injecting
9
- # the consumer class
10
- # @param name [String, Symbol] name of the topic where ActiveJobs jobs should go
11
- def active_job_topic(name)
12
- topic(name) do
13
- consumer App.config.internal.active_job.consumer
14
- end
15
- end
16
- end
17
- end
18
- end