karafka 2.0.0.beta2 → 2.0.0.beta5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/.github/workflows/ci.yml +18 -15
  4. data/CHANGELOG.md +49 -0
  5. data/Gemfile.lock +8 -8
  6. data/bin/benchmarks +2 -2
  7. data/bin/integrations +44 -15
  8. data/bin/scenario +29 -0
  9. data/bin/{stress → stress_many} +0 -0
  10. data/bin/stress_one +13 -0
  11. data/bin/wait_for_kafka +20 -0
  12. data/docker-compose.yml +28 -11
  13. data/karafka.gemspec +2 -2
  14. data/lib/karafka/active_job/routing/extensions.rb +12 -2
  15. data/lib/karafka/app.rb +2 -1
  16. data/lib/karafka/base_consumer.rb +75 -45
  17. data/lib/karafka/connection/client.rb +88 -22
  18. data/lib/karafka/connection/listener.rb +60 -18
  19. data/lib/karafka/connection/pauses_manager.rb +8 -0
  20. data/lib/karafka/connection/rebalance_manager.rb +20 -19
  21. data/lib/karafka/contracts/config.rb +17 -3
  22. data/lib/karafka/contracts/server_cli_options.rb +1 -1
  23. data/lib/karafka/errors.rb +3 -0
  24. data/lib/karafka/instrumentation/logger_listener.rb +34 -10
  25. data/lib/karafka/instrumentation/monitor.rb +3 -1
  26. data/lib/karafka/licenser.rb +26 -7
  27. data/lib/karafka/pro/active_job/consumer.rb +30 -9
  28. data/lib/karafka/pro/active_job/dispatcher.rb +9 -9
  29. data/lib/karafka/pro/active_job/job_options_contract.rb +9 -9
  30. data/lib/karafka/pro/base_consumer.rb +73 -0
  31. data/lib/karafka/pro/loader.rb +38 -20
  32. data/lib/karafka/pro/performance_tracker.rb +9 -9
  33. data/lib/karafka/pro/processing/coordinator.rb +12 -0
  34. data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb +10 -11
  35. data/lib/karafka/pro/processing/jobs_builder.rb +32 -0
  36. data/lib/karafka/pro/processing/scheduler.rb +56 -0
  37. data/lib/karafka/pro/routing/extensions.rb +32 -0
  38. data/lib/karafka/processing/coordinator.rb +84 -0
  39. data/lib/karafka/processing/coordinators_buffer.rb +58 -0
  40. data/lib/karafka/processing/executor.rb +23 -9
  41. data/lib/karafka/processing/executors_buffer.rb +46 -15
  42. data/lib/karafka/processing/jobs/base.rb +8 -3
  43. data/lib/karafka/processing/jobs/consume.rb +11 -4
  44. data/lib/karafka/processing/jobs_builder.rb +29 -0
  45. data/lib/karafka/processing/result.rb +29 -0
  46. data/lib/karafka/processing/scheduler.rb +22 -0
  47. data/lib/karafka/processing/worker.rb +17 -9
  48. data/lib/karafka/routing/consumer_group.rb +1 -1
  49. data/lib/karafka/routing/subscription_group.rb +1 -1
  50. data/lib/karafka/routing/topic.rb +14 -0
  51. data/lib/karafka/setup/config.rb +19 -9
  52. data/lib/karafka/status.rb +1 -3
  53. data/lib/karafka/version.rb +1 -1
  54. data.tar.gz.sig +0 -0
  55. metadata +19 -7
  56. metadata.gz.sig +0 -0
  57. data/lib/karafka/pro/scheduler.rb +0 -54
  58. data/lib/karafka/scheduler.rb +0 -20
@@ -0,0 +1,84 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Processing
5
+ # Basic coordinator that allows us to provide coordination objects into consumers.
6
+ #
7
+ # This is a wrapping layer to simplify management of work to be handled around consumption.
8
+ #
9
+ # @note This coordinator needs to be thread safe. Some operations are performed only in the
10
+ # listener thread, but we go with thread-safe by default for all not to worry about potential
11
+ # future mistakes.
12
+ class Coordinator
13
+ # @return [Karafka::TimeTrackers::Pause]
14
+ attr_reader :pause_tracker
15
+
16
+ # @param pause_tracker [Karafka::TimeTrackers::Pause] pause tracker for given topic partition
17
+ def initialize(pause_tracker)
18
+ @pause_tracker = pause_tracker
19
+ @revoked = false
20
+ @consumptions = {}
21
+ @running_jobs = 0
22
+ @mutex = Mutex.new
23
+ end
24
+
25
+ # Starts the coordinator for given consumption jobs
26
+ def start
27
+ @mutex.synchronize do
28
+ @running_jobs = 0
29
+ # We need to clear the consumption results hash here, otherwise we could end up storing
30
+ # consumption results of consumer instances we no longer control
31
+ @consumptions.clear
32
+ end
33
+ end
34
+
35
+ # Increases number of jobs that we handle with this coordinator
36
+ def increment
37
+ @mutex.synchronize { @running_jobs += 1 }
38
+ end
39
+
40
+ # Decrements number of jobs we handle at the moment
41
+ def decrement
42
+ @mutex.synchronize do
43
+ @running_jobs -= 1
44
+
45
+ return @running_jobs unless @running_jobs.negative?
46
+
47
+ raise Karafka::Errors::InvalidCoordinatorState, @running_jobs
48
+ end
49
+ end
50
+
51
+ # @param consumer [Object] karafka consumer (normal or pro)
52
+ # @return [Karafka::Processing::Result] result object which we can use to indicate
53
+ # consumption processing state.
54
+ def consumption(consumer)
55
+ @mutex.synchronize do
56
+ @consumptions[consumer] ||= Processing::Result.new
57
+ end
58
+ end
59
+
60
+ # Is all the consumption done and finished successfully for this coordinator
61
+ def success?
62
+ @mutex.synchronize { @running_jobs.zero? && @consumptions.values.all?(&:success?) }
63
+ end
64
+
65
+ # Marks given coordinator for processing group as revoked
66
+ #
67
+ # This is invoked in two places:
68
+ # - from the main listener loop when we detect revoked partitions
69
+ # - from the consumer in case checkpointing fails
70
+ #
71
+ # This means, we can end up having consumer being aware that it was revoked prior to the
72
+ # listener loop dispatching the revocation job. It is ok, as effectively nothing will be
73
+ # processed until revocation jobs are done.
74
+ def revoke
75
+ @mutex.synchronize { @revoked = true }
76
+ end
77
+
78
+ # @return [Boolean] is the partition we are processing revoked or not
79
+ def revoked?
80
+ @revoked
81
+ end
82
+ end
83
+ end
84
+ end
@@ -0,0 +1,58 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Processing
5
+ # Buffer used to build and store coordinators per topic partition
6
+ #
7
+ # It provides direct pauses access for revocation
8
+ #
9
+ # @note This buffer operates only from the listener loop, thus we do not have to make it
10
+ # thread-safe.
11
+ class CoordinatorsBuffer
12
+ def initialize
13
+ @pauses_manager = Connection::PausesManager.new
14
+ @coordinator_class = ::Karafka::App.config.internal.processing.coordinator_class
15
+ @coordinators = Hash.new { |h, k| h[k] = {} }
16
+ end
17
+
18
+ # @param topic [String] topic name
19
+ # @param partition [Integer] partition number
20
+ def find_or_create(topic, partition)
21
+ @coordinators[topic][partition] ||= @coordinator_class.new(
22
+ @pauses_manager.fetch(topic, partition)
23
+ )
24
+ end
25
+
26
+ # Resumes processing of partitions for which pause time has ended.
27
+ # @param block we want to run for resumed topic partitions
28
+ # @yieldparam [String] topic name
29
+ # @yieldparam [Integer] partition number
30
+ def resume(&block)
31
+ @pauses_manager.resume(&block)
32
+ end
33
+
34
+ # @param topic [String] topic name
35
+ # @param partition [Integer] partition number
36
+ def revoke(topic, partition)
37
+ @pauses_manager.revoke(topic, partition)
38
+
39
+ # The fact that we delete here does not change the fact that the executor still holds the
40
+ # reference to this coordinator. We delete it here, as we will no longer process any
41
+ # new stuff with it and we may need a new coordinator if we regain this partition, but the
42
+ # coordinator may still be in use
43
+ coordinator = @coordinators[topic].delete(partition)
44
+
45
+ return unless coordinator
46
+
47
+ coordinator.revoke
48
+ end
49
+
50
+ # Clears coordinators and re-created the pauses manager
51
+ # This should be used only for critical errors recovery
52
+ def reset
53
+ @pauses_manager = Connection::PausesManager.new
54
+ @coordinators.clear
55
+ end
56
+ end
57
+ end
58
+ end
@@ -18,18 +18,23 @@ module Karafka
18
18
  # @return [String] subscription group id to which a given executor belongs
19
19
  attr_reader :group_id
20
20
 
21
+ # @return [Karafka::Messages::Messages] messages batch
21
22
  attr_reader :messages
22
23
 
24
+ # Topic accessibility may be needed for the jobs builder to be able to build a proper job
25
+ # based on the topic settings defined by the end user
26
+ #
27
+ # @return [Karafka::Routing::Topic] topic of this executor
28
+ attr_reader :topic
29
+
23
30
  # @param group_id [String] id of the subscription group to which the executor belongs
24
31
  # @param client [Karafka::Connection::Client] kafka client
25
32
  # @param topic [Karafka::Routing::Topic] topic for which this executor will run
26
- # @param pause_tracker [Karafka::TimeTrackers::Pause] fetch pause tracker for pausing
27
- def initialize(group_id, client, topic, pause_tracker)
33
+ def initialize(group_id, client, topic)
28
34
  @id = SecureRandom.uuid
29
35
  @group_id = group_id
30
36
  @client = client
31
37
  @topic = topic
32
- @pause_tracker = pause_tracker
33
38
  end
34
39
 
35
40
  # Builds the consumer instance, builds messages batch and sets all that is needed to run the
@@ -38,13 +43,16 @@ module Karafka
38
43
  # @param messages [Array<Karafka::Messages::Message>]
39
44
  # @param received_at [Time] the moment we've received the batch (actually the moment we've)
40
45
  # enqueued it, but good enough
41
- def prepare(messages, received_at)
46
+ # @param coordinator [Karafka::Processing::Coordinator] coordinator for processing management
47
+ def before_consume(messages, received_at, coordinator)
42
48
  # Recreate consumer with each batch if persistence is not enabled
43
49
  # We reload the consumers with each batch instead of relying on some external signals
44
50
  # when needed for consistency. That way devs may have it on or off and not in this
45
51
  # middle state, where re-creation of a consumer instance would occur only sometimes
46
52
  @consumer = nil unless ::Karafka::App.config.consumer_persistence
47
53
 
54
+ consumer.coordinator = coordinator
55
+
48
56
  # First we build messages batch...
49
57
  consumer.messages = Messages::Builders::Messages.call(
50
58
  messages,
@@ -52,7 +60,7 @@ module Karafka
52
60
  received_at
53
61
  )
54
62
 
55
- consumer.on_prepared
63
+ consumer.on_before_consume
56
64
  end
57
65
 
58
66
  # Runs consumer data processing against given batch and handles failures and errors.
@@ -61,6 +69,11 @@ module Karafka
61
69
  consumer.on_consume
62
70
  end
63
71
 
72
+ # Runs consumer after consumption code
73
+ def after_consume
74
+ consumer.on_after_consume if @consumer
75
+ end
76
+
64
77
  # Runs the controller `#revoked` method that should be triggered when a given consumer is
65
78
  # no longer needed due to partitions reassignment.
66
79
  #
@@ -69,9 +82,12 @@ module Karafka
69
82
  #
70
83
  # @note We run it only when consumer was present, because presence indicates, that at least
71
84
  # a single message has been consumed.
85
+ #
86
+ # @note We do not reset the consumer but we indicate need for recreation instead, because
87
+ # after the revocation, there still may be `#after_consume` running that needs a given
88
+ # consumer instance.
72
89
  def revoked
73
90
  consumer.on_revoked if @consumer
74
- @consumer = nil
75
91
  end
76
92
 
77
93
  # Runs the controller `#shutdown` method that should be triggered when a given consumer is
@@ -83,7 +99,6 @@ module Karafka
83
99
  # There is a case, where the consumer no longer exists because it was revoked, in case like
84
100
  # that we do not build a new instance and shutdown should not be triggered.
85
101
  consumer.on_shutdown if @consumer
86
- @consumer = nil
87
102
  end
88
103
 
89
104
  private
@@ -91,10 +106,9 @@ module Karafka
91
106
  # @return [Object] cached consumer instance
92
107
  def consumer
93
108
  @consumer ||= begin
94
- consumer = @topic.consumer.new
109
+ consumer = @topic.consumer_class.new
95
110
  consumer.topic = @topic
96
111
  consumer.client = @client
97
- consumer.pause_tracker = @pause_tracker
98
112
  consumer.producer = ::Karafka::App.producer
99
113
  consumer
100
114
  end
@@ -11,30 +11,48 @@ module Karafka
11
11
  def initialize(client, subscription_group)
12
12
  @subscription_group = subscription_group
13
13
  @client = client
14
- @buffer = Hash.new { |h, k| h[k] = {} }
14
+ # We need two layers here to keep track of topics, partitions and processing groups
15
+ @buffer = Hash.new { |h, k| h[k] = Hash.new { |h2, k2| h2[k2] = {} } }
15
16
  end
16
17
 
18
+ # Finds or creates an executor based on the provided details
19
+ #
17
20
  # @param topic [String] topic name
18
21
  # @param partition [Integer] partition number
19
- # @param pause [TimeTrackers::Pause] pause corresponding with provided topic and partition
22
+ # @param parallel_key [String] parallel group key
20
23
  # @return [Executor] consumer executor
21
- def fetch(
22
- topic,
23
- partition,
24
- pause
25
- )
26
- ktopic = @subscription_group.topics.find(topic)
24
+ def find_or_create(topic, partition, parallel_key)
25
+ ktopic = find_topic(topic)
27
26
 
28
- ktopic || raise(Errors::TopicNotFoundError, topic)
29
-
30
- @buffer[ktopic][partition] ||= Executor.new(
27
+ @buffer[ktopic][partition][parallel_key] ||= Executor.new(
31
28
  @subscription_group.id,
32
29
  @client,
33
- ktopic,
34
- pause
30
+ ktopic
35
31
  )
36
32
  end
37
33
 
34
+ # Revokes executors of a given topic partition, so they won't be used anymore for incoming
35
+ # messages
36
+ #
37
+ # @param topic [String] topic name
38
+ # @param partition [Integer] partition number
39
+ def revoke(topic, partition)
40
+ ktopic = find_topic(topic)
41
+
42
+ @buffer[ktopic][partition].clear
43
+ end
44
+
45
+ # Finds all the executors available for a given topic partition
46
+ #
47
+ # @param topic [String] topic name
48
+ # @param partition [Integer] partition number
49
+ # @return [Array<Executor>] executors in use for this topic + partition
50
+ def find_all(topic, partition)
51
+ ktopic = find_topic(topic)
52
+
53
+ @buffer[ktopic][partition].values
54
+ end
55
+
38
56
  # Iterates over all available executors and yields them together with topic and partition
39
57
  # info
40
58
  # @yieldparam [Routing::Topic] karafka routing topic object
@@ -42,8 +60,11 @@ module Karafka
42
60
  # @yieldparam [Executor] given executor
43
61
  def each
44
62
  @buffer.each do |ktopic, partitions|
45
- partitions.each do |partition, executor|
46
- yield(ktopic, partition, executor)
63
+ partitions.each do |partition, executors|
64
+ executors.each do |_parallel_key, executor|
65
+ # We skip the parallel key here as it does not serve any value when iterating
66
+ yield(ktopic, partition, executor)
67
+ end
47
68
  end
48
69
  end
49
70
  end
@@ -52,6 +73,16 @@ module Karafka
52
73
  def clear
53
74
  @buffer.clear
54
75
  end
76
+
77
+ private
78
+
79
+ # Finds topic based on its name
80
+ #
81
+ # @param topic [String] topic we're looking for
82
+ # @return [Karafka::Routing::Topic] topic we're interested in
83
+ def find_topic(topic)
84
+ @subscription_group.topics.find(topic) || raise(Errors::TopicNotFoundError, topic)
85
+ end
55
86
  end
56
87
  end
57
88
  end
@@ -5,7 +5,7 @@ module Karafka
5
5
  # Namespace for all the jobs that are suppose to run in workers.
6
6
  module Jobs
7
7
  # Base class for all the jobs types that are suppose to run in workers threads.
8
- # Each job can have 3 main entry-points: `#prepare`, `#call` and `#teardown`
8
+ # Each job can have 3 main entry-points: `#before_call`, `#call` and `#after_call`
9
9
  # Only `#call` is required.
10
10
  class Base
11
11
  extend Forwardable
@@ -23,10 +23,15 @@ module Karafka
23
23
  end
24
24
 
25
25
  # When redefined can run any code that should run before executing the proper code
26
- def prepare; end
26
+ def before_call; end
27
+
28
+ # The main entry-point of a job
29
+ def call
30
+ raise NotImplementedError, 'Please implement in a subclass'
31
+ end
27
32
 
28
33
  # When redefined can run any code that should run after executing the proper code
29
- def teardown; end
34
+ def after_call; end
30
35
 
31
36
  # @return [Boolean] is this a non-blocking job
32
37
  #
@@ -12,23 +12,30 @@ module Karafka
12
12
  # @param executor [Karafka::Processing::Executor] executor that is suppose to run a given
13
13
  # job
14
14
  # @param messages [Karafka::Messages::Messages] karafka messages batch
15
+ # @param coordinator [Karafka::Processing::Coordinator] processing coordinator
15
16
  # @return [Consume]
16
- def initialize(executor, messages)
17
+ def initialize(executor, messages, coordinator)
17
18
  @executor = executor
18
19
  @messages = messages
20
+ @coordinator = coordinator
19
21
  @created_at = Time.now
20
22
  super()
21
23
  end
22
24
 
23
- # Runs the preparations on the executor
24
- def prepare
25
- executor.prepare(@messages, @created_at)
25
+ # Runs the before consumption preparations on the executor
26
+ def before_call
27
+ executor.before_consume(@messages, @created_at, @coordinator)
26
28
  end
27
29
 
28
30
  # Runs the given executor
29
31
  def call
30
32
  executor.consume
31
33
  end
34
+
35
+ # Runs any error handling and other post-consumption stuff on the executor
36
+ def after_call
37
+ executor.after_consume
38
+ end
32
39
  end
33
40
  end
34
41
  end
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Processing
5
+ # Class responsible for deciding what type of job should we build to run a given command and
6
+ # for building a proper job for it.
7
+ class JobsBuilder
8
+ # @param executor [Karafka::Processing::Executor]
9
+ # @param messages [Karafka::Messages::Messages] messages batch to be consumed
10
+ # @param coordinator [Karafka::Processing::Coordinator]
11
+ # @return [Karafka::Processing::Jobs::Consume] consumption job
12
+ def consume(executor, messages, coordinator)
13
+ Jobs::Consume.new(executor, messages, coordinator)
14
+ end
15
+
16
+ # @param executor [Karafka::Processing::Executor]
17
+ # @return [Karafka::Processing::Jobs::Revoked] revocation job
18
+ def revoked(executor)
19
+ Jobs::Revoked.new(executor)
20
+ end
21
+
22
+ # @param executor [Karafka::Processing::Executor]
23
+ # @return [Karafka::Processing::Jobs::Shutdown] shutdown job
24
+ def shutdown(executor)
25
+ Jobs::Shutdown.new(executor)
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Processing
5
+ # A simple object that allows us to keep track of processing state.
6
+ # It allows to indicate if given thing moved from success to a failure or the other way around
7
+ # Useful for tracking consumption state
8
+ class Result
9
+ def initialize
10
+ @success = true
11
+ end
12
+
13
+ # @return [Boolean]
14
+ def success?
15
+ @success
16
+ end
17
+
18
+ # Marks state as successful
19
+ def success!
20
+ @success = true
21
+ end
22
+
23
+ # Marks state as failure
24
+ def failure!
25
+ @success = false
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Processing
5
+ # FIFO scheduler for messages coming from various topics and partitions
6
+ class Scheduler
7
+ # Schedules jobs in the fifo order
8
+ #
9
+ # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
10
+ # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
11
+ def schedule_consumption(queue, jobs_array)
12
+ jobs_array.each do |job|
13
+ queue << job
14
+ end
15
+ end
16
+
17
+ # Both revocation and shutdown jobs can also run in fifo by default
18
+ alias schedule_revocation schedule_consumption
19
+ alias schedule_shutdown schedule_consumption
20
+ end
21
+ end
22
+ end
@@ -19,9 +19,13 @@ module Karafka
19
19
  class Worker
20
20
  include Helpers::Async
21
21
 
22
+ # @return [String] id of this worker
23
+ attr_reader :id
24
+
22
25
  # @param jobs_queue [JobsQueue]
23
26
  # @return [Worker]
24
27
  def initialize(jobs_queue)
28
+ @id = SecureRandom.uuid
25
29
  @jobs_queue = jobs_queue
26
30
  end
27
31
 
@@ -43,19 +47,23 @@ module Karafka
43
47
  job = @jobs_queue.pop
44
48
 
45
49
  if job
46
- job.prepare
50
+ Karafka.monitor.instrument('worker.process', caller: self, job: job)
51
+
52
+ Karafka.monitor.instrument('worker.processed', caller: self, job: job) do
53
+ job.before_call
47
54
 
48
- # If a job is marked as non blocking, we can run a tick in the job queue and if there
49
- # are no other blocking factors, the job queue will be unlocked.
50
- # If this does not run, all the things will be blocking and job queue won't allow to
51
- # pass it until done.
52
- @jobs_queue.tick(job.group_id) if job.non_blocking?
55
+ # If a job is marked as non blocking, we can run a tick in the job queue and if there
56
+ # are no other blocking factors, the job queue will be unlocked.
57
+ # If this does not run, all the things will be blocking and job queue won't allow to
58
+ # pass it until done.
59
+ @jobs_queue.tick(job.group_id) if job.non_blocking?
53
60
 
54
- job.call
61
+ job.call
55
62
 
56
- job.teardown
63
+ job.after_call
57
64
 
58
- true
65
+ true
66
+ end
59
67
  else
60
68
  false
61
69
  end
@@ -38,7 +38,7 @@ module Karafka
38
38
  # @return [Array<Routing::SubscriptionGroup>] all the subscription groups build based on
39
39
  # the consumer group topics
40
40
  def subscription_groups
41
- App.config.internal.subscription_groups_builder.call(topics)
41
+ App.config.internal.routing.subscription_groups_builder.call(topics)
42
42
  end
43
43
 
44
44
  # Hashed version of consumer group that can be used for validation purposes
@@ -44,7 +44,7 @@ module Karafka
44
44
  kafka[:'auto.offset.reset'] ||= @topics.first.initial_offset
45
45
  # Karafka manages the offsets based on the processing state, thus we do not rely on the
46
46
  # rdkafka offset auto-storing
47
- kafka[:'enable.auto.offset.store'] = 'false'
47
+ kafka[:'enable.auto.offset.store'] = false
48
48
  kafka.freeze
49
49
  kafka
50
50
  end
@@ -66,6 +66,20 @@ module Karafka
66
66
  end
67
67
  end
68
68
 
69
+ # @return [Class] consumer class that we should use
70
+ # @note This is just an alias to the `#consumer` method. We however want to use it internally
71
+ # instead of referencing the `#consumer`. We use this to indicate that this method returns
72
+ # class and not an instance. In the routing we want to keep the `#consumer Consumer`
73
+ # routing syntax, but for references outside, we should use this one.
74
+ def consumer_class
75
+ consumer
76
+ end
77
+
78
+ # @return [Boolean] true if this topic offset is handled by the end user
79
+ def manual_offset_management?
80
+ manual_offset_management
81
+ end
82
+
69
83
  # @return [Hash] hash with all the topic attributes
70
84
  # @note This is being used when we validate the consumer_group and its topics
71
85
  def to_h
@@ -83,21 +83,31 @@ module Karafka
83
83
  # @see https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
84
84
  setting :kafka, default: {}
85
85
 
86
- # Namespace for internal settings that should not be modified
86
+ # Namespace for internal settings that should not be modified directly
87
87
  setting :internal do
88
- # option routing_builder [Karafka::Routing::Builder] builder instance
89
- setting :routing_builder, default: Routing::Builder.new
90
88
  # option status [Karafka::Status] app status
91
89
  setting :status, default: Status.new
92
90
  # option process [Karafka::Process] process status
93
91
  # @note In the future, we need to have a single process representation for all the karafka
94
92
  # instances
95
93
  setting :process, default: Process.new
96
- # option subscription_groups_builder [Routing::SubscriptionGroupsBuilder] subscription
97
- # group builder
98
- setting :subscription_groups_builder, default: Routing::SubscriptionGroupsBuilder.new
99
- # option scheduler [Class] scheduler we will be using
100
- setting :scheduler, default: Scheduler.new
94
+
95
+ setting :routing do
96
+ # option builder [Karafka::Routing::Builder] builder instance
97
+ setting :builder, default: Routing::Builder.new
98
+ # option subscription_groups_builder [Routing::SubscriptionGroupsBuilder] subscription
99
+ # group builder
100
+ setting :subscription_groups_builder, default: Routing::SubscriptionGroupsBuilder.new
101
+ end
102
+
103
+ setting :processing do
104
+ # option scheduler [Object] scheduler we will be using
105
+ setting :scheduler, default: Processing::Scheduler.new
106
+ # option jobs_builder [Object] jobs builder we want to use
107
+ setting :jobs_builder, default: Processing::JobsBuilder.new
108
+ # option coordinator [Class] work coordinator we want to user for processing coordination
109
+ setting :coordinator_class, default: Processing::Coordinator
110
+ end
101
111
 
102
112
  # Karafka components for ActiveJob
103
113
  setting :active_job do
@@ -107,7 +117,7 @@ module Karafka
107
117
  # ensuring, that extra job options defined are valid
108
118
  setting :job_options_contract, default: ActiveJob::JobOptionsContract.new
109
119
  # option consumer [Class] consumer class that should be used to consume ActiveJob data
110
- setting :consumer, default: ActiveJob::Consumer
120
+ setting :consumer_class, default: ActiveJob::Consumer
111
121
  end
112
122
  end
113
123
 
@@ -31,9 +31,7 @@ module Karafka
31
31
  # We skip as during this state we do not have yet a monitor
32
32
  return if initializing?
33
33
 
34
- # Trap context disallows to run certain things that we instrument
35
- # so the state changes are executed from a separate thread
36
- Thread.new { Karafka.monitor.instrument("app.#{state}") }.join
34
+ Karafka.monitor.instrument("app.#{state}")
37
35
  end
38
36
  end
39
37
  end
@@ -3,5 +3,5 @@
3
3
  # Main module namespace
4
4
  module Karafka
5
5
  # Current Karafka version
6
- VERSION = '2.0.0.beta2'
6
+ VERSION = '2.0.0.beta5'
7
7
  end
data.tar.gz.sig CHANGED
Binary file