karafka 2.3.4 → 2.4.0.beta1

data/lib/karafka/swarm/supervisor.rb

@@ -23,6 +23,15 @@ module Karafka
         process: %i[internal process]
       )
 
+      # How long extra should we wait on shutdown before forceful termination
+      # We add this time because we send signals and it always can take a bit of time for them
+      # to reach out nodes and be processed to start the shutdown flow. Because of that and
+      # because we always want to give all nodes all the time of `shutdown_timeout` they are
+      # expected to have, we add this just to compensate.
+      SHUTDOWN_GRACE_PERIOD = 1_000
+
+      private_constant :SHUTDOWN_GRACE_PERIOD
+
       def initialize
         @mutex = Mutex.new
         @queue = Processing::TimedQueue.new
@@ -70,7 +79,10 @@ module Karafka
           type: 'swarm.supervisor.error'
         )
 
-        @nodes.terminate
+        manager.terminate
+        manager.cleanup
+
+        raise e
       end
 
       private
@@ -102,10 +114,12 @@ module Karafka
 
         manager.stop
 
+        total_shutdown_timeout = shutdown_timeout + SHUTDOWN_GRACE_PERIOD
+
         # We check from time to time (for the timeout period) if all the threads finished
         # their work and if so, we can just return and normal shutdown process will take place
         # We divide it by 1000 because we use time in ms.
-        ((shutdown_timeout / 1_000) * (1 / supervision_sleep)).to_i.times do
+        ((total_shutdown_timeout / 1_000) * (1 / supervision_sleep)).to_i.times do
           if manager.stopped?
             manager.cleanup
             return

data/lib/karafka/templates/karafka.rb.erb

@@ -37,7 +37,14 @@ class KarafkaApp < Karafka::App
   # interested in logging events for certain environments. Since instrumentation
   # notifications add extra boilerplate, if you want to achieve max performance,
   # listen to only what you really need for given environment.
-  Karafka.monitor.subscribe(Karafka::Instrumentation::LoggerListener.new)
+  Karafka.monitor.subscribe(
+    Karafka::Instrumentation::LoggerListener.new(
+      # Karafka, when the logger is set to info producers logs each time it polls data from an
+      # internal messages wueue. This can be extensive, so you can turn it off by setting below
+      # to false.
+      log_polling: true
+    )
+  )
   # Karafka.monitor.subscribe(Karafka::Instrumentation::ProctitleListener.new)
 
   # This logger prints the producer development info using the Karafka logger.
@@ -52,6 +59,26 @@ class KarafkaApp < Karafka::App
     )
   )
 
+  # You can subscribe to all consumer related errors and record/track then that way
+  #
+  # Karafka.monitor.subscribe 'error.occurred' do |event|
+  #   type = event[:type]
+  #   error = event[:error]
+  #   details = (error.backtrace || []).join("\n")
+  #   ErrorTracker.send_error(error, type, details)
+  # end
+
+  # You can subscribe to all producer related errors and record/track then that way
+  # Please note, that producer and consumer have their own notifications pipeline so you need to
+  # setup error tracking independently for each of them
+  #
+  # Karafka.producer.monitor.subscribe('error.occurred') do |event|
+  #   type = event[:type]
+  #   error = event[:error]
+  #   details = (error.backtrace || []).join("\n")
+  #   ErrorTracker.send_error(error, type, details)
+  # end
+
   routes.draw do
 <% if rails? -%>
     # Uncomment this if you use Karafka with ActiveJob

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.3.4'
+  VERSION = '2.4.0.beta1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.3.4
+  version: 2.4.0.beta1
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-04-11 00:00:00.000000000 Z
+date: 2024-04-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -43,27 +43,27 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.3.0
+        version: 2.4.0.rc1
     - - "<"
       - !ruby/object:Gem::Version
-        version: 2.4.0
+        version: 2.5.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.3.0
+        version: 2.4.0.rc1
     - - "<"
       - !ruby/object:Gem::Version
-        version: 2.4.0
+        version: 2.5.0
 - !ruby/object:Gem::Dependency
   name: waterdrop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.6.12
+        version: 2.7.0.rc1
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -73,7 +73,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.6.12
+        version: 2.7.0.rc1
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -153,6 +153,9 @@ files:
 - lib/karafka/active_job/job_options_contract.rb
 - lib/karafka/admin.rb
 - lib/karafka/admin/acl.rb
+- lib/karafka/admin/configs.rb
+- lib/karafka/admin/configs/config.rb
+- lib/karafka/admin/configs/resource.rb
 - lib/karafka/app.rb
 - lib/karafka/base_consumer.rb
 - lib/karafka/cli.rb
@@ -164,6 +167,14 @@ files:
 - lib/karafka/cli/server.rb
 - lib/karafka/cli/swarm.rb
 - lib/karafka/cli/topics.rb
+- lib/karafka/cli/topics/align.rb
+- lib/karafka/cli/topics/base.rb
+- lib/karafka/cli/topics/create.rb
+- lib/karafka/cli/topics/delete.rb
+- lib/karafka/cli/topics/migrate.rb
+- lib/karafka/cli/topics/plan.rb
+- lib/karafka/cli/topics/repartition.rb
+- lib/karafka/cli/topics/reset.rb
 - lib/karafka/connection/client.rb
 - lib/karafka/connection/conductor.rb
 - lib/karafka/connection/listener.rb
@@ -182,6 +193,9 @@ files:
 - lib/karafka/contracts/consumer_group.rb
 - lib/karafka/contracts/server_cli_options.rb
 - lib/karafka/contracts/topic.rb
+- lib/karafka/deserializers/headers.rb
+- lib/karafka/deserializers/key.rb
+- lib/karafka/deserializers/payload.rb
 - lib/karafka/embedded.rb
 - lib/karafka/env.rb
 - lib/karafka/errors.rb
@@ -192,6 +206,7 @@ files:
 - lib/karafka/helpers/multi_delegator.rb
 - lib/karafka/instrumentation/assignments_tracker.rb
 - lib/karafka/instrumentation/callbacks/error.rb
+- lib/karafka/instrumentation/callbacks/oauthbearer_token_refresh.rb
 - lib/karafka/instrumentation/callbacks/rebalance.rb
 - lib/karafka/instrumentation/callbacks/statistics.rb
 - lib/karafka/instrumentation/logger.rb
@@ -270,6 +285,8 @@ files:
 - lib/karafka/pro/processing/offset_metadata/fetcher.rb
 - lib/karafka/pro/processing/offset_metadata/listener.rb
 - lib/karafka/pro/processing/partitioner.rb
+- lib/karafka/pro/processing/periodic_job/consumer.rb
+- lib/karafka/pro/processing/piping/consumer.rb
 - lib/karafka/pro/processing/schedulers/base.rb
 - lib/karafka/pro/processing/schedulers/default.rb
 - lib/karafka/pro/processing/strategies.rb
@@ -323,6 +340,7 @@ files:
 - lib/karafka/pro/processing/strategies/mom/vp.rb
 - lib/karafka/pro/processing/strategies/vp/default.rb
 - lib/karafka/pro/processing/strategy_selector.rb
+- lib/karafka/pro/processing/subscription_groups_coordinator.rb
 - lib/karafka/pro/routing/features/active_job.rb
 - lib/karafka/pro/routing/features/active_job/builder.rb
 - lib/karafka/pro/routing/features/base.rb
@@ -333,6 +351,12 @@ files:
 - lib/karafka/pro/routing/features/delaying/config.rb
 - lib/karafka/pro/routing/features/delaying/contracts/topic.rb
 - lib/karafka/pro/routing/features/delaying/topic.rb
+- lib/karafka/pro/routing/features/direct_assignments.rb
+- lib/karafka/pro/routing/features/direct_assignments/config.rb
+- lib/karafka/pro/routing/features/direct_assignments/contracts/consumer_group.rb
+- lib/karafka/pro/routing/features/direct_assignments/contracts/topic.rb
+- lib/karafka/pro/routing/features/direct_assignments/subscription_group.rb
+- lib/karafka/pro/routing/features/direct_assignments/topic.rb
 - lib/karafka/pro/routing/features/expiring.rb
 - lib/karafka/pro/routing/features/expiring/config.rb
 - lib/karafka/pro/routing/features/expiring/contracts/topic.rb
@@ -383,6 +407,7 @@ files:
 - lib/karafka/pro/routing/features/periodic_job/topic.rb
 - lib/karafka/pro/routing/features/swarm.rb
 - lib/karafka/pro/routing/features/swarm/config.rb
+- lib/karafka/pro/routing/features/swarm/contracts/routing.rb
 - lib/karafka/pro/routing/features/swarm/contracts/topic.rb
 - lib/karafka/pro/routing/features/swarm/topic.rb
 - lib/karafka/pro/routing/features/throttling.rb
@@ -428,7 +453,6 @@ files:
 - lib/karafka/routing/activity_manager.rb
 - lib/karafka/routing/builder.rb
 - lib/karafka/routing/consumer_group.rb
-- lib/karafka/routing/consumer_mapper.rb
 - lib/karafka/routing/features/active_job.rb
 - lib/karafka/routing/features/active_job/builder.rb
 - lib/karafka/routing/features/active_job/config.rb
@@ -445,6 +469,10 @@ files:
 - lib/karafka/routing/features/declaratives/config.rb
 - lib/karafka/routing/features/declaratives/contracts/topic.rb
 - lib/karafka/routing/features/declaratives/topic.rb
+- lib/karafka/routing/features/deserializers.rb
+- lib/karafka/routing/features/deserializers/config.rb
+- lib/karafka/routing/features/deserializers/contracts/topic.rb
+- lib/karafka/routing/features/deserializers/topic.rb
 - lib/karafka/routing/features/inline_insights.rb
 - lib/karafka/routing/features/inline_insights/config.rb
 - lib/karafka/routing/features/inline_insights/contracts/topic.rb
@@ -460,7 +488,6 @@ files:
 - lib/karafka/routing/topic.rb
 - lib/karafka/routing/topics.rb
 - lib/karafka/runner.rb
-- lib/karafka/serialization/json/deserializer.rb
 - lib/karafka/server.rb
 - lib/karafka/setup/attributes_map.rb
 - lib/karafka/setup/config.rb
@@ -476,7 +503,6 @@ files:
 - lib/karafka/templates/example_consumer.rb.erb
 - lib/karafka/templates/karafka.rb.erb
 - lib/karafka/time_trackers/base.rb
-- lib/karafka/time_trackers/partition_usage.rb
 - lib/karafka/time_trackers/pause.rb
 - lib/karafka/time_trackers/poll.rb
 - lib/karafka/version.rb
@@ -502,7 +528,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 3.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="

data/lib/karafka/routing/consumer_mapper.rb

@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Routing
-    # Default consumer mapper that builds consumer ids based on app id and consumer group name
-    # Different mapper can be used in case of preexisting consumer names or for applying
-    # other naming conventions not compatible with Karafka client_id + consumer name concept
-    #
-    # @example Mapper for using consumer groups without a client_id prefix
-    #   class MyMapper
-    #     def call(raw_consumer_group_name)
-    #       raw_consumer_group_name
-    #     end
-    #   end
-    class ConsumerMapper
-      # @param raw_consumer_group_name [String, Symbol] string or symbolized consumer group name
-      # @return [String] remapped final consumer group name
-      def call(raw_consumer_group_name)
-        "#{Karafka::App.config.client_id}_#{raw_consumer_group_name}"
-      end
-    end
-  end
-end

data/lib/karafka/serialization/json/deserializer.rb

@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Module for all supported by default serialization and deserialization ways.
-  module Serialization
-    # Namespace for json serializers and deserializers.
-    module Json
-      # Default Karafka Json deserializer for loading JSON data.
-      class Deserializer
-        # @param message [Karafka::Messages::Message] Message object that we want to deserialize
-        # @return [Hash] hash with deserialized JSON data
-        def call(message)
-          # nil payload can be present for example for tombstone messages
-          message.raw_payload.nil? ? nil : ::JSON.parse(message.raw_payload)
-        end
-      end
-    end
-  end
-end

data/lib/karafka/time_trackers/partition_usage.rb

@@ -1,56 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module TimeTrackers
-    # Tracker used to keep time reference when we last time dispatched any job related to a given
-    # topic partition.
-    #
-    # We can use it to know when last time a job was scheduled
-    #
-    # @note We do not track revocation as on revocation we clear given topic partition reference
-    #   not to have a potential memory leak
-    #
-    # @note We do not track shutdown jobs as shutdown is finishing the process, so no time
-    #   sensitive operations remain that would use this
-    #
-    # @note We consider partition as active if we scheduled any job related to it within the tick
-    #   interval. This has nothing to do whether a partition is assigned.
-    class PartitionUsage < Base
-      # Creates new partition usage time tracker
-      def initialize
-        super
-
-        @last_usage = Hash.new do |topics_hash, topic_name|
-          topics_hash[topic_name] = Hash.new do |partitions_hash, partition_id|
-            partitions_hash[partition_id] = 0
-          end
-        end
-      end
-
-      # @param topic [String]
-      # @param partition [Integer]
-      # @param interval [Integer] minimum interval
-      # @return [Boolean] was this topic partition active
-      def active?(topic, partition, interval)
-        monotonic_now - @last_usage[topic][partition] < interval
-      end
-
-      # Marks usage of given partition
-      #
-      # @param topic [String]
-      # @param partition [Integer]
-      def track(topic, partition)
-        @last_usage[topic][partition] = monotonic_now
-      end
-
-      # Clears references about given partition. Useful on revocation so we do not store old
-      # unassigned partitions data
-      #
-      # @param topic [String]
-      # @param partition [Integer]
-      def revoke(topic, partition)
-        @last_usage[topic].delete(partition)
-      end
-    end
-  end
-end