karafka 2.4.9 → 2.4.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 69bfb8dac259cdb89eadfa86b8a13b7e901fbf82f7562e0869c7bd88f6111f30
-  data.tar.gz: 68959cc13a83db2611a41556ce99db5fed6f98272626dc12f64e4c18415bc633
+  metadata.gz: 346743e75bc80a6a3e04361aa6b7b5caa540b31697bc25bb138baa490edd0a93
+  data.tar.gz: 6a87b0f7af16210b93732f55e98072b0c2eaf4541b700bb1929864b515a91747
 SHA512:
-  metadata.gz: 54d7bb2ad1d65df9b2e92f28a8ae2255dac95c37fcbc3b1bde554a7d6be0c48c79ee164f6d7b8ebd57d9b09a18ff3f955cc9a193e5ca0207403d503224fc0f61
-  data.tar.gz: 94b08e703378d7b074d32f220dbb24a02a0e3039f948780bb7c434be721df53a52b03ca349d3e26fd8330ff6d7049d389fe27965b8120ef9d4047b1acc3984d1
+  metadata.gz: c20b7bb58d31b7771e593334783285edf13bfaff977350226aeb5a979fe1fd92482bb27acf72385a05a762d69581260196f953bdcea4aafbd1d50d0842fec9a2
+  data.tar.gz: 07b991c9048f20352c1670a5782c028757e177f7f062ce3a1ce113fd01a97184717b8c7d3612d515993cb048e21b3dcf1804b22852c1b6d9bccebb16463ca2aa

data/.ruby-version

@@ -1 +1 @@
-3.3.4
+3.3.5

data/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Karafka Framework Changelog
 
+## 2.4.11 (2024-09-04)
+- [Enhancement] Validate envelope target topic type for Scheduled Messages.
+- [Enhancement] Support for enqueue_after_transaction_commit in rails active job.
+- [Fix] Fix invalid reference to AppSignal version.
+
+## 2.4.10 (2024-09-03)
+- **[Feature]** Provide Kafka based Scheduled Messages to be able to send messages in the future via a proxy topic.
+- [Enhancement] Introduce a `#assigned` hook for consumers to be able to trigger actions when consumer is built and assigned but before first consume/ticking, etc.
+- [Enhancement] Provide `Karafka::Messages::Message#tombstone?` to be able to quickly check if a message is a tombstone message.
+- [Enhancement] Provide more flexible API for Recurring Tasks topics reconfiguration.
+- [Enhancement] Remove no longer needed Rails connection releaser.
+- [Enhancement] Update AppSignal client to support newer versions (tombruijn and hieuk09).
+- [Fix] Fix a case where there would be a way to define multiple subscription groups for same topic with different consumer.
+
 ## 2.4.9 (2024-08-23)
 - **[Feature]** Provide Kafka based Recurring (Cron) Tasks.
 - [Enhancement] Wrap worker work with Rails Reloader/Executor (fusion2004)

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.4.9)
+    karafka (2.4.11)
       base64 (~> 0.2)
       karafka-core (>= 2.4.3, < 2.5.0)
       karafka-rdkafka (>= 0.17.2)
@@ -48,16 +48,16 @@ GEM
       concurrent-ruby (~> 1.0)
     karafka-core (2.4.4)
       karafka-rdkafka (>= 0.15.0, < 0.18.0)
-    karafka-rdkafka (0.17.3)
+    karafka-rdkafka (0.17.6)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
     karafka-testing (2.4.6)
       karafka (>= 2.4.0, < 2.5.0)
       waterdrop (>= 2.7.0)
-    karafka-web (0.10.0)
+    karafka-web (0.10.2)
       erubi (~> 1.4)
-      karafka (>= 2.4.7, < 2.5.0)
+      karafka (>= 2.4.10, < 2.5.0)
       karafka-core (>= 2.4.0, < 2.5.0)
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
@@ -97,7 +97,7 @@ GEM
       karafka-core (>= 2.4.3, < 3.0.0)
       karafka-rdkafka (>= 0.15.1)
       zeitwerk (~> 2.3)
-    zeitwerk (2.6.17)
+    zeitwerk (2.6.18)
 
 PLATFORMS
   ruby
@@ -116,4 +116,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.5.11
+   2.4.22

data/config/locales/errors.yml

@@ -140,6 +140,7 @@ en:
     consumer_group:
       missing: needs to be present
       topics_names_not_unique: all topic names within a single consumer group must be unique
+      topics_many_consumers_same_topic: 'topic within a single consumer group cannot have distinct consumers'
       id_format: 'needs to be a string with a Kafka accepted format'
       topics_format: needs to be a non-empty array
       topics_namespaced_names_not_unique: |

data/config/locales/pro_errors.yml

@@ -64,6 +64,8 @@ en:
       swarm_nodes_with_non_existent_nodes: includes unreachable nodes ids
 
       recurring_tasks.active_format: 'needs to be boolean'
+      scheduled_messages.active_format: 'needs to be boolean'
+      scheduled_messages.active_missing: 'needs to be boolean'
 
       direct_assignments.active_missing: needs to be present
       direct_assignments.active_format: 'needs to be boolean'
@@ -109,6 +111,15 @@ en:
       recurring_tasks.deserializer_format: 'needs to be configured'
       recurring_tasks.logging_format: needs to be a boolean
 
+      scheduled_messages.consumer_class_format: 'must be a class'
+      scheduled_messages.dispatcher_class_format: 'must be a class'
+      scheduled_messages.flush_batch_size_format: needs to be an integer bigger than 0
+      scheduled_messages.interval_format: needs to be an integer bigger or equal to 1000
+      scheduled_messages.deserializers.headers_format: cannot be nil
+      scheduled_messages.deserializers.payload_format: cannot be nil
+      scheduled_messages.group_id_format: 'needs to be a string with a Kafka accepted format'
+      scheduled_messages.states_postfix_format: 'needs to be a string with a Kafka accepted format'
+
     routing:
       swarm_nodes_not_used: 'At least one of the nodes has no assignments'
 
@@ -118,3 +129,10 @@ en:
       enabled_format: needs to be a boolean
       changed_format: needs to be a boolean
       previous_time_format: needs to be a numerical or time
+
+    scheduled_messages_message:
+      key_missing: must be present and should be unique within the partition
+      key_format: needs to be a non-empty string unique within the partition
+      headers_schedule_target_epoch_in_the_past: 'scheduling cannot happen in the past'
+      headers_format: are not correct
+      not_a_scheduled_messages_topic: 'the envelope topic is not a scheduled messages topic'

data/lib/active_job/queue_adapters/karafka_adapter.rb

@@ -29,6 +29,12 @@ module ActiveJob
       def enqueue_at(_job, _timestamp)
         raise NotImplementedError, 'This queueing backend does not support scheduling jobs.'
       end
+
+      # @return [true] should we by default enqueue after the transaction and not during.
+      #   Defaults to true to prevent weird issues during rollbacks, etc.
+      def enqueue_after_transaction_commit?
+        true
+      end
     end
   end
 end

data/lib/karafka/base_consumer.rb

@@ -31,6 +31,20 @@ module Karafka
       @used = false
     end
 
+    # Trigger method running after consumer is fully initialized.
+    #
+    # @private
+    def on_initialized
+      handle_initialized
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.initialized.error'
+      )
+    end
+
     # Can be used to run preparation code prior to the job being enqueued
     #
     # @private
@@ -176,6 +190,15 @@ module Karafka
 
     private
 
+    # Method called post-initialization of a consumer when all basic things are assigned.
+    # Since initialization via `#initialize` is complex and some states are set a bit later, this
+    # hook allows to initialize resources once at a time when topic, partition and other things
+    # are assigned to the consumer
+    #
+    # @note Please keep in mind that it will run many times when persistence is off. Basically once
+    #   each batch.
+    def initialized; end
+
     # Method that will perform business logic and on data received from Kafka (it will consume
     #   the data)
     # @note This method needs to be implemented in a subclass. We stub it here as a failover if

data/lib/karafka/contracts/consumer_group.rb

@@ -25,6 +25,23 @@ module Karafka
         [[%i[topics], :names_not_unique]]
       end
 
+      # Prevent same topics subscriptions in one CG with different consumer classes
+      # This should prevent users from accidentally creating multi-sg one CG setup with weird
+      # different consumer usage. If you need to consume same topic twice, use distinct CGs.
+      virtual do |data, errors|
+        next unless errors.empty?
+
+        topics_consumers = Hash.new { |h, k| h[k] = Set.new }
+
+        data.fetch(:topics).map do |topic|
+          topics_consumers[topic[:name]] << topic[:consumer]
+        end
+
+        next if topics_consumers.values.map(&:size).all? { |count| count == 1 }
+
+        [[%i[topics], :many_consumers_same_topic]]
+      end
+
       virtual do |data, errors|
         next unless errors.empty?
         next unless ::Karafka::App.config.strict_topics_namespacing

data/lib/karafka/instrumentation/logger_listener.rb

@@ -275,6 +275,9 @@ module Karafka
         details = (error.backtrace || []).join("\n")
 
         case type
+        when 'consumer.initialized.error'
+          error "Consumer initialized error: #{error}"
+          error details
         when 'consumer.consume.error'
           error "Consumer consuming error: #{error}"
           error details

data/lib/karafka/instrumentation/notifications.rb

@@ -48,6 +48,9 @@ module Karafka
         connection.listener.stopping
         connection.listener.stopped
 
+        consumer.initialize
+        consumer.initialized
+
         consumer.before_schedule_consume
         consumer.consume
         consumer.consumed

data/lib/karafka/instrumentation/vendors/appsignal/client.rb

@@ -23,11 +23,16 @@ module Karafka
           # @param action_name [String] action name. For processing this should be equal to
           #   consumer class + method name
           def start_transaction(action_name)
-            transaction = ::Appsignal::Transaction.create(
-              SecureRandom.uuid,
-              namespace_name,
-              ::Appsignal::Transaction::GenericRequest.new({})
-            )
+            transaction =
+              if version_4_or_newer?
+                ::Appsignal::Transaction.create(namespace_name)
+              else
+                ::Appsignal::Transaction.create(
+                  SecureRandom.uuid,
+                  namespace_name,
+                  ::Appsignal::Transaction::GenericRequest.new({})
+                )
+              end
 
             transaction.set_action_if_nil(action_name)
           end
@@ -45,10 +50,10 @@ module Karafka
           def metadata=(metadata_hash)
             return unless transaction?
 
-            transaction = ::Appsignal::Transaction.current
+            current_transaction = transaction
 
             stringify_hash(metadata_hash).each do |key, value|
-              transaction.set_metadata(key, value)
+              current_transaction.set_metadata(key, value)
             end
           end
 
@@ -78,15 +83,20 @@ module Karafka
             )
           end
 
-          # Sends the error that occurred to Appsignal
+          # Report the error that occurred to Appsignal
           #
           # @param error [Object] error we want to ship to Appsignal
-          def send_error(error)
+          def report_error(error)
+            if ::Appsignal.respond_to?(:report_error)
+              # This helper will always report the error
+              ::Appsignal.report_error(error) do |transaction|
+                transaction.set_namespace(namespace_name)
+              end
             # If we have an active transaction we should use it instead of creating a generic one
             # That way proper namespace and other data may be transferred
             #
             # In case there is no transaction, a new generic background job one will be used
-            if transaction?
+            elsif transaction?
               transaction.set_error(error)
             else
               ::Appsignal.send_error(error) do |transaction|
@@ -99,7 +109,11 @@ module Karafka
           # @param name [Symbol] probe name
           # @param probe [Proc] code to run every minute
           def register_probe(name, probe)
-            ::Appsignal::Minutely.probes.register(name, probe)
+            if ::Appsignal::Probes.respond_to?(:register)
+              ::Appsignal::Probes.register(name, probe)
+            else
+              ::Appsignal::Minutely.probes.register(name, probe)
+            end
           end
 
           private
@@ -129,6 +143,13 @@ module Karafka
           def namespace_name
             @namespace_name ||= ::Appsignal::Transaction::BACKGROUND_JOB
           end
+
+          # @return [Boolean] is this v4+ version of Appsignal gem or older. Used for backwards
+          #   compatibility checks.
+          def version_4_or_newer?
+            @version_4_or_newer ||=
+              Gem::Version.new(::Appsignal::VERSION) >= Gem::Version.new('4.0.0')
+          end
         end
       end
     end

data/lib/karafka/instrumentation/vendors/appsignal/errors_listener.rb

@@ -21,7 +21,7 @@ module Karafka
           #
           # @param event [Karafka::Core::Monitoring::Event]
           def on_error_occurred(event)
-            client.send_error(event[:error])
+            client.report_error(event[:error])
           end
         end
       end

data/lib/karafka/messages/message.rb

@@ -51,6 +51,12 @@ module Karafka
         @deserialized
       end
 
+      # @return [Boolean] true if the message has a key and raw payload is nil, it is a tombstone
+      #   event. Otherwise it is not.
+      def tombstone?
+        !raw_key.nil? && @raw_payload.nil?
+      end
+
       private
 
       # @return [Object] deserialized data

data/lib/karafka/pro/loader.rb

@@ -76,7 +76,8 @@ module Karafka
           [
             Encryption,
             Cleaner,
-            RecurringTasks
+            RecurringTasks,
+            ScheduledMessages
           ]
         end
 

data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb

@@ -22,9 +22,7 @@ module Karafka
             #
             # @param active [Boolean] should recurring tasks be active. We use a boolean flag to
             #   have API consistency in the system, so it matches other routing related APIs.
-            # @param block [Proc] optional reconfiguration of the tasks topic definitions.
-            # @note Since we cannot provide two blocks, reconfiguration of logs topic can be only
-            #   done if user explicitly redefines it in the routing.
+            # @param block [Proc] optional reconfiguration of the topics definitions.
             def recurring_tasks(active = false, &block)
               return unless active
 
@@ -39,7 +37,7 @@ module Karafka
               consumer_group tasks_cfg.group_id do
                 # Registers the primary topic that we use to control schedules execution. This is
                 # the one that we use to trigger recurring tasks.
-                topic(topics_cfg.schedules) do
+                schedules_topic = topic(topics_cfg.schedules) do
                   consumer tasks_cfg.consumer_class
                   deserializer tasks_cfg.deserializer
                   # Because the topic method name as well as builder proxy method name is the same
@@ -85,14 +83,15 @@ module Karafka
                     during_retry: false
                   )
 
-                  next unless block
-
-                  instance_eval(&block)
+                  # If this is the direct schedules redefinition style, we run it
+                  # The second one (see end of this method) allows for linear reconfiguration of
+                  # both the topics
+                  instance_eval(&block) if block && block.arity.zero?
                 end
 
                 # This topic is to store logs that we can then inspect either from the admin or via
                 # the Web UI
-                topic(topics_cfg.logs) do
+                logs_topic = topic(topics_cfg.logs) do
                   active(false)
                   deserializer tasks_cfg.deserializer
                   target.recurring_tasks(true)
@@ -104,6 +103,8 @@ module Karafka
                     'retention.ms': 604_800_000
                   )
                 end
+
+                yield(schedules_topic, logs_topic) if block && block.arity.positive?
               end
             end
 

data/lib/karafka/pro/routing/features/scheduled_messages/builder.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class ScheduledMessages < Base
+          # Routing extensions for scheduled messages
+          module Builder
+            # Enabled scheduled messages operations and adds needed topics and other stuff.
+            #
+            # @param group_name [String, false] name for scheduled messages topic that is also used
+            #   as a group identifier. Users can have multiple schedule topics flows to prevent key
+            #   collisions, prioritize and do other stuff. `false` if not active.
+            # @param block [Proc] optional reconfiguration of the topics definitions.
+            # @note Namespace for topics should include the divider as it is not automatically
+            #   added.
+            def scheduled_messages(group_name = false, &block)
+              return unless group_name
+
+              # Load zlib only if user enables scheduled messages
+              require 'zlib'
+
+              # We set it to 5 so we have enough space to handle more events. All related topics
+              # should have same partition count.
+              default_partitions = 5
+              msg_cfg = App.config.scheduled_messages
+
+              consumer_group msg_cfg.group_id do
+                # Registers the primary topic that we use to control schedules execution. This is
+                # the one that we use to trigger scheduled messages.
+                messages_topic = topic(group_name) do
+                  instance_eval(&block) if block && block.arity.zero?
+
+                  consumer msg_cfg.consumer_class
+
+                  deserializers(
+                    headers: msg_cfg.deserializers.headers
+                  )
+
+                  # Because the topic method name as well as builder proxy method name is the same
+                  # we need to reference it via target directly
+                  target.scheduled_messages(true)
+
+                  # We manage offsets directly because messages can have both schedules and
+                  # commands and we need to apply them only when we need to
+                  manual_offset_management(true)
+
+                  # We use multi-batch operations and in-memory state for schedules. This needs to
+                  # always operate without re-creation.
+                  consumer_persistence(true)
+
+                  # This needs to be enabled for the eof to work correctly
+                  kafka('enable.partition.eof': true, inherit: true)
+                  eofed(true)
+
+                  # Since this is a topic that gets replayed because of schedule management, we do
+                  # want to get more data faster during recovery
+                  max_messages(10_000)
+
+                  max_wait_time(1_000)
+
+                  # This is a setup that should allow messages to be compacted fairly fast. Since
+                  # each dispatched message should be removed via tombstone, they do not have to
+                  # be present in the topic for too long.
+                  config(
+                    partitions: default_partitions,
+                    # Will ensure, that after tombstone is present, given scheduled message, that
+                    # has been dispatched is removed by Kafka
+                    'cleanup.policy': 'compact',
+                    # When 10% or more dispatches are done, compact data
+                    'min.cleanable.dirty.ratio': 0.1,
+                    # Frequent segment rotation to support intense compaction
+                    'segment.ms': 3_600_000,
+                    'delete.retention.ms': 3_600_000,
+                    'segment.bytes': 52_428_800
+                  )
+
+                  # This is the core of execution. Since we dispatch data in time intervals, we
+                  # need to be able to do this even when no new data is coming
+                  periodic(
+                    interval: msg_cfg.interval,
+                    during_pause: false,
+                    during_retry: false
+                  )
+
+                  # If this is the direct schedules redefinition style, we run it
+                  # The second one (see end of this method) allows for linear reconfiguration of
+                  # both the topics
+                  instance_eval(&block) if block && block.arity.zero?
+                end
+
+                # Holds states of scheduler per each of the partitions since they tick
+                # independently. We only hold future statistics not to have to deal with
+                # any type of state restoration
+                states_topic = topic("#{group_name}#{msg_cfg.states_postfix}") do
+                  active(false)
+                  target.scheduled_messages(true)
+                  config(
+                    partitions: default_partitions,
+                    'cleanup.policy': 'compact',
+                    'min.cleanable.dirty.ratio': 0.1,
+                    'segment.ms': 3_600_000,
+                    'delete.retention.ms': 3_600_000,
+                    'segment.bytes': 52_428_800
+                  )
+                  deserializers(
+                    payload: msg_cfg.deserializers.payload
+                  )
+                end
+
+                yield(messages_topic, states_topic) if block && block.arity.positive?
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/scheduled_messages/config.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class ScheduledMessages < Base
+          # Scheduled messages configuration
+          Config = Struct.new(
+            :active,
+            keyword_init: true
+          ) { alias_method :active?, :active }
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/scheduled_messages/contracts/topic.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class ScheduledMessages < Base
+          # Namespace for scheduled messages contracts
+          module Contracts
+            # Rules around scheduled messages settings
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested(:scheduled_messages) do
+                required(:active) { |val| [true, false].include?(val) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/scheduled_messages/proxy.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class ScheduledMessages < Base
+          # Routing proxy extensions for scheduled messages
+          module Proxy
+            include Builder
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/scheduled_messages/topic.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class ScheduledMessages < Base
+          # Topic extensions to be able to check if given topic is a scheduled messages topic
+          # Please note, that this applies to both the schedules topic and logs topic
+          module Topic
+            # @param active [Boolean] should this topic be considered related to scheduled messages
+            def scheduled_messages(active = false)
+              @scheduled_messages ||= Config.new(active: active)
+            end
+
+            # @return [Boolean] is this an ActiveJob topic
+            def scheduled_messages?
+              scheduled_messages.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus scheduled
+            # messages namespace settings
+            def to_h
+              super.merge(
+                scheduled_messages: scheduled_messages.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end