shared_broker 1.6.0 → 1.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 24d5ac30bc2bef8c5fd99edbcb8e7e986fa8005fde7cfdb39bfb4f3c6010efa8
-  data.tar.gz: d66d983238edb492bddf145c48866ce83a55b88bf6806472910bdafdbbd418ec
+  metadata.gz: 972f349828dcd6839a7e95c7e5d285603fcbec36c7ef52c4953e85ee67038f34
+  data.tar.gz: 19a1b4d987ea960bb1f2ab362bb3acc1afea685e6d4f019dfac14f5ec9b7c24c
 SHA512:
-  metadata.gz: c25d32bc67d7c28427c533b1f3e23b048a7513740f0e8a51e067f5f5d861b761a3189b7d84d06b977400b21bf022c003c50d4515e46cfdc6cee064d36acbdb38
-  data.tar.gz: 1ab4165d6750a4f1d28f3ce67c070d58135d10c793b4eac944ba257b8b01f3dcd0665e36018aa0c32b27076439297e6e8ee0b789f5b943fa2820ef9472d2f09f
+  metadata.gz: 4d86395b12e3f37e71a5014ba8b6196436f4d28ed06cdc1227363a20be5e62943edf1280819d68cbafee1bf249aae89414a28da50abfbacefaaffb3e4dfcabe8
+  data.tar.gz: aef9fe8f2f60ccb9f3812470f98ddacccc75b2699d75e3f286525636fbdb5e278900e5befad76432c172c734cfc8de0058d7d00dcf349545f9106627edc3c669

data/CHANGELOG.md

@@ -5,6 +5,22 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.8.0] - 2026-06-21
+
+### Added
+- **Resilience & Advanced Processing**:
+  - **Batch Publishing**: Added `publish_batch` method in `SharedBroker::Client` and adapter classes, verifying schemas and encrypting payloads individually before transmission.
+  - **Deduplication / Idempotency**: Created `SharedBroker::Middlewares::Idempotency` to deduplicate consumed messages by tracking their `correlation_id` against a memory or cache-based store (such as `Rails.cache`).
+  - **DLQ Redrive**: Created `SharedBroker::DLQ::Redriver` utility along with `redrive_dlq` on memory, Redis, and RabbitMQ adapters to pull failed messages from DLQs back into active queues.
+
+## [1.7.0] - 2026-06-18
+
+### Added
+- **Automatic Payload Compression**: Decoupled payload compression mechanism for publishing and subscribing.
+- `SharedBroker::Compressor` supporting standard `:gzip` and `:deflate` algorithms.
+- Configurable global `compression_algorithm` and `compression_threshold` limits to prevent compressing small payloads.
+- Automatic versioning of encrypted and compressed payloads via the `_compression` tag.
+
 ## [1.6.0] - 2026-06-16
 
 ### Added

data/README.md

@@ -195,6 +195,24 @@ With a key provider registry configured:
 - **Subscribing**: The gem automatically reads the `_key_id` from the payload envelope and decrypts it using the correct historical key version.
 - **Fallback**: If no `_key_id` is present on a received message (e.g., legacy message), it falls back to the key associated with the topic pattern.
 
+#### F. Automatic Payload Compression
+
+To optimize network bandwidth and storage costs, `SharedBroker` can automatically compress large payloads before encrypting them. You can configure compression globally:
+
+```ruby
+# Enable compression using either :gzip or :deflate (nil by default)
+SharedBroker.compression_algorithm = :gzip
+
+# Set the threshold in bytes. Payloads smaller than this will NOT be compressed.
+# This avoids the overhead of compression for tiny messages.
+SharedBroker.compression_threshold = 1024 # 1 KB
+```
+
+When compression is active:
+- **Publishing**: If the payload size exceeds the threshold, it is compressed, marked with the `_compression` tag in the metadata envelope, and then encrypted.
+- **Subscribing**: The consumer checks for the `_compression` tag, decrypts the payload, and automatically decompresses it before passing it to your subscriber block.
+- **Compatibility**: If a consumer receives a message that has no `_compression` tag, it will bypass decompression and decrypt normally.
+
 ---
 
 ## Usage
@@ -234,6 +252,49 @@ SPOT_BROKER.subscribe(
 end
 ```
 
+
+### Publishing Events in Batch
+Send multiple events at once:
+
+```ruby
+events = [
+  { id: 1, email: "alice@example.com" },
+  { id: 2, email: "bob@example.com" }
+]
+
+# Validates and encrypts all payloads, then publishes them in a batch
+SPOT_BROKER.publish_batch("user.created", events)
+```
+
+### Subscriber Idempotency Middleware
+Configure deduplication of messages using the Idempotency Middleware. It skips already processed messages by checking their `correlation_id` (uses an internal in-memory store by default, but accepts duck-typed stores like `Rails.cache`):
+
+```ruby
+# Initialize with custom store and cache expiration (default 3600 seconds)
+idempotency = SharedBroker::Middlewares::Idempotency.new(
+  store: Rails.cache,
+  expires_in: 86400 # 1 day
+)
+
+SPOT_BROKER = SharedBroker::Client.new(
+  adapter: BROKER_ADAPTER,
+  middlewares: [idempotency]
+)
+```
+
+### DLQ Redrive Utility
+Move failed messages back to the original queue for reprocessing after bug fixes:
+
+```ruby
+# Redrives up to 50 messages from DLQ list/queue to original topic
+SharedBroker::DLQ::Redriver.redrive(
+  SPOT_BROKER, 
+  "my_consumption_queue.dlq", 
+  "user.created", 
+  limit: 50
+)
+```
+
 ---
 
 ## Running Gem Tests

data/lib/shared_broker/adapters/base.rb

@@ -1,15 +1,29 @@
-# frozen_string_literal: true
-
-module SharedBroker
-  module Adapters
-    class Base
-      def publish(topic, message, correlation_id: nil)
-        raise NotImplementedError, "#{self.class.name} must implement #publish"
-      end
-
-      def subscribe(topic, queue_name, max_retries: 3, backoff_base: 2, &block)
-        raise NotImplementedError, "#{self.class.name} must implement #subscribe"
-      end
-    end
-  end
-end
+# frozen_string_literal: true
+
+module SharedBroker
+  module Adapters
+    class Base
+      def publish(topic, message, correlation_id: nil)
+        raise NotImplementedError, "#{self.class.name} must implement #publish"
+      end
+
+      def publish_batch(topic, messages, correlation_id: nil)
+        unless messages.is_a?(Array)
+          raise ArgumentError, "Expected messages to be an Array, got #{messages.class} with value #{messages.inspect}"
+        end
+
+        messages.each do |message|
+          publish(topic, message, correlation_id: correlation_id)
+        end
+      end
+
+      def subscribe(topic, queue_name, max_retries: 3, backoff_base: 2, &block)
+        raise NotImplementedError, "#{self.class.name} must implement #subscribe"
+      end
+
+      def redrive_dlq(dlq_name, original_topic, limit: nil)
+        raise NotImplementedError, "#{self.class.name} must implement #redrive_dlq"
+      end
+    end
+  end
+end

data/lib/shared_broker/adapters/in_memory.rb

@@ -1,51 +1,70 @@
-# frozen_string_literal: true
-
-require "time"
-require_relative "base"
-
-module SharedBroker
-  module Adapters
-    class InMemory < Base
-      def initialize
-        @storage = Hash.new { |h, k| h[k] = [] }
-        @subscribers = Hash.new { |h, k| h[k] = [] }
-      end
-
-      def publish(topic, message, correlation_id: nil)
-        msg_with_metadata = message.merge(_correlation_id: correlation_id)
-        @storage[topic] << msg_with_metadata
-        
-        @subscribers[topic].each do |sub|
-          attempts = 0
-          begin
-            sub[:block].call(msg_with_metadata)
-          rescue => e
-            attempts += 1
-            if attempts <= sub[:max_retries]
-              # Sleep briefly or not at all in memory to keep tests fast
-              sleep(0.001 * sub[:backoff_base]**attempts)
-              retry
-            else
-              dlq_topic = "#{sub[:queue_name]}.dlq"
-              dlq_msg = msg_with_metadata.merge(
-                _x_original_routing_key: topic,
-                _x_failed_at: Time.now.utc.iso8601,
-                _x_exception_class: e.class.name,
-                _x_exception_message: e.message
-              )
-              @storage[dlq_topic] << dlq_msg
-            end
-          end
-        end
-      end
-
-      def subscribe(topic, queue_name, max_retries: 3, backoff_base: 2, &block)
-        @subscribers[topic] << { queue_name: queue_name, max_retries: max_retries, backoff_base: backoff_base, block: block }
-      end
-
-      def published_messages(topic)
-        @storage[topic]
-      end
-    end
-  end
-end
+# frozen_string_literal: true
+
+require "time"
+require_relative "base"
+
+module SharedBroker
+  module Adapters
+    class InMemory < Base
+      def initialize
+        @storage = Hash.new { |h, k| h[k] = [] }
+        @subscribers = Hash.new { |h, k| h[k] = [] }
+      end
+
+      def publish(topic, message, correlation_id: nil)
+        msg_with_metadata = message.merge(_correlation_id: correlation_id)
+        @storage[topic] << msg_with_metadata
+        
+        @subscribers[topic].each do |sub|
+          attempts = 0
+          begin
+            sub[:block].call(msg_with_metadata)
+          rescue SharedBroker::ShutdownError => e
+            raise e
+          rescue => e
+            attempts += 1
+            if attempts <= sub[:max_retries]
+              # Sleep briefly or not at all in memory to keep tests fast
+              sleep(0.001 * sub[:backoff_base]**attempts)
+              retry
+            else
+              dlq_topic = "#{sub[:queue_name]}.dlq"
+              dlq_msg = msg_with_metadata.merge(
+                _x_original_routing_key: topic,
+                _x_failed_at: Time.now.utc.iso8601,
+                _x_exception_class: e.class.name,
+                _x_exception_message: e.message
+              )
+              @storage[dlq_topic] << dlq_msg
+            end
+          end
+        end
+      end
+
+      def subscribe(topic, queue_name, max_retries: 3, backoff_base: 2, &block)
+        @subscribers[topic] << { queue_name: queue_name, max_retries: max_retries, backoff_base: backoff_base, block: block }
+      end
+
+      def published_messages(topic)
+        @storage[topic]
+      end
+
+      def redrive_dlq(dlq_name, original_topic, limit: nil)
+        dlq_messages = @storage[dlq_name]
+        return if dlq_messages.empty?
+
+        to_redrive = limit ? dlq_messages.first(limit) : dlq_messages.dup
+        to_redrive.each do |msg|
+          cleaned_msg = msg.dup
+          cleaned_msg.delete(:_x_original_routing_key)
+          cleaned_msg.delete(:_x_failed_at)
+          cleaned_msg.delete(:_x_exception_class)
+          cleaned_msg.delete(:_x_exception_message)
+
+          publish(original_topic, cleaned_msg, correlation_id: cleaned_msg[:_correlation_id])
+          @storage[dlq_name].delete(msg)
+        end
+      end
+    end
+  end
+end

data/lib/shared_broker/adapters/kafka.rb

@@ -41,6 +41,8 @@ module SharedBroker
             attempts = 0
             begin
               block.call(data)
+            rescue SharedBroker::ShutdownError
+              consumer.stop
             rescue => e
               attempts += 1
               if attempts <= max_retries

data/lib/shared_broker/adapters/rabbit_mq.rb

@@ -1,84 +1,104 @@
-# frozen_string_literal: true
-
-require "bunny"
-require "json"
-require "time"
-require_relative "base"
-
-module SharedBroker
-  module Adapters
-    class RabbitMQ < Base
-      EXCHANGE_NAME = "shared_broker_events"
-
-      def initialize(amqp_url:)
-        @connection = Bunny.new(amqp_url)
-        @connection.start
-        @channel = @connection.create_channel
-        @exchange = @channel.topic(EXCHANGE_NAME, durable: true)
-      end
-
-      def publish(topic, message, correlation_id: nil)
-        unless message.is_a?(Hash)
-          raise ArgumentError, "Message must be a Hash, got #{message.class} with value #{message.inspect}"
-        end
-
-        options = { routing_key: topic }
-        options[:correlation_id] = correlation_id if correlation_id
-        @exchange.publish(message.to_json, options)
-      end
-
-      def subscribe(topic, queue_name, max_retries: 3, backoff_base: 2, &block)
-        queue = @channel.queue(queue_name, durable: true)
-        queue.bind(@exchange, routing_key: topic)
-        
-        queue.subscribe(manual_ack: true) do |delivery_info, metadata, payload|
-          data = JSON.parse(payload, symbolize_names: true)
-          if metadata.respond_to?(:correlation_id) && metadata.correlation_id
-            data[:_correlation_id] = metadata.correlation_id
-          end
-
-          attempts = 0
-          begin
-            block.call(data)
-            @channel.acknowledge(delivery_info.delivery_tag, false)
-          rescue => e
-            attempts += 1
-            if attempts <= max_retries
-              sleep(backoff_base**attempts)
-              retry
-            else
-              publish_to_dlq(queue_name, payload, delivery_info, metadata, e)
-              @channel.acknowledge(delivery_info.delivery_tag, false)
-            end
-          end
-        end
-      end
-
-      def close
-        @channel.close if @channel
-        @connection.close if @connection
-      end
-
-      private
-
-      def publish_to_dlq(queue_name, payload, delivery_info, metadata, exception)
-        dlq_name = "#{queue_name}.dlq"
-        dlq_queue = @channel.queue(dlq_name, durable: true)
-        
-        headers = {
-          x_original_routing_key: delivery_info.routing_key,
-          x_failed_at: Time.now.utc.iso8601,
-          x_exception_class: exception.class.name,
-          x_exception_message: exception.message
-        }
-        
-        @channel.default_exchange.publish(
-          payload,
-          routing_key: dlq_queue.name,
-          correlation_id: metadata.correlation_id,
-          headers: headers
-        )
-      end
-    end
-  end
-end
+# frozen_string_literal: true
+
+require "bunny"
+require "json"
+require "time"
+require_relative "base"
+
+module SharedBroker
+  module Adapters
+    class RabbitMQ < Base
+      EXCHANGE_NAME = "shared_broker_events"
+
+      def initialize(amqp_url:)
+        @connection = Bunny.new(amqp_url)
+        @connection.start
+        @channel = @connection.create_channel
+        @exchange = @channel.topic(EXCHANGE_NAME, durable: true)
+      end
+
+      def publish(topic, message, correlation_id: nil)
+        unless message.is_a?(Hash)
+          raise ArgumentError, "Message must be a Hash, got #{message.class} with value #{message.inspect}"
+        end
+
+        options = { routing_key: topic }
+        options[:correlation_id] = correlation_id if correlation_id
+        @exchange.publish(message.to_json, options)
+      end
+
+      def subscribe(topic, queue_name, max_retries: 3, backoff_base: 2, &block)
+        queue = @channel.queue(queue_name, durable: true)
+        queue.bind(@exchange, routing_key: topic)
+        
+        queue.subscribe(manual_ack: true) do |delivery_info, metadata, payload|
+          data = JSON.parse(payload, symbolize_names: true)
+          if metadata.respond_to?(:correlation_id) && metadata.correlation_id
+            data[:_correlation_id] = metadata.correlation_id
+          end
+
+          attempts = 0
+          begin
+            block.call(data)
+            @channel.acknowledge(delivery_info.delivery_tag, false)
+          rescue SharedBroker::ShutdownError
+            # Do not acknowledge or send to DLQ. Let RabbitMQ re-queue the message.
+          rescue => e
+            attempts += 1
+            if attempts <= max_retries
+              sleep(backoff_base**attempts)
+              retry
+            else
+              publish_to_dlq(queue_name, payload, delivery_info, metadata, e)
+              @channel.acknowledge(delivery_info.delivery_tag, false)
+            end
+          end
+        end
+      end
+
+      def redrive_dlq(dlq_name, original_topic, limit: nil)
+        dlq_queue = @channel.queue(dlq_name, durable: true)
+        count = 0
+
+        loop do
+          break if limit && count >= limit
+          delivery_info, metadata, payload = dlq_queue.pop(manual_ack: true)
+          break unless delivery_info
+
+          options = { routing_key: original_topic }
+          options[:correlation_id] = metadata.correlation_id if metadata&.correlation_id
+          @exchange.publish(payload, options)
+
+          @channel.acknowledge(delivery_info.delivery_tag, false)
+          count += 1
+        end
+      end
+
+      def close
+        @channel.close if @channel
+        @connection.close if @connection
+      end
+
+      private
+
+      def publish_to_dlq(queue_name, payload, delivery_info, metadata, exception)
+        dlq_name = "#{queue_name}.dlq"
+        dlq_queue = @channel.queue(dlq_name, durable: true)
+        
+        headers = {
+          x_original_routing_key: delivery_info.routing_key,
+          x_failed_at: Time.now.utc.iso8601,
+          x_exception_class: exception.class.name,
+          x_exception_message: exception.message
+        }
+        
+        @channel.default_exchange.publish(
+          payload,
+          routing_key: dlq_queue.name,
+          correlation_id: metadata.correlation_id,
+          headers: headers
+        )
+      end
+    end
+  end
+end

data/lib/shared_broker/adapters/redis.rb

@@ -1,64 +1,80 @@
-# frozen_string_literal: true
-
-require_relative "base"
-require "json"
-require "time"
-
-module SharedBroker
-  module Adapters
-    class Redis < Base
-      def initialize(redis_url:)
-        begin
-          require "redis"
-        rescue LoadError
-          raise unless defined?(::Redis)
-        end
-        @redis = ::Redis.new(url: redis_url)
-      end
-
-      def publish(topic, message, correlation_id: nil)
-        unless message.is_a?(Hash)
-          raise ArgumentError, "Expected message to be a Hash, got #{message.class} with value #{message.inspect}"
-        end
-
-        payload = message.merge(_correlation_id: correlation_id)
-        @redis.publish(topic, payload.to_json)
-      end
-
-      def subscribe(topic, queue_name, max_retries: 3, backoff_base: 2, &block)
-        Thread.new do
-          @redis.subscribe(topic) do |on|
-            on.message do |_channel, msg_json|
-              data = JSON.parse(msg_json, symbolize_names: true)
-              attempts = 0
-              begin
-                block.call(data)
-              rescue => e
-                attempts += 1
-                if attempts <= max_retries
-                  sleep(backoff_base**attempts)
-                  retry
-                else
-                  publish_to_dlq(topic, queue_name, msg_json, e)
-                end
-              end
-            end
-          end
-        end
-      end
-
-      private
-
-      def publish_to_dlq(topic, queue_name, payload_json, exception)
-        dlq_key = "dlq:#{topic}:#{queue_name}"
-        dlq_payload = {
-          payload: JSON.parse(payload_json, symbolize_names: true),
-          x_failed_at: Time.now.utc.iso8601,
-          x_exception_class: exception.class.name,
-          x_exception_message: exception.message
-        }
-        @redis.rpush(dlq_key, dlq_payload.to_json)
-      end
-    end
-  end
-end
+# frozen_string_literal: true
+
+require_relative "base"
+require "json"
+require "time"
+
+module SharedBroker
+  module Adapters
+    class Redis < Base
+      def initialize(redis_url:)
+        begin
+          require "redis"
+        rescue LoadError
+          raise unless defined?(::Redis)
+        end
+        @redis = ::Redis.new(url: redis_url)
+      end
+
+      def publish(topic, message, correlation_id: nil)
+        unless message.is_a?(Hash)
+          raise ArgumentError, "Expected message to be a Hash, got #{message.class} with value #{message.inspect}"
+        end
+
+        payload = message.merge(_correlation_id: correlation_id)
+        @redis.publish(topic, payload.to_json)
+      end
+
+      def subscribe(topic, queue_name, max_retries: 3, backoff_base: 2, &block)
+        Thread.new do
+          @redis.subscribe(topic) do |on|
+            on.message do |_channel, msg_json|
+              data = JSON.parse(msg_json, symbolize_names: true)
+              attempts = 0
+              begin
+                block.call(data)
+              rescue SharedBroker::ShutdownError
+                @redis.unsubscribe
+              rescue => e
+                attempts += 1
+                if attempts <= max_retries
+                  sleep(backoff_base**attempts)
+                  retry
+                else
+                  publish_to_dlq(topic, queue_name, msg_json, e)
+                end
+              end
+            end
+          end
+        end
+      end
+
+      def redrive_dlq(dlq_name, original_topic, limit: nil)
+        count = 0
+        loop do
+          break if limit && count >= limit
+          raw_json = @redis.lpop(dlq_name)
+          break unless raw_json
+
+          dlq_data = JSON.parse(raw_json, symbolize_names: true)
+          original_payload = dlq_data[:payload]
+          publish(original_topic, original_payload, correlation_id: original_payload[:_correlation_id])
+          count += 1
+        end
+      end
+
+      private
+
+      def publish_to_dlq(topic, queue_name, payload_json, exception)
+        dlq_key = "dlq:#{topic}:#{queue_name}"
+        dlq_payload = {
+          payload: JSON.parse(payload_json, symbolize_names: true),
+          x_failed_at: Time.now.utc.iso8601,
+          x_exception_class: exception.class.name,
+          x_exception_message: exception.message
+        }
+        @redis.rpush(dlq_key, dlq_payload.to_json)
+      end
+    end
+  end
+end

data/lib/shared_broker/cipher.rb

@@ -22,13 +22,20 @@ module SharedBroker
 
       metadata = payload_hash.select { |k, _| k.to_s.start_with?("_") }
       data_to_encrypt = payload_hash.reject { |k, _| k.to_s.start_with?("_") }
+      json_data = data_to_encrypt.to_json
+
+      compression_alg = nil
+      if SharedBroker.compression_algorithm && json_data.bytesize > SharedBroker.compression_threshold
+        compression_alg = SharedBroker.compression_algorithm
+        json_data = SharedBroker::Compressor.compress(json_data, compression_alg)
+      end
 
       cipher = OpenSSL::Cipher.new(ALGORITHM)
       cipher.encrypt
       cipher.key = key
       iv = cipher.random_iv
 
-      encrypted_data = cipher.update(data_to_encrypt.to_json) + cipher.final
+      encrypted_data = cipher.update(json_data) + cipher.final
       auth_tag = cipher.auth_tag
 
       envelope = {
@@ -38,6 +45,7 @@ module SharedBroker
         _data: Base64.strict_encode64(encrypted_data)
       }
       envelope[:_key_id] = key_id.to_s if key_id
+      envelope[:_compression] = compression_alg.to_s if compression_alg
 
       metadata.merge(envelope)
     end
@@ -56,7 +64,14 @@ module SharedBroker
       cipher.auth_tag = Base64.strict_decode64(payload_hash[:_auth_tag])
 
       encrypted_bytes = Base64.strict_decode64(payload_hash[:_data])
-      decrypted_json = cipher.update(encrypted_bytes) + cipher.final
+      decrypted_raw = cipher.update(encrypted_bytes) + cipher.final
+
+      compression_alg = payload_hash[:_compression]
+      decrypted_json = if compression_alg
+                         SharedBroker::Compressor.decompress(decrypted_raw, compression_alg)
+                       else
+                         decrypted_raw
+                       end
 
       decrypted_data = JSON.parse(decrypted_json, symbolize_names: true)
       
@@ -93,6 +108,7 @@ module SharedBroker
       metadata.delete(:_auth_tag)
       metadata.delete(:_data)
       metadata.delete(:_key_id)
+      metadata.delete(:_compression)
     end
   end
 end

data/lib/shared_broker/compressor.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require "zlib"
+
+module SharedBroker
+  module Compressor
+    class CompressionError < StandardError; end
+    class DecompressionError < StandardError; end
+
+    SUPPORTED_ALGORITHMS = %w[gzip deflate].freeze
+
+    def self.compress(data, algorithm)
+      validate_algorithm!(algorithm)
+      
+      case algorithm.to_s
+      when "gzip"
+        Zlib.gzip(data)
+      when "deflate"
+        Zlib::Deflate.deflate(data)
+      end
+    rescue => e
+      raise CompressionError, "Failed to compress data using #{algorithm.inspect}. Error: #{e.message}"
+    end
+
+    def self.decompress(data, algorithm)
+      validate_algorithm!(algorithm)
+
+      case algorithm.to_s
+      when "gzip"
+        Zlib.gunzip(data)
+      when "deflate"
+        Zlib::Inflate.inflate(data)
+      end
+    rescue => e
+      raise DecompressionError, "Failed to decompress data using #{algorithm.inspect}. Error: #{e.message}"
+    end
+
+    def self.validate_algorithm!(algorithm)
+      return if SUPPORTED_ALGORITHMS.include?(algorithm.to_s)
+
+      raise ArgumentError, "Unsupported compression algorithm: #{algorithm.inspect}. Expected one of: #{SUPPORTED_ALGORITHMS.inspect}"
+    end
+    private_class_method :validate_algorithm!
+  end
+end

data/lib/shared_broker/dlq/redriver.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module SharedBroker
+  module DLQ
+    class Redriver
+      def self.redrive(client, dlq_name, original_topic, limit: nil)
+        adapter = client.send(:resolve_adapter, original_topic)
+        adapter.redrive_dlq(dlq_name, original_topic, limit: limit)
+      end
+    end
+  end
+end

data/lib/shared_broker/middlewares/idempotency.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module SharedBroker
+  module Middlewares
+    class Idempotency
+      class MemoryStore
+        def initialize
+          @store = {}
+        end
+
+        def exists?(key)
+          prune
+          @store.key?(key)
+        end
+
+        def write(key, value, expires_in: 3600)
+          @store[key] = Time.now + expires_in
+        end
+
+        def prune
+          now = Time.now
+          @store.delete_if { |_, expiry| expiry < now }
+        end
+      end
+
+      def initialize(store: nil, expires_in: 3600)
+        @store = store || MemoryStore.new
+        @expires_in = expires_in
+      end
+
+      def call(topic, message, metadata)
+        return yield unless metadata[:operation] == :subscribe
+
+        correlation_id = metadata[:correlation_id] || message[:_correlation_id]
+        return yield unless correlation_id
+
+        key = "shared_broker:idempotency:#{topic}:#{correlation_id}"
+        return if duplicate?(key)
+
+        mark_processed(key)
+        yield
+      end
+
+      private
+
+      def duplicate?(key)
+        if @store.respond_to?(:exist?)
+          @store.exist?(key)
+        elsif @store.respond_to?(:exists?)
+          @store.exists?(key)
+        else
+          false
+        end
+      end
+
+      def mark_processed(key)
+        @store.write(key, true, expires_in: @expires_in)
+      rescue ArgumentError
+        @store.write(key, @expires_in)
+      end
+    end
+  end
+end

data/lib/shared_broker/schema_registry/providers/http.rb

@@ -29,19 +29,42 @@ module SharedBroker
 
         def clear_cache
           @cache.clear
+          if SharedBroker.cache_store.respond_to?(:clear)
+            SharedBroker.cache_store.clear
+          end
         end
 
         private
 
         def fetch_schema(topic)
-          cached = @cache[topic.to_s]
-          return cached[:schema] if cached && cached[:expires_at] > Time.now
+          cache_key = "shared_broker:schema:#{topic}"
+          cached_schema = read_cache(cache_key)
+          return cached_schema if cached_schema
 
           schema = download_schema(topic)
-          @cache[topic.to_s] = { schema: schema, expires_at: Time.now + @cache_ttl } if schema
+          write_cache(cache_key, schema) if schema
           schema
         end
 
+        def read_cache(key)
+          if SharedBroker.cache_store
+            return SharedBroker.cache_store.read(key)
+          end
+
+          cached = @cache[key]
+          return cached[:schema] if cached && cached[:expires_at] > Time.now
+          nil
+        end
+
+        def write_cache(key, value)
+          if SharedBroker.cache_store
+            SharedBroker.cache_store.write(key, value, expires_in: @cache_ttl)
+            return
+          end
+
+          @cache[key] = { schema: value, expires_at: Time.now + @cache_ttl }
+        end
+
         def download_schema(topic)
           uri = URI("#{@base_url}/schemas/#{topic}.json")
           request = Net::HTTP::Get.new(uri)

data/lib/shared_broker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SharedBroker
-  VERSION = "1.6.0"
+  VERSION = "1.8.0"
 end

data/lib/shared_broker.rb

@@ -9,10 +9,13 @@ require_relative "shared_broker/schema_registry/providers/http"
 require_relative "shared_broker/validation"
 require_relative "shared_broker/cipher"
 require_relative "shared_broker/key_provider"
+require_relative "shared_broker/compressor"
 require_relative "shared_broker/concurrency/semaphore"
 require_relative "shared_broker/concurrency/limiter"
 require_relative "shared_broker/middleware_pipeline"
 require_relative "shared_broker/middlewares/open_telemetry_propagation"
+require_relative "shared_broker/middlewares/idempotency"
+require_relative "shared_broker/dlq/redriver"
 require_relative "shared_broker/adapters/base"
 require_relative "shared_broker/adapters/in_memory"
 require_relative "shared_broker/adapters/rabbit_mq"
@@ -20,13 +23,41 @@ require_relative "shared_broker/adapters/kafka"
 require_relative "shared_broker/adapters/redis"
 
 module SharedBroker
+  ShutdownError = Class.new(StandardError)
+
   class << self
-    attr_accessor :encryption_key, :key_provider
+    attr_accessor :encryption_key, :key_provider, :compression_algorithm, :compression_threshold, :cache_store, :shutdown_requested, :registered_clients
   end
 
   # Default key for development/test if not set
   @encryption_key = ENV.fetch("SHARED_BROKER_ENCRYPTION_KEY") { "a" * 32 }
   @key_provider = nil
+  @compression_algorithm = nil
+  @compression_threshold = 1024
+  @cache_store = nil
+  @shutdown_requested = false
+  @registered_clients = []
+  @registered_clients_mutex = Mutex.new
+
+  def self.register_client(client)
+    @registered_clients_mutex.synchronize do
+      @registered_clients << client
+    end
+  end
+
+  def self.shutdown!(timeout: 10)
+    @shutdown_requested = true
+    
+    threads = @registered_clients_mutex.synchronize do
+      @registered_clients.flat_map(&:active_threads)
+    end
+
+    threads.each { |t| t.join(timeout) }
+  end
+
+  def self.reset_shutdown!
+    @shutdown_requested = false
+  end
 
   class Client
     attr_reader :circuit_breaker, :middleware_pipeline, :adapters, :routing
@@ -37,6 +68,21 @@ module SharedBroker
       resolved_middlewares = middlewares || [SharedBroker::Middlewares::OpenTelemetryPropagation.new]
       @middleware_pipeline = MiddlewarePipeline.new(resolved_middlewares)
       @key_provider = key_provider
+      @running_threads = []
+      @running_threads_mutex = Mutex.new
+      SharedBroker.register_client(self)
+    end
+
+    def register_thread(thread)
+      @running_threads_mutex.synchronize do
+        @running_threads << thread
+      end
+    end
+
+    def active_threads
+      @running_threads_mutex.synchronize do
+        @running_threads.select(&:alive?)
+      end
     end
 
     def publish(topic, message, correlation_id: nil)
@@ -51,6 +97,24 @@ module SharedBroker
       end
     end
 
+    def publish_batch(topic, messages, correlation_id: nil)
+      unless messages.is_a?(Array)
+        raise ArgumentError, "Expected messages to be an Array, got #{messages.class} with value #{messages.inspect}"
+      end
+
+      processed_messages = messages.map do |message|
+        SharedBroker::Validation.validate!(topic, message)
+        SharedBroker::Cipher.encrypt(message, active_key_provider, topic: topic)
+      end
+
+      metadata = { correlation_id: correlation_id, operation: :publish_batch }
+      @middleware_pipeline.execute(topic, processed_messages, metadata) do
+        @circuit_breaker.run do
+          resolve_adapter(topic).publish_batch(topic, processed_messages, correlation_id: correlation_id)
+        end
+      end
+    end
+
     def subscribe(topic, queue_name, max_retries: 3, backoff_base: 2, max_concurrency: nil, backpressure_check: nil, backpressure_backoff: 1.0, &block)
       limiter = SharedBroker::Concurrency::Limiter.new(
         max_concurrency: max_concurrency,
@@ -58,8 +122,12 @@ module SharedBroker
         backpressure_backoff: backpressure_backoff
       )
 
-      resolve_adapter(topic).subscribe(topic, queue_name, max_retries: max_retries, backoff_base: backoff_base) do |raw_message|
+      res = resolve_adapter(topic).subscribe(topic, queue_name, max_retries: max_retries, backoff_base: backoff_base) do |raw_message|
+        raise SharedBroker::ShutdownError, "Shutdown requested" if SharedBroker.shutdown_requested
+
         limiter.run do
+          raise SharedBroker::ShutdownError, "Shutdown requested" if SharedBroker.shutdown_requested
+
           decrypted_msg = SharedBroker::Cipher.decrypt(raw_message, active_key_provider, topic: topic)
           SharedBroker::Validation.validate!(topic, decrypted_msg)
 
@@ -69,6 +137,9 @@ module SharedBroker
           end
         end
       end
+
+      register_thread(res) if res.is_a?(Thread)
+      res
     end
 
     private

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: shared_broker
 version: !ruby/object:Gem::Version
-  version: 1.6.0
+  version: 1.8.0
 platform: ruby
 authors:
 - Wesley Lima
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-06-16 00:00:00.000000000 Z
+date: 2026-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -155,10 +155,13 @@ files:
 - lib/shared_broker/adapters/redis.rb
 - lib/shared_broker/cipher.rb
 - lib/shared_broker/circuit_breaker.rb
+- lib/shared_broker/compressor.rb
 - lib/shared_broker/concurrency/limiter.rb
 - lib/shared_broker/concurrency/semaphore.rb
+- lib/shared_broker/dlq/redriver.rb
 - lib/shared_broker/key_provider.rb
 - lib/shared_broker/middleware_pipeline.rb
+- lib/shared_broker/middlewares/idempotency.rb
 - lib/shared_broker/middlewares/open_telemetry_propagation.rb
 - lib/shared_broker/schema_registry.rb
 - lib/shared_broker/schema_registry/providers/http.rb