mqtt-core 0.0.1.ci.release

data/lib/mqtt/core/client/memory_session_store.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require_relative 'qos2_session_store'
+require 'concurrent_monitor'
+
+module MQTT
+  module Core
+    class Client
+      # A Session Store held in memory for the duration of this process only
+      # Supports QoS 2 to recover from network errors but cannot recover from a crash
+      class MemorySessionStore < Qos2SessionStore
+        # @!visibility private
+        # Allow server-assigned client ids (and anonymous sessions in MQTT 3)
+        attr_writer :client_id
+
+        def initialize(expiry_interval: nil, client_id: '')
+          super
+          @clean = true
+          # outgoing packet store, waiting for ACK
+          @store = {}
+        end
+
+        def connected!
+          @expiry_timeout = ConcurrentMonitor::TimeoutClock.new(expiry_interval)
+          @clean = false
+        end
+
+        def disconnected!
+          # We can start the timeout here rather than tracking all packet activity because there is no
+          # method to recover an in memory session from a full crash.
+          @expiry_timeout&.start!
+        end
+
+        def expired?
+          @expiry_timeout&.expired?
+        end
+
+        def clean?
+          @clean
+        end
+
+        def store_packet(packet, replace: false)
+          raise KeyError, 'packet id already exists' if !replace && stored_packet?(packet.id)
+
+          @store[packet.id] = packet
+        end
+
+        def delete_packet(id)
+          @store.delete(id)
+        end
+
+        def stored_packet?(id)
+          @store.key?(id)
+        end
+
+        def retry_packets
+          @store.values
+        end
+
+        def qos2_recover
+          [] # nothing to recover
+        end
+
+        def qos_unhandled_packets
+          {} # nothing was persisted
+        end
+
+        def store_qos_received(packet, unique_id)
+          # For memory store, we don't need to persist received packets
+          # This is just for tracking during the current session
+        end
+
+        def qos_handled(packet, unique_id)
+          # For memory store, we don't need to persist handled status
+          # This is just for tracking during the current session
+        end
+
+        def restart_clone
+          self # don't actually clone.
+        end
+      end
+    end
+  end
+end

data/lib/mqtt/core/client/qos0_session_store.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require_relative 'session_store'
+
+module MQTT
+  module Core
+    class Client
+      # A minimal Session Store limited to handling QoS0 packets only
+      # Always a clean session on restart
+      class Qos0SessionStore < SessionStore
+        # @!visibility private
+        # Allow server-assigned client ids (and anonymous sessions in MQTT 3)
+        attr_writer :client_id
+
+        def initialize(client_id: '')
+          super(expiry_interval: 0, client_id:)
+          @store = nil
+        end
+
+        def max_qos
+          0
+        end
+
+        def connected!
+          @store = Set.new
+        end
+
+        def disconnected!
+          @store&.clear
+        end
+
+        # Always use a clean session
+        def clean?
+          true
+        end
+
+        # We still need to track packet id for subscribe/unsubscribe
+        def store_packet(packet, **)
+          raise KeyError, 'packet id already exists' unless @store.add?(packet.id)
+        end
+
+        def delete_packet(id)
+          @store.delete(id)
+        end
+
+        def stored_packet?(id)
+          @store.include?(id)
+        end
+
+        def retry_packets
+          []
+        end
+      end
+    end
+  end
+end

data/lib/mqtt/core/client/qos2_session_store.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require_relative 'session_store'
+module MQTT
+  module Core
+    class Client
+      # Session store that supports QoS 1/2 messages
+      # @abstract
+      class Qos2SessionStore < SessionStore
+        class SessionNotRecoverable < Error; end
+
+        def max_qos
+          2
+        end
+
+        # Initialise recovery of the persistent session.
+        # @!method qos2_recover
+        # @raise SessionNotRecoverable if there are unhandled QOS2 messages not explicitly marked to retry
+        # @return [Array<Integer>] list of QOS 2 packets ids waiting for PUBREL
+
+        # Load the unhandled QoS 1 and 2 PUBLISH packets that should be re-delivered for this session
+        # @!method qos_unhandled_packets(&deserializer)
+        # @return [Hash<Packet,String>] map of deserialized packets to their unique session id
+
+        # Persist a received QoS 1/2 packet
+        # @!method store_qos_received(packet, unique_id)
+
+        # Check if a QoS2 PUBLISH packet has previously been received
+        # @!method qos2_published?(packet_id)
+        #  @param [Integer] packet_id
+        #  @return [Boolean] true if this packet_id was already received but is still waiting for PUBREL.
+
+        # Release a received packet id (before we send PUBCOMP)
+        # @!method qos2_release(packet_id)
+
+        # Mark the received packet as handled from the client application perspective
+        # @!method qos_handled(packet, unique_id)
+
+        # Mark a previously received QOS1/2 packet as handled
+        # @!method qos_handled(packet, unique_id)
+        #   @return [void]
+      end
+    end
+  end
+end

data/lib/mqtt/core/client/qos_tracker.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+module MQTT
+  module Core
+    class Client
+      # Modules for tracking received QOS packets in a Session
+      module QosTracker
+        def qos_initialize
+          @birth_complete = false
+
+          # This is only manipulated from the 'receive' thread, does not require synchronisation
+          @qos2_pending = Set.new
+          @qos2_pending.merge(session_store.qos2_recover) if session_store.max_qos == 2
+
+          # QOS packets - count down tracking
+          @qos_packets = {}
+          qos_load { |io| deserialize(io) } if session_store.max_qos.positive?
+        end
+
+        # Have we already received a QOS2 packet with this packet id?
+        def qos2_published?(id)
+          !@qos2_pending.add?(id)
+        end
+
+        # Called when a QOS1/2 PUBLISH arrives from the server and has been matched against available subscriptions
+        def qos_received(packet, subs)
+          if birth_complete? && subs.zero?
+            log.warn("No subscription for #{packet.topic_name}")
+            return
+          end
+
+          pkt_info = { unique_id: format('%013d', Time.now.to_f * 1000), counter: subs, subscribed: subs.positive? }
+          session_store.store_qos_received(packet, pkt_info[:unique_id])
+          synchronize { @qos_packets[packet] = pkt_info }
+        end
+
+        # Release the pending qos2 packet (return true if we had previously seen it)
+        # rubocop:disable Naming/PredicateMethod
+        def qos2_release(id)
+          session_store.qos2_release(id)
+          !!@qos2_pending.delete?(id)
+        end
+        # rubocop:enable Naming/PredicateMethod
+
+        # Called when a new subscription arrives.
+        # @return [Array<PUBLISH>] a list of matching packets to enqueue on the subscription.
+        def qos_subscribed(&matcher)
+          return [] if @birth_complete
+
+          synchronize do
+            return if @birth_complete
+
+            @qos_packets.filter_map do |packet, data|
+              next false unless matcher.call(packet)
+
+              data[:counter] += 1
+              data[:subscribed] = true
+              packet
+            end
+          end
+        end
+
+        # Called when topics are explicitly unsubscribed
+        def qos_unsubscribed(&matcher)
+          return if @birth_complete
+
+          synchronize do
+            return if @birth_complete
+
+            @qos_packets.delete_if { |packet, data| !data[:subscribed] && matcher.call(packet) }
+          end
+        end
+
+        # Called when a Subscription completes handling of a QoS1/2 message.
+        def handled!(packet)
+          unique_id = synchronize do
+            counter = (@qos_packets[packet][:counter] -= 1)
+            return unless counter.zero? && @birth_complete
+
+            @qos_packets.delete(packet)[:unique_id]
+          end
+
+          session_store.qos_handled(packet, unique_id)
+        end
+
+        # Complete the birth phase and process all pending zero-counter messages
+        def birth_complete!
+          handled = synchronize do
+            @birth_complete = true
+
+            @qos_packets.select { |_pkt, data| data[:counter].zero? }.tap do |handled|
+              handled.each do |pkt, data|
+                log.warn("No subscription for #{pkt.topic_name}") unless data[:subscribed]
+                @qos_packets.delete(pkt)
+              end
+            end
+          end
+
+          handled.each { |(packet, data)| session_store.qos_handled(packet, data[:unique_id]) }
+        end
+
+        # @return [Boolean] True if the birth phase is complete
+        def birth_complete?
+          @birth_complete || synchronize { @birth_complete }
+        end
+
+        private
+
+        # These are unhandled QoS 1/2 packets, mapped to their unique id (timestamps)
+        # @return [Array<Packet>] the list of unhandled packets to send to subscriptions as they connect
+        def qos_load(&)
+          @qos_packets.merge!(session_store.qos_unhandled_packets(&).transform_values do |v|
+            { unique_id: v, counter: 0 }
+          end)
+        end
+      end
+    end
+  end
+end

data/lib/mqtt/core/client/retry_strategy.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require_relative '../../errors'
+
+module MQTT
+  module Core
+    class Client
+      # Implements retry strategy with configurable backoff and jitter
+      class RetryStrategy
+        include Logger
+
+        # @return [Integer] maximum number of retry attempts (default 0 = retry forever)
+        attr_reader :max_attempts
+
+        # @return [Float] initial interval in seconds to wait before retrying (default 1.0)
+        attr_reader :base_interval
+
+        # @return [Float] multiplier for exponential backoff (default 1.5)
+        attr_reader :backoff
+
+        # @return [Float] maximum interval in seconds (default 300)
+        attr_reader :max_interval
+
+        # @return [Float] percentage of random jitter to add to retry intervals, 0-100 (default 25.0)
+        attr_reader :jitter
+
+        def initialize(max_attempts: 0, base_interval: 1.0, backoff: 1.5, max_interval: 300, jitter: 25.0)
+          @max_attempts = max_attempts.to_i
+          @base_interval = base_interval.to_f
+          @backoff = backoff.to_f
+          @max_interval = max_interval.to_f
+          @jitter = jitter.to_f
+        end
+
+        # This is the retry strategy interface. Caller will retry on completion of this method
+        # if it does not raise an exception.
+        # @param [Integer] retry_count
+        # @param [Proc] raiser
+        # @raise [StandardError] the error raised by raiser if retry count has exceeded max attempts
+        # @return [Integer] slept duration in seconds
+        def retry!(retry_count, &raiser)
+          raiser.call
+        rescue Error::Retriable, *RETRIABLE_NETWORK_ERRORS => e
+          raise e if max_attempts.positive? && retry_count >= max_attempts
+
+          log.error(e)
+          duration = calculate_retry_duration(retry_count)
+          log.warn { "Retry attempt #{retry_count} in #{duration.round(2)}s" }
+          sleep(duration)
+        end
+
+        private
+
+        # Calculate the retry duration with exponential backoff and jitter
+        def calculate_retry_duration(retry_count)
+          base_duration = [base_interval * (backoff**retry_count), max_interval].min
+          max_jitter_amount = (base_duration * jitter / 100.0)
+          base_duration + rand(-max_jitter_amount..max_jitter_amount)
+        end
+      end
+    end
+  end
+end

data/lib/mqtt/core/client/session.rb

@@ -0,0 +1,195 @@
+# frozen_string_literal: true
+
+require_relative 'socket_factory'
+require_relative 'qos_tracker'
+require_relative '../../logger'
+require 'forwardable'
+require 'concurrent_monitor'
+
+module MQTT
+  module Core
+    class Client
+      # @!visibility private
+      # Common session handling across MQTT protocol versions
+      #   * responsible for building packets from user data for PUBLISH, SUBSCRIBE, UNSUBSCRIBE
+      #   * handles the protocol for a given client id - packet identifier assignment, QOS handling
+      #   * spans connections - retries by resending packets from the unacknowledged packet store
+      # @abstract - MQTT Protocol version specific session behaviour in concrete implementations
+      class Session
+        extend Forwardable
+        include Logger
+        include ConcurrentMonitor
+        include QosTracker
+
+        def initialize(client:, monitor:, session_store:)
+          @client = client
+          @monitor = monitor.new_monitor
+          @session_store = session_store
+
+          qos_initialize
+        end
+
+        def connect_data(**connect)
+          check_session_managed_fields(:connect, connect, :client_id)
+          { client_id: session_store.client_id }
+        end
+
+        def disconnect_data(**_disconnect)
+          {}
+        end
+
+        def expired!(clean: clean?)
+          return if clean || !session_store.expired?
+
+          raise SessionExpiacks.deletered, "Session #{session_store} for '#{client_id}' has expired"
+        end
+
+        def connected!(_connect, connack)
+          return session_store.connected! if connack.session_present? || clean?
+
+          expired!(clean: false)
+          raise SessionNotPresent, "Server does not have a session for '#{client_id}'"
+        end
+
+        # Sending a message, store a duplicate packet in the packet store for resending
+        def publish(qos: 0, **publish)
+          check_session_managed_fields(__method__, publish, :packet_identifier, :dup)
+          validate_qos!(qos)
+
+          dup = packet_with_id(:publish, qos:, dup: true, **publish) if qos.positive?
+
+          p = build_packet(:publish, qos:, dup: false, packet_identifier: dup&.id, **publish)
+          yield p
+        end
+
+        def subscribe(**subscribe)
+          check_session_managed_fields(__method__, subscribe, :packet_identifier)
+          pkt = packet_with_id(:subscribe, **subscribe)
+          validate_qos!(pkt.max_qos)
+
+          yield pkt
+        end
+
+        def unsubscribe(**unsubscribe)
+          check_session_managed_fields(__method__, unsubscribe, :packet_identifier)
+          yield packet_with_id(:unsubscribe, **unsubscribe)
+        end
+
+        # Receiving a message
+        def handle_publish(packet)
+          # Notify the client the message has been received (unless we are qos and have already seen it)
+          unless packet.qos == 2 && qos2_published?(packet.id)
+            matched_subs = receive_publish(packet)
+            qos_received(packet, matched_subs.size) if packet.qos.positive?
+            matched_subs.each { |sub| sub.put(packet) }
+          end
+
+          return unless packet.qos.positive?
+
+          # Build and send the appropriate ACK packet
+          ack = build_packet(packet.qos == 2 ? :pubrec : :puback, packet_identifier: packet.id)
+          push_packet(ack)
+        end
+
+        # replace the stored publish message with a pubrel message
+        def handle_pubrec(packet)
+          packet.success!
+          # We don't need to sync this as the only possible conflict would be a protocol error
+          # anyway, so we will always see pub_rec after publish.
+          pubrel = store_packet(qos2_response(:pubrel, packet.id, stored_packet?(packet.id)), replace: true)
+          push_packet(pubrel)
+        rescue ResponseError => _e
+          release_packet(packet)
+        end
+
+        def handle_pubrel(packet)
+          push_packet(qos2_response(:pubcomp, packet.id, qos2_release(packet.id)))
+        end
+
+        def release_packet(packet)
+          # Notify the client the request has been acknowledged
+          receive_ack(packet)
+        ensure
+          delete_packet(packet.id)
+        end
+
+        alias handle_puback release_packet
+        alias handle_pubcomp release_packet
+        alias handle_suback release_packet
+        alias handle_unsuback release_packet
+        private :release_packet
+
+        MAX_PACKET_ID = 65_535
+
+        def max_packet_id
+          MAX_PACKET_ID
+        end
+
+        def_delegators :session_store, :disconnected!, :client_id, :retry_packets, :expiry_interval=
+
+        private
+
+        attr_reader :session_store
+
+        # Client helpers and callbacks
+        def_delegators :@client, :build_packet, :deserialize, :push_packet, :receive_ack, :receive_publish
+
+        # Session store interface
+        def_delegators :session_store, :clean?, :expired?, :stored_packet?, :store_packet, :delete_packet,
+                       :max_qos, :validate_qos!, :qos2_published?, :qos2_release
+
+        # used in handle PUBREC/PUBREL to handle unknown packet id errors
+        def qos2_response(response_name, id, exists, **data)
+          raise ProtocolError, "Packet id #{id} does not exist in session for #{client_id}" unless exists
+
+          build_packet(response_name, packet_identifier: id, **data)
+        end
+
+        def packet_with_id(packet_type, **packet_data)
+          next_packet_id { |id| build_packet(packet_type, packet_identifier: id, **packet_data) }
+        end
+
+        # Generates unique packet IDs for a single client session.
+        # Uses random allocation and initial optimistic collision check before any synchronisation around use
+        # of the session store.
+        # The cost of generating a random number is low compared to the RTT for QOS 1/2 flow.
+        def next_packet_id(&)
+          init_id = rand(1..max_packet_id)
+          attempts = 0
+          loop do
+            result = claim_id(((init_id + attempts) % max_packet_id) + 1, &)
+
+            return result if result
+
+            packet_id_backoff(attempts += 1)
+          end
+        end
+
+        def claim_id(id)
+          return nil if stored_packet?(id)
+
+          packet = yield id
+
+          synchronize { packet.tap { store_packet(packet) } unless stored_packet?(packet.id) }
+        end
+
+        def packet_id_backoff(attempts)
+          return unless attempts >= max_packet_id
+
+          # if we've done a whole lap of available ids, start backing off...
+          log.warn { "Packet id contention: #{attempts} attempts" } if (attempts % max_packet_id).zero?
+          sleep(0.01 * Math.log(attempts - max_packet_id + 2))
+        end
+
+        def check_session_managed_fields(packet_type, data, *invalid_fields)
+          data.delete_if do |k, _v|
+            next false unless invalid_fields.include?(k)
+
+            log.warn { "#{packet_type}: Ignoring session managed property #{k}" }
+            true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mqtt/core/client/session_store.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+require_relative 'client_id_generator'
+
+module MQTT
+  module Core
+    class Client
+      # @abstract The session store interface
+      # The Session Store is responsible for...
+      #  * Keeping track of packet ids and ACK status of packets that we are sending.
+      #    These incomplete packets are retried on reconnecting to an established session.
+      #  * Meeting the Quality of Service guarantees for received PUBLISH packets.
+      class SessionStore
+        include ClientIdGenerator
+
+        # @!visibility private
+        def self.extract_uri_params(uri_params)
+          # amazonq-ignore-next-line
+          uri_params.slice(*%w[client_id expiry_interval]).transform_keys(&:to_sym).tap do |params|
+            params[:expiry_interval] = Integer(params[:expiry_interval]) if params.key?(:expiry_interval)
+          end
+        end
+
+        # Raised from PUBLISH or SUBSCRIBE if the session store does not support the requested QoS level
+        class QoSNotSupported < Error; end
+
+        # Maximum Session Expiry Interval
+        # The spec says this means 'never' expire, but it also equates to 136 years so there is no practical
+        # need for special handling.
+        MAX_EXPIRY_INTERVAL = 0xFFFFFFFF
+
+        include MQTT::Logger
+
+        # @!attribute [r] expiry_interval
+        #  @return [Integer] duration in seconds before the session data expires after disconnect.
+        # @note the writer method is api private for MQTT 5.0 sever assigned expiry.
+        attr_accessor :expiry_interval
+
+        # @!attribute [r] client_id
+        #   @return [String]
+        # @note the writer method (if defined in subclass) is ap private for MQTT 5.0 server assigned client id
+        attr_reader :client_id
+
+        # @param [String|nil] client_id
+        #   * Empty string (default) is a session local anonymous or auto-assigned id (version-dependent handling)
+        #   * Explicitly `nil` to generate random id
+        #   * Otherwise a valid client_id for the server
+        # @param [Integer|nil] expiry_interval
+        def initialize(expiry_interval:, client_id:)
+          init_client_id(client_id)
+          init_expiry_interval(expiry_interval)
+        end
+
+        def validate_qos!(requested_qos)
+          return if requested_qos <= max_qos
+
+          raise QoSNotSupported, "QoS #{requested_qos} is not supported by #{self.class}"
+        end
+
+        # @!method retry_packets
+        #   @return [Array<Packet>] the list of unacknowledged packets to resend on re-connect
+
+        # @!method clean?
+        #   @return [Boolean] true if reconnection should establish a new session
+
+        # @!method connected!
+        #   Connection has been acknowledged server-assigned client_id and expiry_interval are available
+        #   @return [void]
+
+        # @!method disconnected!
+        #   Connection has been disconnected
+        #   @return [void]
+
+        # @!method expired?
+        #   @return [Boolean] true if the {expiry_interval} has passed since the latest activity
+
+        # @!method stored_packet?(packet_id)
+        #   @param [Integer] packet_id
+        #   @return [Boolean] true if the packet_id is in use and waiting acknowledgement
+
+        # @!method store_packet(packet, replace: false)
+        #   @param [Packet] packet the packet (with packet_identifier) to store
+        #   @param [Boolean] replace allow overwriting of packet_id (part of QOS2 flow)
+        #   @return [void]
+        #   @raise [KeyError] if packet_id is in use and replace is not true
+
+        # Discard the packet with this packet id
+        # @!method release_packet(packet_id)
+        #   @param [Integer] packet_id
+        #   @return [void]
+
+        # Maximum Quality of Service level supported by this session store
+        # @!method max_qos
+        #   @return [Integer]
+
+        private
+
+        def init_client_id(client_id)
+          client_id ||= generate_client_id
+          @client_id = client_id
+
+          return if client_id.length.positive? || allow_server_assigned_client_id?
+
+          raise ArgumentError, "#{self.class} requires a non-empty client id"
+        end
+
+        # client id can only be empty if the session store allows server-assigned client ids
+        # which it can do by defining a writer for :client_id
+        def allow_server_assigned_client_id?
+          respond_to?(:client_id=)
+        end
+
+        def init_expiry_interval(expiry_interval)
+          expiry_interval ||= MAX_EXPIRY_INTERVAL
+          unless expiry_interval.between?(0, MAX_EXPIRY_INTERVAL)
+            raise ArgumentError, "expiry_interval must be between 0 and #{MAX_EXPIRY_INTERVAL}"
+          end
+
+          if expiry_interval.zero? && max_qos.positive?
+            raise ArgumentError, "#{self.class} requires a non-zero expiry_interval"
+          end
+
+          @expiry_interval = expiry_interval
+        end
+      end
+    end
+  end
+end