mqtt-core 0.0.1.ci.release

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 03cdb9467d4797a91a9751dd5f0421d4f0a231a3084314dbbf9b58d34812c7a8
+  data.tar.gz: 17800776d5fc4fd0698758bbcc79fc29d64f65a7c1d7d3ce91265faf55646c2a
+SHA512:
+  metadata.gz: eedd90f6fe41a5e72ea11a990507c168b970e203af0a30d079ca70a76dc71b1f8bdac6c915b84b7de4abb7097ba99593683938c9e23e5f14a869d4849950ea8b
+  data.tar.gz: ced170d3f87f4fee052ddc4e5e9f282b3d3be257c0eb2c2b50278e3577e087d80f9fe1b7eb1e740474224f1244ca2b7e2f14f35c8c88a653dd80f9498b27fd57

data/lib/mqtt/core/client/acknowledgement.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require_relative '../../errors'
+
+module MQTT
+  module Core
+    class Client
+      # In MQTT the acknowledgement is always a packet
+      class Acknowledgement < ConcurrentMonitor::Future
+        def initialize(packet, monitor:, &deferred)
+          @packet = packet
+          @deferred = deferred
+          super(monitor:)
+        end
+
+        #  @return [MQTT::Packet] the packet that this acknowledgement is for
+        attr_reader :packet
+
+        def cancel(exception)
+          reject(exception)
+        end
+
+        def fulfill(ack_packet)
+          super(@deferred ? @deferred.call(ack_packet) : [packet, ack_packet])
+        rescue StandardError => e
+          reject(e)
+        end
+      end
+    end
+  end
+end

data/lib/mqtt/core/client/client_id_generator.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+
+module MQTT
+  module Core
+    class Client
+      # Client ID Helpers
+      module ClientIdGenerator
+        # Base method to generate a client id consisting of a prefix and random sequence of alphanumerics
+        # @param [String] prefix
+        # @param [Integer] length
+        def generate_client_id(prefix: 'rb', length: 21)
+          "#{prefix}#{SecureRandom.alphanumeric(length)}"
+        end
+      end
+    end
+  end
+end

data/lib/mqtt/core/client/connection.rb

@@ -0,0 +1,193 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+require 'concurrent_monitor'
+require_relative '../../logger'
+
+module MQTT
+  module Core
+    class Client
+      # @!visibility private
+      # Represents a single network connection to an MQTT Server
+      class Connection
+        extend Forwardable
+        include Logger
+        include ConcurrentMonitor
+
+        def initialize(client:, session:, io:, monitor:)
+          @client = client
+          @monitor = monitor.new_monitor
+          @session = session
+          @io = io
+        end
+
+        # Connect runs synchronously reading packets and getting responses
+        # until the connection handshakes are complete.
+        def connect(**connect_data)
+          [send_packet(connect_packet(**connect_data)), complete_connection(receive_packet)]
+        end
+
+        # Build and send a disconnect packet
+        def disconnect(exception = nil, **disconnect)
+          if exception
+            io.close
+          else
+            yield disconnect_packet(**disconnect)
+          end
+        end
+
+        # @yield(packet, action:)
+        # @yieldparam [Packet] packet the packet to handle
+        # @yieldparam [Symbol] action the action to take,
+        #   :send to forward a packet as part of protocol flow
+        #   :acknowledge the packet is a response to a client request
+        #   :handle other types of packets
+        # @yieldreturn [void]
+        def receive_loop
+          loop { return unless handle_packet(receive_packet) }
+        ensure
+          io.close_read if io.respond_to?(:close_read) && !io.closed?
+        end
+
+        def send_loop
+          packet = pingreq_packet
+          loop do
+            packet = (yield ping_remaining(packet)) || pingreq_packet
+            return unless send_packet(packet)
+          end
+        ensure
+          io.close_write if io.respond_to?(:close_write) && !io.closed?
+        end
+
+        # Session handles publish, subscribe, and unsubscribe, but versioned subclasses may override
+        def_delegators :session, :publish, :subscribe, :unsubscribe
+
+        def connected?
+          !io.closed?
+        end
+
+        def close
+          io.close
+        end
+
+        attr_reader :keep_alive, :ping_clock
+
+        private
+
+        attr_reader :io, :session, :client
+
+        def_delegators :session, :max_packet_id, :generate_client_id, :retry_packets
+        def_delegators :client, :build_packet, :push_packet, :deserialize
+
+        def ping_remaining(prev_packet)
+          return 0 unless ping_clock
+
+          # if client is only sending QOS 0 publish, we need to send a ping to
+          # prevent the recv loop timing out, and thus proving we have bidirectional connectivity
+          prev_packet = pingreq_packet.tap { |ping| send_packet(ping) } if ping_clock.expired?
+          ping_clock.start! unless prev_packet.packet_name == :publish && prev_packet.qos.zero?
+
+          ping_clock.remaining
+        end
+
+        def send_packet(packet)
+          log.debug { "SEND: #{packet}" }
+          client.handle_event(:send, packet == :eof ? nil : packet)
+          return false if packet == :eof
+
+          packet.serialize(io)
+
+          # cause loop to stop after disconnect is sent
+          return false if packet.packet_name == :disconnect
+
+          packet
+        end
+
+        def receive_packet
+          deserialize(io)
+            .tap { |p| log.debug { "RECV: #{p || :eof}" } }
+            .tap { |p| client.handle_event(:receive, p) }
+        end
+
+        # Allow version-specific subclass to handle additional connection flow (eg auth in 5.0)
+        def complete_connection(received_packet)
+          raise EOFError unless received_packet
+
+          received_packet.tap do |connack|
+            handle_connack(connack)
+          end
+        end
+
+        def connect_packet(**connect)
+          build_packet(:connect, **connect_data(**connect)).tap do |p|
+            self.keep_alive = p.keep_alive
+          end
+        end
+
+        def keep_alive=(val)
+          @keep_alive = val
+          @ping_clock = (val&.positive? ? ConcurrentMonitor::TimeoutClock.timeout(val) : nil)
+          io.timeout = (val&.positive? ? val * 2.0 : nil)
+        end
+
+        def connect_data(**connect)
+          connect.merge!(session.connect_data(**connect))
+        end
+
+        def pingreq_packet
+          @pingreq_packet ||= build_packet(:pingreq)
+        end
+
+        def disconnect_packet(**disconnect)
+          build_packet(:disconnect, **disconnect_data(**disconnect))
+        end
+
+        def disconnect_data(**disconnect)
+          disconnect.merge!(session.disconnect_data(**disconnect))
+        end
+
+        def handle_packet(packet)
+          # an empty packet means end of stream in the read loop, we'll try and gracefully inform the send loop
+          return handle_eof if !packet && !io.closed? && io.eof?
+          return nil unless packet
+
+          send("handle_#{packet.packet_name}", packet)
+          true
+        end
+
+        def handle_connack(packet)
+          raise ProtocolError, 'Unexpected packet type' unless packet.packet_name == :connack
+
+          packet.success!
+        end
+
+        def handle_pingresp(_packet)
+          # nothing to handle
+        end
+
+        # QOS 1/2 flow - when this client has sent a message via PUBLISH
+        def_delegators :session, :handle_puback, :handle_pubrec, :handle_pubcomp
+
+        # QOS 1/2 flow - when this client has received a message
+        def_delegators :session, :handle_publish, :handle_pubrel
+
+        # Other acks
+        def_delegators :session, :handle_suback, :handle_unsuback
+
+        # This is if the server explicitly sends a disconnect,  which would always expect to be an error
+        def handle_disconnect(packet)
+          packet.success!
+        ensure
+          io.close
+        end
+
+        # rubocop:disable Naming/PredicateMethod
+        def handle_eof
+          client.receive_eof
+          false
+        end
+        # rubocop:enable Naming/PredicateMethod
+      end
+    end
+  end
+end

data/lib/mqtt/core/client/enumerable_subscription.rb

@@ -0,0 +1,224 @@
+# frozen_string_literal: true
+
+require_relative 'subscription'
+
+module MQTT
+  module Core
+    class Client
+      # Enumerable subscription supporting iteration over received messages
+      #
+      # Method variants
+      #
+      #   - methods with aliases suffixed `messages` yield deconstructed topic, payload, and attributes
+      #   - methods suffixed with `packets` yield raw `PUBLISH` packets
+      #   - methods prefixed with `async` perform enumeration in a new thread
+      #   - methods suffixed with bang `!` ensure {#unsubscribe}
+      #   - methods prefixed with `lazy` return lazy enumerators for advanced chaining
+      #
+      class EnumerableSubscription < Subscription
+        include Enumerable
+
+        # @!visibility private
+        def put(packet)
+          handler.enqueue(packet)
+        end
+
+        # @!macro [new] yield_messages
+        #   @yield [topic, payload, **attributes]
+        #   @yieldparam [String] topic the message topic.
+        #   @yieldparam [String] payload the message payload.
+        #   @yieldparam [Hash<Symbol>] attributes additional `PUBLISH` packet attributes.
+        #   @yieldreturn [$0]
+
+        # @!macro [new] yield_packets
+        #   @yield [packet]
+        #   @yieldparam [Packet] packet a `PUBLISH` packet.
+        #   @yieldreturn [$0]
+
+        # @!macro [new] enum_return
+        #   @return [void] when block given
+        #   @return [Enumerator] an enumerator when no block given.
+
+        # @!macro [new] lazy_enum_return
+        #   @return [void] when block given
+        #   @return [Enumerator::Lazy] a lazy enumerator when no block given.
+
+        # @!macro [new] async_return
+        #   @return [self, ConcurrentMonitor::Task]
+        #     a pair containing self and the task that is iterating over the messages.
+
+        # @!macro [new] qos_note
+        #   @note QoS 1/2 packets are marked as completely handled in the session store when the given block completes.
+        #     If no block is given, completion is marked *before* the packet is returned.
+
+        # Get one packet, blocking until available
+        #
+        # @!macro qos_note
+        # @!macro yield_packets(Object)
+        # @return [Packet] a `PUBLISH` packet when no block given
+        # @return [Object] the block result when block given
+        # @return [nil] when unsubscribed or disconnected
+        def get_packet(&)
+          handle(handler.dequeue, &)
+        end
+
+        # Get one message, blocking until available
+        #
+        # @!macro qos_note
+        # @!macro yield_messages(Object)
+        # @return [String, String, Hash<Symbol>] topic, payload, and attributes when no block given
+        # @return [Object] the block result when block given
+        # @return [nil] when unsubscribed or disconnected
+        def get(&)
+          get_packet { |pkt| pkt.deconstruct_message(&) }
+        end
+        alias get_message get
+
+        # Read one packet, blocking until available, for use in loops
+        # @!macro yield_packets(Object)
+        # @return [Packet] a `PUBLISH` packet when no block given
+        # @return [Object] the block result when block given
+        # @raise [StopIteration] when unsubscribed or disconnected
+        def read_packet(&)
+          get_packet do |packet|
+            raise StopIteration unless packet
+
+            block_given? ? yield(packet) : packet
+          end
+        end
+
+        # Read one message, blocking until available, for use in loops
+        # @!macro yield_messages(Object)
+        # @return [String, String, Hash<Symbol>] topic, payload, and attributes when no block given
+        # @return [Object] the block result when block given
+        # @raise [StopIteration] when unsubscribed or disconnected
+        def read(&)
+          read_packet { |pkt| pkt.deconstruct_message(&) }
+        end
+        alias read_message read
+
+        # Enumerate packets
+        # @!macro yield_packets(void)
+        # @!macro enum_return
+        def each_packet(&)
+          return enum_for(__method__) unless block_given?
+
+          loop { read_packet(&) }
+        end
+
+        # Enumerate packets, ensuring {#unsubscribe}
+        # @!macro yield_packets(void)
+        # @!macro enum_return
+        def each_packet!(&) = enum_for!(__method__, &)
+
+        # Enumerate messages
+        # @!macro yield_messages(void)
+        # @!macro enum_return
+        def each
+          return enum_for(__method__) unless block_given?
+
+          each_packet { |pkt| yield(*pkt.deconstruct_message) }
+        end
+
+        alias each_message each
+
+        # Enumerate messages, ensuring {#unsubscribe}
+        # @!macro yield_messages(void)
+        # @!macro enum_return
+        def each!(&) = enum_for!(__method__, &)
+        alias each_message! each!
+
+        # Return a lazy enumerator for advanced chaining
+        # @return [Enumerator::Lazy<String, String, Hash>] lazy enumerator yielding [topic, payload, **attributes]
+        # @example
+        #   sub.lazy.select { |t, p| p.size > 100 }.map { |t, p| JSON.parse(p) }.take(5)
+        def lazy
+          each.lazy
+        end
+        alias lazy_messages lazy
+
+        # Return a lazy enumerator ensuring {#unsubscribe}
+        # @return [Enumerator::Lazy<String, String, Hash>] lazy enumerator yielding [topic, payload, **attributes]
+        # @example
+        #   sub.lazy!.select { |t, p| p.size > 100 }.map { |t, p| JSON.parse(p) }.take(5)
+        def lazy!
+          each!.lazy
+        end
+        alias lazy_messages! lazy!
+
+        # Return a lazy packet enumerator for advanced chaining
+        # @return [Enumerator::Lazy<Packet>] lazy enumerator yielding PUBLISH packets
+        def lazy_packets
+          each_packet.lazy
+        end
+
+        # Return a lazy packet enumerator with auto-unsubscribe
+        # @return [Enumerator::Lazy<Packet>] lazy enumerator yielding PUBLISH packets
+        def lazy_packets!
+          each_packet!.lazy
+        end
+
+        # Enumerate messages in a new thread
+        # @overload async(&)
+        # @!macro yield_messages(void)
+        # @!macro async_return
+        # @see each
+        def async(method = :each, &)
+          raise ArgumentError, 'block is required for async enumeration' unless block_given?
+
+          [self, client.async("sub.#{method}") { send(method, &) }]
+        end
+        alias async_messages async
+
+        # Enumerate messages in a new thread, ensuring {#unsubscribe}
+        # @!macro yield_messages(void)
+        # @!macro async_return
+        # @see each!
+        def async!(&) = async(:each!, &)
+        alias async_messages! async!
+
+        # Enumerate packets in a new thread
+        # @!macro yield_packets(void)
+        # @!macro async_return
+        # @see each_packet
+        def async_packets(&) = async(:each_packet, &)
+
+        # Enumerate packets in a new thread, ensuring {#unsubscribe}
+        # @!macro yield_packets(void)
+        # @!macro async_return
+        # @see each_packet!
+        def async_packets!(&) = async(:each_packet!, &)
+
+        # Delegates Enumerable methods ending in `!` to {#each!}, ensuring {#unsubscribe}
+        #
+        # @note Methods that require consuming the entire enumerable (e.g., `map`, `select`)
+        #   will block indefinitely on an infinite subscription stream unless combined with
+        #   limiting methods like `take` or terminated by a `break` condition.
+        #
+        # @example
+        #   sub.take!(5)             # => Array of 5 messages, then unsubscribes
+        #   sub.first!               # => First message, then unsubscribes
+        #   sub.find! { |t, p| ... } # => Matching message, then unsubscribes
+        def method_missing(method, *, &)
+          if method.end_with?('!') && Enumerable.public_instance_methods.include?(method[..-2].to_sym)
+            each!.public_send(method[..-2], *, &)
+          else
+            super
+          end
+        end
+
+        private
+
+        def respond_to_missing?(method, include_private = false)
+          (method.end_with?('!') && Enumerable.public_instance_methods.include?(method[..-2].to_sym)) || super
+        end
+
+        def enum_for!(method, &)
+          return enum_for(method) unless block_given?
+
+          with! { send(method[..-2], &) }
+        end
+      end
+    end
+  end
+end

data/lib/mqtt/core/client/filesystem_session_store.rb

@@ -0,0 +1,197 @@
+# frozen_string_literal: true
+
+require 'pathname'
+require 'tmpdir'
+require 'tempfile'
+require 'fileutils'
+require_relative 'qos2_session_store'
+module MQTT
+  module Core
+    class Client
+      # A Session Store that holds packets in the filesystem.
+      class FilesystemSessionStore < Qos2SessionStore
+        attr_reader :client_dir, :base_dir, :session_expiry_file
+
+        # @param [String] base_dir the base directory to store session files in
+        # @param [String|nil] client_id
+        #   empty string is not permitted, but nil can be used to force the generation of a random id.
+        # @param [Integer|nil] expiry_interval
+        #   zero is not permitted, but nil represents never expire (server may negotiate a lower value)
+        def initialize(client_id:, expiry_interval:, base_dir: Dir.mktmpdir('mqtt'))
+          @base_dir = Pathname.new(base_dir)
+          @client_dir = (base_dir + client_id)
+          super(client_id:, expiry_interval:)
+
+          @session_expiry_file = (base_dir + "#{client_id}.expiry")
+          cleanup_tmp
+          log.info { "client_dir: #{@client_dir}, clean?: #{clean?}" }
+        end
+
+        def restart_clone
+          self.class.new(base_dir, client_id:, expiry_interval:)
+        end
+
+        def clean?
+          !client_dir.exist?
+        end
+
+        def connected!
+          client_dirs.each(&:mkpath)
+
+          # record the previous session expiry duration so we can check it on a future restart
+          session_expiry_file.open('w') { |f| f.write(expiry_interval.to_s) }
+        end
+
+        def disconnected!
+          session_expiry_file.utime(nil, nil) # now
+        end
+
+        def expired?
+          return false unless session_expiry_file.exist?
+
+          # choose the most recent of...
+          #   * the directory modification times (updated each time a packet file is added or removed),
+          #   * the session_expiry_file modification time (updated on disconnect)
+          # A hard crash without a clean disconnect will potentially expire a session earlier than the server
+          Time.now - (client_dirs + [session_expiry_file]).map(&:mtime).max > session_expiry_file.read.to_i
+        end
+
+        def store_packet(packet, replace: false)
+          raise KeyError, 'packet id already exists' if !replace && stored_packet?(packet.id)
+
+          packet_file(packet.id).open('wb') { |f| packet.serialize(f) }
+        end
+
+        def delete_packet(id)
+          packet_file(id).delete
+        end
+
+        def stored_packet?(id)
+          packet_file(id).exist?
+        end
+
+        def retry_packets(&)
+          @client_dir.glob('pkt.*').sort_by(&:mtime).map { |f| f.open('r', &) }
+        end
+
+        def packet_file(id)
+          @client_dir + format('pkt/%04x.mqtt', id)
+        end
+
+        # QOS Receive
+        # Unique ID is sortable (fixed width timestamp)
+        # qos1 live:      `/#{client_id}/qos1/#{unique_id}_#{packet_id}.live` write on PUBLISH, deleted on handled.
+        # qos2 live:      `/#{client_id}/qos2/#{unique_id}_#{packet_id}.live` (unhandled, unreleased)
+        # qos2 handled:   `/#{client_id}/qos2/#{unique_id}_#{packet_id}.handled` (handled, unreleased)
+        # qos2 released:  `/#{client_id}/qos2/#{unique_id}_#{packet_id}.released` (unhandled, released)
+        # qos2 replay:    '/#{client_id}/qos2/#{unique_id}_#{packet_id}.replay_[live|handled]
+
+        # TODO: Recover utility
+        #    * cleanup_tmp
+        #    * qos2/*.live     - rename to .replay_live or .handled
+        #    * qos2/*.released - rename to .replay_released or delete
+
+        def store_qos_received(packet, unique_id)
+          client_dir + qos_path(packet.qos, packet.id, unique_id).tap do |live_file|
+            tmp_file = live_file.sub_ext('live', 'tmp')
+            tmp_file.open('wb') { |f| packet.serialize(f) }
+            tmp_file.rename(live_file)
+          end
+        end
+
+        # Release the pending qos2 (return true if we had previously seen it)
+        def qos2_release(id)
+          qos2_live = find_qos2_file(id)
+
+          if qos2_live&.extname == '.live'
+            qos2_live.rename(qos2_live.sub_ext('.live', '.released'))
+          else
+            qos2_live&.delete
+          end
+
+          super
+        rescue Errno::ENOENT
+          retry
+        end
+
+        def qos_handled(packet, unique_id)
+          if packet.qos == 1
+            qos1_handled(packet, unique_id)
+          elsif packet.qos == 2
+            qos2_handled(packet, unique_id)
+          end
+        end
+
+        # Called once at initialize.
+        # rubocop:disable Metrics/AbcSize
+        def qos2_recover
+          # Abort if there are unmarked files to potentially replay
+          if (client_dir.glob('qos2/*.live') + client_dir.glob('qos2/*.released')).any?
+            raise SessionNotRecoverable, "Unhandled QOS2 messages in #{"#{client_dir}/qos2"}. Run recover utility"
+          end
+
+          client_dir.glob('qos2/*.replay_live').each { |q2| q2.rename(q2.sub_ext('.live')) }
+          client_dir.glob('qos2/*.replay_released').each { |q2| q2.rename(q2.sub_ext('.released')) }
+
+          client_dir.glob(%w[qos2/*.live qos2/*.handled]).map { |f| f.basename.to_s.split('_').last.to_i(16) }
+        end
+        # rubocop:enable Metrics/AbcSize
+
+        # Load the unhandled packets with their unique id, only called once per session store
+        def qos_unhandled_packets(&)
+          client_dir.glob(%w[qos?/*.live qos2/*.released]).sort_by(&:basename)
+                    .to_h { |f| [f.open('r', &), f.basename.to_s.split('_').first] }
+        end
+
+        private
+
+        def cleanup_tmp
+          # Cleanup crashed .tmp files
+          client_dir.glob('qos?/*.tmp').each(&:delete)
+        end
+
+        # Make directories.
+        #   pkt  - packets we are sending, waiting to be acked
+        #   qos1 - qos1 packets received, waiting to be handled
+        #   qos2 - qos2 packets received, waiting to be released and handled
+        def client_dirs
+          %w[pkt qos1 qos2].map { |d| client_dir + d }
+        end
+
+        def qos2_handled(packet, unique_id)
+          live_file = client_dir + qos_path(2, packet.id, unique_id)
+          rel_file = client_dir + qos_path(2, packet.id, unique_id, 'released')
+
+          live_file.rename(live_file.sub_ext('.handled')) if live_file.exist?
+          rel_file.unlink if rel_file.exist?
+        rescue Errno::ENOENT
+          retry
+        end
+
+        def qos1_handled(packet, unique_id)
+          live_file = (client_dir + qos_path(1, packet.id, unique_id))
+          live_file.unlink
+        rescue Errno::ENOENT
+          log.warn { "qos_handled: #{live_file} unexpectedly not exists" }
+        end
+
+        # @return [String]
+        def qos_path(qos, packet_id, unique_id, ext = 'live')
+          format('qos%<qos>i/%<unique_id>s_%<packet_id>05x.%<ext>s', qos:, unique_id:, packet_id:, ext:)
+        end
+
+        def find_qos2_file(id)
+          # search live and handled separately to avoid race while renaming
+          live_files = client_dir.glob(qos_path(2, id, '*', 'live'))
+          raise ProtocolError, "QOS(#{id}): more than one packet: #{live_files}" if live_files.size > 1
+          return live_files.first if live_files.size == 1
+
+          handled_files = client_dir.glob(qos_path(2, id, '*', 'handled'))
+          raise ProtocolError, "QOS(#{id}): more than one packet: #{handled_files}" if handled_files.size > 1
+
+          handled_files.first
+        end
+      end
+    end
+  end
+end