librats 2.1.4 → 2.3.0

package/native-src/src/librats/storage/storage.h

@@ -11,18 +11,40 @@
  *
  * Replication model — an epidemic LWW broadcast:
  *   - A local put/remove builds a StorageEntry (a delete is a tombstone entry
- *     with `deleted=true`) and broadcasts it to all connected peers.
+ *     with `deleted=true`) and sends it to every connected peer.
  *   - On receiving an entry, a node applies it under LWW. It re-forwards the
  *     entry to its *other* peers ONLY if the entry actually won (carried new
  *     information). A duplicate loses LWW and is not forwarded, so flooding
  *     terminates naturally — no separate dedup table needed.
- *   - On peer connect, both sides exchange a full snapshot (anti-entropy) so a
- *     late joiner catches up. LWW makes the merge order-independent.
+ *   - On peer connect, both sides ask each other for a full snapshot
+ *     (anti-entropy) so a late joiner catches up. LWW makes the merge
+ *     order-independent.
+ *
+ * Backpressure — why a snapshot is a *stream*, not a message:
+ *   A database is unbounded, a peer's send queue is not. One message carrying
+ *   the whole store stops being sendable at the connection's low-water mark,
+ *   gets the peer dropped as a slow consumer past its high-water mark, and is
+ *   not even representable past the block-size ceiling — and since both ends
+ *   request a snapshot on connect, two of them cross on every single link. So
+ *   a snapshot is served as a sequence of bounded SYNC_CHUNK messages walking
+ *   the (ordered) key space, one chunk at a time, and the next chunk is only
+ *   produced while `PeerNetwork::peer_writable()` says there is room. Every
+ *   send() return value is honoured: a peer that fills up stops receiving
+ *   individual entries and is instead owed a fresh snapshot, which the sync
+ *   thread starts once on_peer_writable says the link has drained. Losing live
+ *   updates to a congested peer is safe precisely because the store is LWW —
+ *   the snapshot that follows carries the winning state either way.
+ *
+ *   All of that runs on this module's own sync thread. Serialising even one
+ *   chunk happens off the reactor thread, so a reactor never spends time (or
+ *   holds storage_mutex_) proportional to the size of the database.
  *
  * Wire format (MessageType::Storage payload, opcode in byte 0):
  *   ENTRY:         [1][StorageEntry.serialize()]
  *   SYNC_REQUEST:  [2]
- *   SYNC_RESPONSE: [3][count:u32][StorageEntry.serialize()] * count
+ *   SYNC_CHUNK:    [3][flags:u8][count:u32][StorageEntry.serialize()] * count
+ *                  flags bit0 (LAST) marks the final chunk of a snapshot; a
+ *                  snapshot of an empty store is one chunk with count 0.
  *
  * The class is also usable standalone (no network attached) as a local,
  * persistent key-value store; all network operations no-op until attach().
@@ -38,6 +60,7 @@
 #include <string>
 #include <vector>
 #include <functional>
+#include <map>
 #include <memory>
 #include <mutex>
 #include <unordered_map>
@@ -113,7 +136,23 @@ struct RATS_API StorageEntry {
     // Serialize entry to binary format
     std::vector<uint8_t> serialize() const;
 
-    // Deserialize entry from binary format
+    /// Append the serialized entry to `out`. What serialize() is built on: a
+    /// batch appends entry after entry into one buffer instead of allocating a
+    /// vector per entry only to copy it away again.
+    void serialize_into(std::vector<uint8_t>& out) const;
+
+    /// Serialized size in bytes, without serializing.
+    size_t serialized_size() const;
+
+    /// Deserialize one entry from `data[offset..]`, setting `bytes_read` to the
+    /// bytes it consumed. Every field is bounds-checked against the entry's own
+    /// declared length as well as the buffer, so a hostile length can neither
+    /// read past the buffer nor reach into the entry that follows.
+    static bool deserialize(const uint8_t* data, size_t size, size_t offset,
+                            StorageEntry& entry, size_t& bytes_read);
+
+    /// Vector overload of the above; the wire path uses the pointer form to
+    /// parse straight out of the receive buffer without copying it first.
     static bool deserialize(const std::vector<uint8_t>& data, size_t offset,
                            StorageEntry& entry, size_t& bytes_read);
 
@@ -136,23 +175,51 @@ struct StorageChangeEvent {
 };
 
 /**
- * Storage configuration
+ * Storage configuration.
+ *
+ * The two size limits are not free parameters: both are bounded by what one
+ * peer's send queue can hold. A connection drops its peer as a slow consumer
+ * past its high-water mark (8 MiB by default, NodeConfig::send_queue_limit) and
+ * reports "no room" at a quarter of it, so a single value — and a single
+ * snapshot chunk — must stay well inside that quarter, or the very first one
+ * would kill the link it travels on. Both are clamped to kMaxValueSize /
+ * kMaxSyncBatchBytes on construction and in set_config().
  */
 struct StorageConfig {
+    /// Ceiling on max_value_size: the default connection low-water mark. One
+    /// value at the ceiling makes a peer unwritable and is then waited out,
+    /// rather than counting toward the high-water mark that drops it.
+    static constexpr uint32_t kMaxValueSize      = 2 * 1024 * 1024;
+    /// Ceiling on sync_batch_bytes, for the same reason.
+    static constexpr uint32_t kMaxSyncBatchBytes = 2 * 1024 * 1024;
+    /// Floor on sync_batch_bytes: below this a snapshot costs more in per-message
+    /// framing and round trips than it saves in queue occupancy.
+    static constexpr uint32_t kMinSyncBatchBytes = 16 * 1024;
+
     std::string data_directory;         // Directory for storage files
     std::string database_name;          // Database filename prefix
     bool enable_sync;                   // Enable network synchronization
     uint32_t compaction_threshold;      // Number of tombstones before compaction
-    uint32_t max_value_size;            // Maximum value size in bytes
+    uint32_t max_value_size;            // Maximum value size in bytes (<= kMaxValueSize)
     bool persist_to_disk;               // Whether to persist data to disk
+    /// Target payload size of one snapshot chunk. The walk stops at the first
+    /// entry that takes the buffer past this, so a chunk is this size plus at
+    /// most one entry — which is why max_value_size shares the same ceiling.
+    uint32_t sync_batch_bytes;
+    /// Minimum gap between two snapshots served to the same peer. Bounds what a
+    /// peer can make us spend by asking, and paces the re-sync a peer is owed
+    /// after it has been congested.
+    uint32_t sync_min_interval_ms;
 
     StorageConfig()
         : data_directory("./storage"),
           database_name("rats_storage"),
           enable_sync(true),
           compaction_threshold(1000),
-          max_value_size(16 * 1024 * 1024),  // 16MB max value size
-          persist_to_disk(true) {}
+          max_value_size(1024 * 1024),        // 1 MiB max value size
+          persist_to_disk(true),
+          sync_batch_bytes(256 * 1024),       // 256 KiB per snapshot chunk
+          sync_min_interval_ms(5000) {}
 };
 
 /**
@@ -167,6 +234,9 @@ struct StorageStatistics {
     uint64_t entries_sent;              // Entries sent to peers
     uint64_t sync_requests_received;    // Number of sync requests received
     uint64_t sync_requests_sent;        // Number of sync requests sent
+    uint64_t sync_chunks_sent;          // Snapshot chunks put on the wire
+    uint64_t sync_chunks_received;      // Snapshot chunks applied from peers
+    uint64_t resyncs_scheduled;         // Snapshots owed to peers that filled up
     std::chrono::steady_clock::time_point last_sync_time;  // Last sync timestamp
     StorageSyncStatus sync_status;      // Current sync status
 };
@@ -216,6 +286,9 @@ public:
     // Configuration
     // =========================================================================
 
+    /// Replace the configuration. Size limits are clamped (see StorageConfig).
+    /// Call it before start(): the sync thread reads its tuning once when it
+    /// starts, and the store's other threads read the config without a lock.
     void set_config(const StorageConfig& config);
     const StorageConfig& get_config() const;
 
@@ -287,20 +360,60 @@ private:
     // Network message handlers (run on a reactor thread).
     void on_storage_message(const PeerId& from, ByteView payload);
     void on_peer_connected(const PeerId& peer_id);
+    void on_peer_disconnected(const PeerId& peer_id);
+    void on_peer_writable(const PeerId& peer_id);
 
     PeerNetwork* network_ = nullptr;
     StorageConfig config_;
 
-    // In-memory storage
+    // In-memory storage.
+    //
+    // Ordered, not hashed, and that is load-bearing: a snapshot is streamed one
+    // bounded chunk at a time, and between chunks the only thing carried over is
+    // the last key sent. An ordered map turns that key back into a position with
+    // upper_bound(), so a stream costs nothing to keep alive and cannot be
+    // derailed by concurrent writes — an unordered_map would need either a
+    // materialised key list per peer or iterators that a rehash invalidates.
+    // It also makes keys_with_prefix() a range scan instead of a full walk.
     mutable std::mutex storage_mutex_;
-    std::unordered_map<std::string, StorageEntry> entries_;
+    std::map<std::string, StorageEntry> entries_;
 
-    // Sync state
+    // Sync state.
+    //
+    // Lock order, exactly one rule: storage_mutex_ and sync_mutex_ are never held
+    // at the same time, and neither is ever held across a call into PeerNetwork.
     mutable std::mutex sync_mutex_;
     StorageSyncStatus sync_status_;
     bool initial_sync_complete_;
     std::chrono::steady_clock::time_point last_sync_time_;
 
+    /// What we owe one peer. A snapshot is served as a walk over the key space
+    /// whose whole resumable state is `cursor` — the last key already sent.
+    struct PeerSync {
+        bool                                  streaming = false;  ///< a snapshot is in flight
+        bool                                  started   = false;  ///< cursor is meaningful
+        std::string                           cursor;             ///< last key sent
+        bool                                  owed      = false;  ///< a (re)snapshot is due
+        std::chrono::steady_clock::time_point last_start{};       ///< for sync_min_interval_ms
+    };
+    std::unordered_map<PeerId, PeerSync, PeerId::Hash> peers_;    ///< guarded by sync_mutex_
+    /// Bumped, under sync_mutex_, whenever something the sync thread would act on
+    /// changes — a snapshot scheduled, a link drained, a peer gone. The thread
+    /// reads it before it scans and compares it before it waits, because between
+    /// those two points it is outside the lock (serializing and sending), and a
+    /// notify that arrives there would otherwise be one nobody is waiting for.
+    uint64_t sync_epoch_ = 0;
+
+    // The sync thread: serializes and paces every snapshot chunk, so no reactor
+    // thread ever does work proportional to the size of the database.
+    std::atomic<bool>       sync_running_{false};
+    std::thread             sync_thread_;
+    std::condition_variable sync_cv_;   ///< paired with sync_mutex_
+    // Sync tuning, copied out of the config when the thread starts and read-only
+    // afterwards, so the thread never races a set_config() on another thread.
+    size_t                    batch_bytes_{0};
+    std::chrono::milliseconds sync_interval_{0};
+
     // Statistics
     mutable std::mutex stats_mutex_;
     StorageStatistics stats_;
@@ -317,14 +430,37 @@ private:
     bool dirty_;  // Flag indicating unsaved changes
 
     // Wire opcodes (MessageType::Storage payload, byte 0)
-    static constexpr uint8_t OP_ENTRY         = 1;
-    static constexpr uint8_t OP_SYNC_REQUEST  = 2;
-    static constexpr uint8_t OP_SYNC_RESPONSE = 3;
+    static constexpr uint8_t OP_ENTRY        = 1;
+    static constexpr uint8_t OP_SYNC_REQUEST = 2;
+    static constexpr uint8_t OP_SYNC_CHUNK   = 3;
+
+    // SYNC_CHUNK flags (byte 1)
+    static constexpr uint8_t FLAG_LAST = 0x01;   ///< final chunk of this snapshot
+
+    /// Refuse an inbound Storage message larger than this before parsing it. Our
+    /// own sender never comes near it (one chunk is sync_batch_bytes plus at most
+    /// one value, both capped at 2 MiB), while the block layer would happily hand
+    /// us 64 MiB from a peer that means us harm.
+    static constexpr size_t kMaxInboundMessage = 8 * 1024 * 1024;
 
     // Private methods
     void initialize();
     void shutdown();
     void persistence_thread_loop();
+    void start_sync_thread();
+    void stop_sync_thread();
+    void sync_thread_loop();
+    /// What one turn of a peer's snapshot stream left behind.
+    enum class ChunkResult {
+        Continue,   ///< a chunk went out and the link has room for the next
+        Blocked,    ///< a chunk went out and filled the link; retry when it drains
+        Finished    ///< the snapshot is complete (or the peer is gone)
+    };
+    /// Serialize and send the next chunk of `peer`'s snapshot.
+    ChunkResult stream_snapshot_chunk(const PeerId& peer);
+    /// Note that `peer` is due a snapshot; the sync thread starts it once the
+    /// link has room and the per-peer interval has elapsed.
+    void schedule_snapshot(const PeerId& peer, bool requested_by_peer);
 
     // Internal put with full control
     bool put_internal(const std::string& key, StorageValueType type,
@@ -341,14 +477,20 @@ private:
     double deserialize_double(const std::vector<uint8_t>& data) const;
     std::string deserialize_string(const std::vector<uint8_t>& data) const;
 
-    // Network operations
-    void broadcast_entry(const StorageEntry& entry);              ///< to all peers
-    void forward_entry(const StorageEntry& entry, const PeerId& except);  ///< re-flood
+    // Network operations. Every one of them honours send()'s return value: a peer
+    // that answers "no room" is owed a snapshot instead of further entries.
+    /// Send one entry to every connected peer except `except` (null for none).
+    void replicate_entry(const StorageEntry& entry, const PeerId* except);
+    void broadcast_entry(const StorageEntry& entry) { replicate_entry(entry, nullptr); }
+    void forward_entry(const StorageEntry& entry, const PeerId& except) {
+        replicate_entry(entry, &except);
+    }
     void send_sync_request(const PeerId& peer_id);
-    void send_sync_response(const PeerId& peer_id);
 
     // Apply a remote entry with LWW; fills `out_event` and returns true if applied.
     bool apply_remote_entry(const StorageEntry& entry, StorageChangeEvent* out_event);
+    /// Parse and apply the entries in one SYNC_CHUNK body. @return how many won.
+    uint32_t apply_chunk(const PeerId& from, const uint8_t* data, size_t size, uint32_t count);
 
     // File path helpers
     std::string get_data_file_path() const;
@@ -359,6 +501,8 @@ private:
     bool read_data_file();
 
     // Utility
+    /// Clamp the size limits to what one connection's send queue can carry.
+    static void sanitize_config(StorageConfig& config);
     uint64_t get_current_timestamp_ms() const;
     std::string get_our_peer_id() const;
     void notify_change(const StorageChangeEvent& event);

package/native-src/src/librats/subsystems/file_transfer.cpp

@@ -80,7 +80,7 @@ void FileTransfer::attach(NodeContext& ctx) {
     network_ = &ctx.network;
     network_->on(MessageType::FileChunk,
                  [this](const Peer& peer, ByteView payload) { on_message(peer, payload); });
-    network_->on_peer_disconnected([this](const PeerId& id) {
+    network_->on_peer_disconnected([this](const PeerId& id, CloseReason) {
         // Fail every in-flight transfer with the departed peer and reclaim temps.
         std::vector<std::shared_ptr<Outgoing>> out;
         std::vector<std::shared_ptr<Incoming>> in;

package/native-src/src/librats/subsystems/hole_punch.cpp

@@ -1,6 +1,7 @@
 #include "librats/subsystems/hole_punch.h"
 #include "librats/node/dial_service.h"
 #include "librats/node/node_context.h"
+#include "librats/peer/peer_info.h"
 #include "librats/util/logger.h"
 
 #include <algorithm>
@@ -135,6 +136,7 @@ void HolePunch::attach(NodeContext& ctx) {
     // does not know its own external one simply cannot punch — it can still relay.
     dialer_   = ctx.services.get<DialService>();
     external_ = ctx.services.get<ExternalAddressService>();
+    services_ = &ctx.services;
 
     // Offer punching to sibling modules. Registered even without a dialer: punch()
     // answers false on its own then, which is exactly what a caller expects from a
@@ -151,17 +153,30 @@ void HolePunch::attach(NodeContext& ctx) {
     // reconciles sessions against the peer set, so this is the fast path, not the
     // only one.
     network_->on_peer_connected([this](const Peer& peer) {
+        // A relayed link is not what a punch was for — it is the fallback the punch
+        // is trying to make unnecessary, and a punch may well have been started
+        // BECAUSE it came up (see subsystems/relay.h). Retiring the session here
+        // would call the upgrade off the moment it began.
+        const auto info = peer.info();
+        if (info && info->transport == TransportKind::Relay) return;
         std::lock_guard<std::mutex> lock(mutex_);
         sessions_.erase(peer.id());
     });
 }
 
 void HolePunch::start() {
+    // Resolved here rather than in attach(): Relay may be attached after us, and
+    // every attach() runs before any start(). Stored before running_ goes true, so
+    // an escalation on a reactor thread never sees a half-initialised state.
+    if (config_.relay_on_failure && services_)
+        relay_.store(services_->get<RelayService>());
+
     if (running_.exchange(true)) return;
     worker_ = std::thread([this] { loop(); });
 }
 
 void HolePunch::stop() {
+    relay_.store(nullptr);
     if (!running_.exchange(false)) return;
     cv_.notify_all();
     if (worker_.joinable()) worker_.join();
@@ -178,14 +193,20 @@ bool HolePunch::punch(const PeerId& target) {
     if (target == network_->local_id()) return false;
 
     // Already reachable the ordinary way. Checked before any state is created so a
-    // caller can punch() freely on discovery without having to check first.
-    for (const PeerId& id : network_->connected_peers())
+    // caller can punch() freely on discovery without having to check first. A
+    // RELAYED peer is deliberately not "already reachable": punching it is how the
+    // circuit gets replaced by a direct link (see subsystems/relay.h).
+    for (const PeerId& id : directly_connected())
         if (id == target) return false;
 
     if (config_.skip_when_endpoint_dependent && external_ &&
         external_->udp_mapping() == NatMapping::EndpointDependent) {
         LOG_DEBUG("punch", "Not punching to " << target.short_hex()
                   << ": our own mapping is per-destination (symmetric NAT)");
+        // Nothing about this will improve with time or retries: no endpoint we can
+        // advertise is the one the target's packets would arrive on. This is
+        // exactly the case a relayed path exists for.
+        escalate_to_relay(target);
         return false;
     }
 
@@ -193,6 +214,7 @@ bool HolePunch::punch(const PeerId& target) {
     if (addresses.empty()) {
         LOG_DEBUG("punch", "Not punching to " << target.short_hex()
                   << ": no external datagram endpoint to advertise yet");
+        escalate_to_relay(target);
         return false;
     }
 
@@ -210,8 +232,11 @@ bool HolePunch::punch(const PeerId& target) {
     }
 
     if (!send_connect(target, /*opening=*/true, nullptr)) {
-        std::lock_guard<std::mutex> lock(mutex_);
-        sessions_.erase(target);   // nobody could carry it; nothing is in flight
+        {
+            std::lock_guard<std::mutex> lock(mutex_);
+            sessions_.erase(target);   // nobody could carry it; nothing is in flight
+        }
+        escalate_to_relay(target);
         return false;
     }
     LOG_DEBUG("punch", "Punch rendezvous started with " << target.short_hex());
@@ -461,10 +486,12 @@ void HolePunch::service_sessions() {
 
     // Connected peers first, without the lock: a session whose target is now a peer
     // succeeded, however it got there (our burst, or theirs arriving as inbound).
-    std::vector<PeerId> connected = network_->connected_peers();
+    // Direct links only — a relayed one is what a punch is trying to replace.
+    std::vector<PeerId> connected = directly_connected();
 
     struct Retry { PeerId target; PeerId via; bool have_via; };
     std::vector<Retry> retry;
+    std::vector<PeerId> exhausted;   ///< targets to hand on to the relay, once unlocked
     {
         std::lock_guard<std::mutex> lock(mutex_);
 
@@ -496,7 +523,10 @@ void HolePunch::service_sessions() {
                 // first). Calling the target off here would drop the retries it is
                 // about to send, and block this node's own later punch to it, for
                 // the whole cooldown. So a responder just forgets the round.
-                if (initiator) begin_cooldown(target);
+                if (initiator) {
+                    begin_cooldown(target);
+                    exhausted.push_back(target);
+                }
                 it = sessions_.erase(it);
                 continue;
             }
@@ -510,9 +540,52 @@ void HolePunch::service_sessions() {
 
     for (const Retry& r : retry) {
         if (send_connect(r.target, /*opening=*/true, r.have_via ? &r.via : nullptr)) continue;
-        std::lock_guard<std::mutex> lock(mutex_);
-        sessions_.erase(r.target);
+        {
+            std::lock_guard<std::mutex> lock(mutex_);
+            sessions_.erase(r.target);
+        }
+        exhausted.push_back(r.target);
     }
+
+    // The rungs above this one are spent: no address we can advertise got through.
+    // Handing the target on is the difference between a peer that is unreachable
+    // and one that is merely expensive to reach.
+    for (const PeerId& target : exhausted) escalate_to_relay(target);
+}
+
+void HolePunch::escalate_to_relay(const PeerId& target) {
+    RelayService* relay = relay_.load();
+    if (!relay) return;   // no Relay attached, or the fallback is off
+    if (!relay->connect_via_relay(target)) return;
+
+    LOG_DEBUG("punch", "Punching to " << target.short_hex()
+              << " is not going to work; looking for a relay instead");
+
+    // The target has just changed hands, and the cooldown a give-up started must not
+    // outlive that. It exists to stop us hammering a peer we cannot reach — a job
+    // the relay now owns, with an attempt timeout and a cooldown of its own.
+    //
+    // And if that attempt lands, the circuit is precisely the new information that
+    // makes another punch worth trying: the two ends are peers at last, so the
+    // rendezvous can travel over the very circuit carrying them, which is what
+    // Relay asks for the moment the circuit comes up (see subsystems/relay.h).
+    // Leaving the cooldown standing would refuse that upgrade before it started —
+    // and nothing would ever ask again, so the circuit would outlive its purpose and
+    // keep costing a third node bandwidth for the life of the peer.
+    //
+    // Bounded, not a loop: an upgrade punch that fails lands here again, and
+    // connect_via_relay then answers false — the target is already a peer — so the
+    // fresh cooldown stands. Taken after the call, never around it: Relay resolves
+    // the reverse direction through us, and this mutex must not be held into it.
+    std::lock_guard<std::mutex> lock(mutex_);
+    cooldown_.erase(target);
+}
+
+std::vector<PeerId> HolePunch::directly_connected() const {
+    std::vector<PeerId> ids;
+    for (const PeerInfo& info : network_->peers())
+        if (info.transport != TransportKind::Relay) ids.push_back(info.id);
+    return ids;
 }
 
 bool HolePunch::in_cooldown(const PeerId& target) const {

package/native-src/src/librats/subsystems/hole_punch.h

@@ -95,6 +95,7 @@
 #include "librats/peer/peer.h"
 #include "librats/peer/peer_id.h"
 #include "librats/subsystems/hole_punch_service.h"
+#include "librats/subsystems/relay_service.h"
 
 #include <atomic>
 #include <chrono>
@@ -108,6 +109,7 @@
 namespace librats {
 
 class DialService;
+class ServiceRegistry;
 
 /// Published as HolePunchService, so a module that discovers a peer it cannot dial
 /// (PeerExchange) can hand the id over without depending on this class.
@@ -163,6 +165,12 @@ public:
 
         /// Session bookkeeping cadence — retries, timeouts, cooldown expiry.
         std::chrono::milliseconds tick{250};
+
+        /// When a punch cannot be attempted at all, or has been given up on, hand
+        /// the target to RelayService — the next rung down the ladder (see
+        /// subsystems/relay.h). Costs nothing when no Relay is attached: the
+        /// service simply does not resolve.
+        bool relay_on_failure = true;
     };
 
     HolePunch();
@@ -235,6 +243,17 @@ private:
     void service_sessions();
 
     // — helpers —
+    /// Hand `target` to the relay module, if one is attached and the fallback is
+    /// on. Called when a punch is impossible or has run out of attempts — the
+    /// point at which a relayed path stops being the worse option and becomes the
+    /// only one. Retires the target's cooldown when the relay takes it on, so that
+    /// the upgrade punch a circuit asks for is not refused by the very give-up that
+    /// produced the circuit. Caller must NOT hold the mutex.
+    void                 escalate_to_relay(const PeerId& target);
+    /// Peers reachable over a DIRECT link. A relayed peer is deliberately not one:
+    /// a punch to it is an upgrade in progress, and counting the circuit as success
+    /// would retire the session before it had done anything.
+    std::vector<PeerId>  directly_connected() const;
     std::vector<Address> own_punch_addresses() const;
     bool                 relay_budget_ok(const PeerId& from);
     bool                 in_cooldown(const PeerId& target) const;   ///< caller holds mutex_
@@ -244,6 +263,11 @@ private:
     PeerNetwork*           network_  = nullptr;
     DialService*           dialer_   = nullptr;
     ExternalAddressService* external_ = nullptr;
+    ServiceRegistry*       services_ = nullptr;
+    /// Resolved in start(), not attach(): Relay may be attached after us, and every
+    /// attach() runs before any start(). Atomic because it is read from reactor
+    /// threads and the worker alike.
+    std::atomic<RelayService*> relay_{nullptr};
 
     std::atomic<bool>     running_{false};
     std::atomic<uint64_t> punches_started_{0};

package/native-src/src/librats/subsystems/pubsub.cpp

@@ -106,7 +106,7 @@ void PubSub::attach(NodeContext& ctx) {
     network_->on(MessageType::Gossip,
                          [this](const Peer& peer, ByteView payload) { on_gossip(peer, payload); });
     network_->on_peer_connected([this](const Peer& peer) { on_new_peer(peer); });
-    network_->on_peer_disconnected([this](const PeerId& id) { on_peer_gone(id); });
+    network_->on_peer_disconnected([this](const PeerId& id, CloseReason) { on_peer_gone(id); });
 }
 
 void PubSub::start() {

package/native-src/src/librats/subsystems/reconnection.cpp

@@ -74,7 +74,7 @@ std::vector<Address> ReconnectionService::known_peers(size_t n) const {
 void ReconnectionService::attach(NodeContext& ctx) {
     network_ = &ctx.network;
     network_->on_peer_connected([this](const Peer& peer) { on_connected(peer); });
-    network_->on_peer_disconnected([this](const PeerId& id) { on_disconnected(id); });
+    network_->on_peer_disconnected([this](const PeerId& id, CloseReason) { on_disconnected(id); });
     network_->on_dial_failed([this](const Address& addr) { on_dial_failed(addr); });
 }