librats 2.1.4 → 2.3.0

package/native-src/src/librats/node/circuit_service.h

@@ -0,0 +1,84 @@
+#pragma once
+
+/**
+ * @file circuit_service.h
+ * @brief Turning a relayed byte stream into a peer connection — the capability a
+ *        relay module needs and PeerNetwork deliberately does not offer.
+ *
+ * A relay module speaks a protocol (who forwards what to whom) and owns the byte
+ * stream that comes out of it (transport/relay_link.h). What it cannot do on its
+ * own is the one remaining step: making that stream a Connection, so the ordinary
+ * Noise handshake, framing, identify and peer table all run over it and the peer
+ * at the far end becomes an ordinary peer. That step needs the reactor pool, which
+ * a Subsystem has no business holding.
+ *
+ * So this is the narrow escape hatch, published by the Node in its ServiceRegistry
+ * next to DialService, and resolved by whoever needs it:
+ *
+ *     if (auto* circuits = ctx.services.get<CircuitService>())
+ *         auto route = circuits->adopt_circuit(relay_id, std::move(link),
+ *                                              ConnRole::Outbound, false);
+ *
+ * ── One thread, chosen for you ──────────────────────────────────────────────
+ * The node places the circuit on the reactor that owns the CARRIER's connection,
+ * and that is the whole reason this call exists rather than a bare "adopt this
+ * link somewhere". Every byte of a circuit arrives on the carrier's connection and
+ * leaves through it, so putting the two on one thread means the entire relayed
+ * path is handled without a lock, a queue or a hand-off — the same shared-nothing
+ * property every other connection has. Any other placement would move bytes
+ * between reactor threads twice per message.
+ *
+ * The route comes back synchronously (the ConnId is reserved before the adoption
+ * is posted, exactly as Reactor::connect does), so the caller can wake and close
+ * the circuit from the moment it asks for one.
+ *
+ * ── What is deliberately NOT here ───────────────────────────────────────────
+ * Nothing about relaying: no relay selection, no protocol, no policy about when a
+ * circuit is worth opening. Those belong to the module. This interface would serve
+ * any transport that arrives from outside the reactor pool.
+ */
+
+#include "librats/core/types.h"
+#include "librats/peer/peer_id.h"
+#include "librats/peer/peer_table.h"   // PeerRoute
+#include "librats/transport/link.h"
+
+#include <cstdint>
+#include <memory>
+#include <optional>
+
+namespace librats {
+
+class CircuitService {
+public:
+    virtual ~CircuitService() = default;
+
+    /// Adopt `link` as a connection carried by the peer `carrier`, on the reactor
+    /// that owns the carrier's own connection.
+    ///
+    /// @param role      Outbound for a circuit we opened, Inbound for one opened to
+    ///                  us. It is what the secure handshake and the peer table's
+    ///                  duplicate resolution read, so it must say who asked.
+    /// @param connected whether the far end is already there. An inbound circuit is
+    ///                  (the opener would not have sent it otherwise); an outbound
+    ///                  one is not until its acceptance arrives, and until then the
+    ///                  connection waits in Connecting for a PollOut.
+    /// @return the new connection's route, or nullopt when the carrier is no longer
+    ///         a peer, or when an inbound circuit would push this node past its peer
+    ///         limit. Nothing is started in that case and `link` is released.
+    virtual std::optional<PeerRoute> adopt_circuit(const PeerId& carrier,
+                                                   std::unique_ptr<Link> link,
+                                                   ConnRole role, bool connected) = 0;
+
+    /// Deliver poll-equivalent events (PollIn / PollOut / PollErr, see
+    /// core/io_poller.h) to a circuit connection — the events a socket-backed link
+    /// would have got from the poller. Thread-safe, and a no-op for a route that is
+    /// already gone.
+    virtual void wake_circuit(PeerRoute route, uint32_t events) = 0;
+
+    /// Tear a circuit connection down. Thread-safe; a no-op for a route that is
+    /// already gone.
+    virtual void close_circuit(PeerRoute route, CloseReason reason) = 0;
+};
+
+} // namespace librats

package/native-src/src/librats/node/config.h

@@ -52,8 +52,15 @@ struct RATS_API NodeConfig {
     /// connection. 0 disables the fallback: only the preferred transport is tried.
     uint32_t transport_fallback_ms = 1200;
 
-    /// Bytes a peer's send queue may hold before the peer is dropped as a slow
-    /// consumer. 0 uses the library default (8 MiB).
+    /// Bytes a peer's send queue may hold before an application that keeps
+    /// sending anyway has the peer dropped as a slow consumer. 0 uses the library
+    /// default (8 MiB).
+    ///
+    /// It is not a maximum message size. A single message is always queued
+    /// whatever its size — a message cannot be sent by halves, and a healthy
+    /// connection must not die over one large frame — so the queue's real ceiling
+    /// is this plus one message. What gets a peer dropped is offering *another*
+    /// message while the queue is still over the mark.
     ///
     /// A quarter of this is the mark at which send() starts answering "no room"
     /// and on_peer_writable is what says the room is back — so lowering it makes

package/native-src/src/librats/node/node.cpp

@@ -94,6 +94,7 @@ Node::Node(NodeConfig config)
     // are narrow capabilities a module asks for by interface (see
     // node/dial_service.h, node/nat_status.h).
     services_.provide<DialService>(this);
+    services_.provide<CircuitService>(this);
     services_.provide<ExternalAddressService>(&nat_status_);
 }
 
@@ -402,6 +403,9 @@ bool Node::dial_direct(const Address& addr, TransportKind kind, const DialProfil
     if (!addr.is_valid()) return false;
     if (kind == TransportKind::Udp && !reactors_->has_udp()) return false;
     if (kind == TransportKind::Tcp && !config_.enable_tcp)   return false;
+    // A relayed circuit is not dialed: it is negotiated with a third node and then
+    // adopted (see node/circuit_service.h). There is no address to aim at here.
+    if (kind == TransportKind::Relay) return false;
 
     // Straight to the reactor, deliberately around the Dialer: this dial is not a
     // race and must not become one (see node/dial_service.h). The Dialer is left
@@ -412,6 +416,41 @@ bool Node::dial_direct(const Address& addr, TransportKind kind, const DialProfil
     return id != kInvalidConnId;
 }
 
+// ── CircuitService: a relayed byte stream becomes a peer connection ─────────
+
+std::optional<PeerRoute> Node::adopt_circuit(const PeerId& carrier, std::unique_ptr<Link> link,
+                                             ConnRole role, bool connected) {
+    if (!link) return std::nullopt;
+
+    // The carrier has to still be a peer: it is the only thing this circuit's bytes
+    // can travel through, and a connection whose carrier is already gone would
+    // simply sit there until the establish deadline reaped it.
+    const auto carrier_route = peers_.route(carrier);
+    if (!carrier_route) return std::nullopt;
+
+    // Inbound circuits go through the same admission gate as an accepted socket.
+    // A relayed connection is cheaper for whoever opens it than a direct one — no
+    // NAT to get through, someone else's bandwidth — so if anything it deserves the
+    // gate more, not less.
+    if (role == ConnRole::Inbound && !admit_inbound()) return std::nullopt;
+
+    Reactor& reactor = reactors_->by_index(carrier_route->reactor);
+    // Same reactor as the carrier, which is the whole point: every byte of this
+    // circuit arrives on the carrier's connection and leaves through it, so the two
+    // share a thread and the relayed path needs no locks and no hand-offs.
+    const ConnId id = reactor.reserve_conn_id();
+    reactor.adopt_link(id, std::move(link), role, connected);
+    return PeerRoute{carrier_route->reactor, id};
+}
+
+void Node::wake_circuit(PeerRoute route, uint32_t events) {
+    reactors_->by_index(route.reactor).wake(route.conn, events);
+}
+
+void Node::close_circuit(PeerRoute route, CloseReason reason) {
+    reactors_->by_index(route.reactor).close(route.conn, reason);
+}
+
 void Node::report_dial_failed(const std::string& host, uint16_t port) {
     // Only a numeric target can be handed on as an Address; a dial by hostname has
     // no dialable Address to report (peers cannot use our resolver), and the
@@ -449,7 +488,15 @@ bool Node::send(const PeerId& to, std::string_view channel, ByteView payload) {
 
 bool Node::peer_writable(const PeerId& id) const {
     const auto dest = peers_.destination(id);
-    return dest && dest->writable;
+    if (!dest) return false;
+    // Both halves of the backlog, exactly as send() weighs them: what the reactor
+    // has already queued (`writable`) *and* what a caller has handed over that the
+    // reactor has not taken up yet (`owed`). Reporting only the first would answer
+    // "there is room" to a caller that has just filled the queue itself and is
+    // waiting to hear otherwise — the queue it filled has not been looked at yet,
+    // so nothing about it has changed, and no event is coming either.
+    return dest->writable &&
+           dest->owed->load(std::memory_order_relaxed) <= send_low_water();
 }
 
 bool Node::broadcast(std::string_view channel, ByteView payload) {
@@ -643,7 +690,7 @@ void Node::on_closed(Connection& conn, CloseReason reason) {
     // let a long-gone peer keep voting on how our NAT behaves.
     nat_status_.forget(id);
     LOG_INFO("node", "Peer " << id.short_hex() << " disconnected (" << to_string(reason) << ")");
-    for (auto& cb : peer_disconnected_) cb(id);
+    for (auto& cb : peer_disconnected_) cb(id, reason);
 }
 
 void Node::on_writable_changed(Connection& conn, bool writable) {
@@ -810,9 +857,16 @@ std::vector<Address> Node::observed_addresses() const {
 
 // ── Peer handle methods (defined here for the full Node type) ────────────────
 
-void Peer::send(std::string_view channel, ByteView payload) const {
-    node_->route_send(route_, FrameHeader{MessageType::App, 0, MessageRouter::channel_id(channel)},
-                      payload.to_bytes());
+bool Peer::send(std::string_view channel, ByteView payload) const {
+    // Deliberately by id, not by this handle's route. The route existed to skip the
+    // directory lookup on the reply path — but answering "is there room?" needs the
+    // peer's writability and its in-transit counter, both of which live in the
+    // directory, so the lookup is unavoidable the moment send() has an answer to
+    // give. Once it is being paid anyway, the captured route buys nothing and can
+    // only be wrong: a handle outlives its connection (a relayed link superseded by
+    // a direct one, a dial race, a reconnect), and sending on the superseded one
+    // drops the message with nothing said. The peer is what the caller means.
+    return node_->send(id_, channel, payload);
 }
 
 void Peer::disconnect() const {

package/native-src/src/librats/node/node.h

@@ -48,6 +48,7 @@
 #include "librats/peer/peer_info.h"
 #include "librats/security/identity.h"
 #include "librats/security/handshaker.h"  // SecurityProvider
+#include "librats/node/circuit_service.h"
 #include "librats/node/config.h"
 #include "librats/node/dial_service.h"
 #include "librats/node/dialer.h"
@@ -74,7 +75,10 @@ namespace librats {
 class NetworkMonitor;  // util/network_monitor.h — owned via unique_ptr, included in node.cpp
 class MessageJson;     // subsystems/message_json.h — reached via json() (json.h stays out of node.h)
 
-class RATS_API Node final : public ConnectionDelegate, public PeerNetwork, public DialService {
+class RATS_API Node final : public ConnectionDelegate,
+                            public PeerNetwork,
+                            public DialService,
+                            public CircuitService {
 public:
     /// Construct a node from its configuration (see NodeConfig). This only loads
     /// the identity and prepares the layers; no socket is opened until start().
@@ -163,6 +167,13 @@ public:
     bool dial_direct(const Address& addr, TransportKind kind,
                      const DialProfile& profile) override;
 
+    // — CircuitService: make a relayed byte stream an ordinary peer connection
+    //   (see node/circuit_service.h; used by the relay module) —
+    std::optional<PeerRoute> adopt_circuit(const PeerId& carrier, std::unique_ptr<Link> link,
+                                           ConnRole role, bool connected) override;
+    void                     wake_circuit(PeerRoute route, uint32_t events) override;
+    void                     close_circuit(PeerRoute route, CloseReason reason) override;
+
     // — peer admission limit (0 = unlimited; guards inbound, not our own dials) —
     size_t max_peers() const noexcept { return max_peers_.load(std::memory_order_relaxed); }
     void   set_max_peers(size_t n) noexcept { max_peers_.store(n, std::memory_order_relaxed); }
@@ -182,6 +193,14 @@ public:
     ///         low-water mark, and an application that keeps going regardless
     ///         will eventually have the peer dropped as a slow consumer. Wait for
     ///         on_peer_writable instead. Also false if the peer is not connected.
+    ///
+    ///         "Stop" is meant literally, and yielding is part of it: the mark is
+    ///         re-tested inside the reactor task this call hands off to, and it is
+    ///         that test which flips the peer to unwritable and later raises
+    ///         on_peer_writable. A caller that answers a false by looping straight
+    ///         back into send() therefore starves the very thread that would tell
+    ///         it to stop — the queue keeps growing while the signal it is waiting
+    ///         for never gets a turn to be produced.
     bool send(const PeerId& to, std::string_view channel, ByteView payload);
     /// Send raw bytes on a named channel to every connected peer.
     /// @return whether *every* one of them still has room — a fan-out can only
@@ -194,12 +213,24 @@ public:
     /// transfer, a large stream — address peers individually with send().
     bool broadcast(std::string_view channel, ByteView payload);
 
-    /// Whether a peer's send queue currently has room — the same answer send()
-    /// returns, without sending anything. False for a peer that is not connected.
-    /// Note that the answer describes the queue as the reactor last left it: a
-    /// message handed over a moment ago may not be counted yet, which is why the
-    /// signal to stop is the *return of send()* rather than a poll before it.
-    bool peer_writable(const PeerId& id) const;
+    /// Whether a peer's send queue has room for more. False for a peer that is
+    /// not connected.
+    ///
+    /// The same question send() answers, asked without sending anything: it
+    /// weighs both halves of what the peer is carrying — the bytes the reactor
+    /// has queued, and the bytes a caller has handed to send() that the reactor
+    /// has not taken up yet. So it is safe to poll: a caller that has just filled
+    /// the queue in a tight loop keeps being told "no room" until the reactor has
+    /// actually looked at what it was given, rather than being told "go on"
+    /// because nothing observable has changed yet.
+    ///
+    /// It stays a hint about a queue that drains as it is read, so the ordinary
+    /// flow is unchanged: the signal to stop is the return of send(), and the
+    /// signal to resume is on_peer_writable. This is for a caller that must wait
+    /// for room on a thread of its own — the event alone cannot serve it, because
+    /// a queue that filled *only* with bytes still in transit never crossed
+    /// anything on the connection and so raises no event when they drain.
+    bool peer_writable(const PeerId& id) const override;
 
     // — events (register before start(); invoked on a reactor thread). Multiple
     //   listeners are supported, so subsystems and the app can both subscribe. —

package/native-src/src/librats/node/peer_network.h

@@ -17,6 +17,7 @@
 #include "librats/peer/peer_id.h"
 #include "librats/peer/peer_info.h"
 #include "librats/core/address.h"
+#include "librats/core/types.h"  // CloseReason
 
 #include <cstdint>
 #include <functional>
@@ -33,7 +34,12 @@ public:
     using MessageHandler = std::function<void(const Peer&, ByteView)>;
 
     using PeerEventHandler       = std::function<void(const Peer&)>;
-    using PeerDisconnectHandler  = std::function<void(const PeerId&)>;
+    /// A peer went away, and why. The reason matters as much as the event: an
+    /// application dropped as a slow consumer (CloseReason::SlowConsumer) has to
+    /// slow down, while one whose peer simply left should reconnect — and without
+    /// the reason those look identical, so the usual answer to both is to redial
+    /// and repeat whatever caused it.
+    using PeerDisconnectHandler  = std::function<void(const PeerId&, CloseReason)>;
     using DialFailedHandler      = std::function<void(const Address&)>;
 
     virtual const PeerId&       local_id() const = 0;
@@ -49,7 +55,9 @@ public:
     /// Send to one peer. @return whether that peer's send queue still has room;
     /// false means "stop and wait for on_peer_writable" — the message is queued
     /// either way, but continuing past this is what gets a peer dropped as a slow
-    /// consumer. Also false if the peer is not connected.
+    /// consumer. Also false if the peer is not connected. Wait on the event and
+    /// not on a poll of the queue's state: the answer is re-derived on the reactor
+    /// thread, so a caller that spins instead of yielding never lets it change.
     virtual bool                send(const PeerId& to, MessageType type, ByteView payload) = 0;
     /// Send to every connected peer. @return whether *every* one of them still
     /// has room, so a subsystem that fans out can pause on the slowest.
@@ -72,6 +80,15 @@ public:
     /// subsystem that never checks that return never needs this either. Runs on a
     /// reactor thread. Default: not offered (nothing subscribes).
     virtual void                on_peer_writable(PeerEventHandler /*handler*/) {}
+    /// The same question send() answers — "has this peer room for more?" — asked
+    /// without sending anything. For a subsystem that paces a long fan-out from a
+    /// thread of its own (StorageManager streaming a snapshot): the event alone
+    /// cannot serve it, because a queue filled *only* with bytes handed to send()
+    /// that the reactor has not taken up yet never crossed anything on the
+    /// connection, so nothing raises on_peer_writable when they drain. Safe to
+    /// poll — it weighs those in-flight bytes too. False for an unknown peer.
+    /// Default: always writable, all a mock that merely moves messages can claim.
+    virtual bool                peer_writable(const PeerId& /*id*/) const { return true; }
 };
 
 struct NodeContext;  // node/node_context.h — bundles network + events + services

package/native-src/src/librats/peer/peer.h

@@ -4,10 +4,12 @@
  * @file peer.h
  * @brief A lightweight handle to a connected peer.
  *
- * Peer is a value passed to callbacks. It carries the peer's id and its
- * route (which reactor + connection), so send()/disconnect() reach the right
- * reactor directly — no PeerTable lookup on the reply path. info() does
- * consult the directory, on demand, for metadata.
+ * Peer is a value passed to callbacks. It carries the peer's id and its route
+ * (which reactor + connection). disconnect() uses the route, reaching that exact
+ * connection with no directory lookup. send() goes by id instead: it has to
+ * consult the directory anyway for the backpressure answer it returns, and by
+ * then the peer's *current* route is the better destination — a handle can
+ * outlive the link it names. info() consults the directory on demand too.
  */
 
 #include "librats/util/rats_export.h"
@@ -27,8 +29,17 @@ class RATS_API Peer {
 public:
     const PeerId& id() const noexcept { return id_; }
 
-    /// Send bytes on a named application channel.
-    void send(std::string_view channel, ByteView payload) const;
+    /// Send bytes on a named application channel, to this peer over whichever
+    /// connection currently serves it.
+    ///
+    /// @return whether that peer's queue still has room — the same answer, and the
+    ///         same contract, as Node::send(): false means stop and wait for
+    ///         on_peer_writable rather than keep going. A handler that replies
+    ///         through this handle is the most ordinary way to write to a peer, so
+    ///         it has to be able to feel backpressure like any other sender; while
+    ///         this returned void it could not, and the bytes it queued were
+    ///         invisible to peer_writable() into the bargain.
+    bool send(std::string_view channel, ByteView payload) const;
 
     /// Request this peer be disconnected.
     void disconnect() const;

package/native-src/src/librats/peer/peer_table.cpp

@@ -5,6 +5,30 @@
 
 namespace librats {
 
+namespace {
+
+/// How much a transport is worth when two connections to the same peer have to be
+/// ranked. Both ends see the same pair of transports and apply the same order, so
+/// they converge on the same survivor without exchanging a word.
+///
+///   Udp   — one socket, one NAT mapping, a source port that can be dialed back
+///           (see dialer.h). The best link there is.
+///   Tcp   — a direct link all the same, just a less useful one to a NAT.
+///   Relay — not a direct link at all: every byte costs a third node bandwidth and
+///           a round trip. Ranked last here for completeness; in practice a relayed
+///           link is settled before this by the rule in add(), which lets ANY direct
+///           link beat it regardless of role.
+int transport_rank(TransportKind t) noexcept {
+    switch (t) {
+        case TransportKind::Udp:   return 2;
+        case TransportKind::Tcp:   return 1;
+        case TransportKind::Relay: return 0;
+    }
+    return 0;
+}
+
+} // namespace
+
 PeerTable::AddOutcome PeerTable::add(const PeerInfo& info, PeerRoute route,
                                              bool prefer_outbound) {
     std::unique_lock<std::shared_mutex> lock(mutex_);
@@ -26,23 +50,32 @@ PeerTable::AddOutcome PeerTable::add(const PeerInfo& info, PeerRoute route,
     // first" gets a different answer at each end, and two ends that each keep the
     // link the other just dropped are left with no link at all.
     //
+    //   one relayed, one not ⇒ not a race at all but an upgrade (or a late relay
+    //                        arriving under a direct link): the direct one wins,
+    //                        whatever the roles. Ranked FIRST, ahead of the role
+    //                        rule, because a relayed link that won on role would
+    //                        keep costing a third node bandwidth for the life of a
+    //                        peer we can reach ourselves. Both ends see the same
+    //                        pair of transports, so both reach this same verdict.
     //   opposite roles     ⇒ cross-connect: keep the link started by the smaller id
     //                        (prefer_outbound, opposite at each end by construction).
     //   different wires    ⇒ a dial race whose attempts both got through. Both ends
-    //                        see the transports, and there is a right answer: the
-    //                        datagram link (one socket, one NAT mapping, a source
-    //                        port that can be dialed back — see dialer.h). "Is it
-    //                        UDP" ranks the pair only because there are exactly two
-    //                        wires; a third would need an explicit order, or the two
-    //                        ends could rank the same pair differently.
+    //                        see the transports and rank them by the same explicit
+    //                        order (transport_rank above), so neither can rank the
+    //                        same pair differently.
     //   same role and wire ⇒ a reconnect, not a simultaneous pair: the old link is
     //                        stale or already dead, so the newcomer is the live one.
+    const bool cur_relayed = cur.info.transport == TransportKind::Relay;
+    const bool new_relayed = info.transport == TransportKind::Relay;
+
     bool keep_new = true;
-    if (cur.info.direction != info.direction) {
+    if (cur_relayed != new_relayed) {
+        keep_new = !new_relayed;
+    } else if (cur.info.direction != info.direction) {
         const ConnRole survivor = prefer_outbound ? ConnRole::Outbound : ConnRole::Inbound;
         keep_new = (info.direction == survivor);
     } else if (cur.info.transport != info.transport) {
-        keep_new = (info.transport == TransportKind::Udp);
+        keep_new = transport_rank(info.transport) > transport_rank(cur.info.transport);
     }
 
     if (keep_new) {

package/native-src/src/librats/security/noise_security.cpp

@@ -44,6 +44,8 @@ public:
         return true;
     }
 
+    size_t overhead() const noexcept override { return librats::NOISE_TAG_SIZE; }
+
     const PeerId& remote_id() const override { return remote_id_; }
     bool is_secure() const override { return true; }
 

package/native-src/src/librats/security/plaintext_security.h

@@ -32,6 +32,7 @@ class PlaintextSession final : public Session {
 public:
     explicit PlaintextSession(PeerId remote) : remote_id_(remote) {}
     bool encrypt(ByteView plain, Bytes& out) override { out.assign(plain.begin(), plain.end()); return true; }
+    size_t overhead() const noexcept override { return 0; }
     bool decrypt(ByteView cipher, Bytes& out) override { out.assign(cipher.begin(), cipher.end()); return true; }
     const PeerId& remote_id() const override { return remote_id_; }
     bool is_secure() const override { return false; }

package/native-src/src/librats/security/session.h

@@ -30,6 +30,13 @@ public:
     /// The remote peer's identity, proven during the handshake.
     virtual const PeerId& remote_id() const = 0;
 
+    /// Bytes encrypt() adds to a plaintext of any size (an AEAD tag, typically).
+    /// The send path needs the ciphertext's size *before* encrypting it, because
+    /// a message it turns out it cannot frame must be refused without a nonce
+    /// having been spent on it — the counters run in lockstep at both ends, so a
+    /// message encrypted and then not sent would break every one after it.
+    virtual size_t overhead() const noexcept = 0;
+
     /// True if traffic is actually encrypted (false for the plaintext passthrough).
     virtual bool is_secure() const = 0;
 };