librats 1.0.2 → 2.2.0

package/native-src/src/librats/subsystems/pubsub.h

@@ -0,0 +1,175 @@
+#pragma once
+
+/**
+ * @file pubsub.h
+ * @brief Topic-based publish/subscribe — a GossipSub implementation over PeerNetwork.
+ *
+ * This is the real GossipSub, not plain floodsub. Each subscribed topic maintains
+ * a bounded *mesh* of full-message peers (formed and torn down with GRAFT/PRUNE),
+ * published messages are eagerly pushed along the mesh, and a lazy-pull gossip
+ * layer (IHAVE/IWANT, driven by the heartbeat) lets a peer recover a message it
+ * missed. Publishing to a topic we are not subscribed to uses a short-lived
+ * *fanout* set instead of a mesh. A node-id+sequence dedup key stops a message
+ * looping around the mesh, and a message cache keeps recent payloads so IWANT can
+ * be answered with real content.
+ *
+ * It depends on nothing but PeerNetwork — no Node, no reactor internals, no
+ * `friend`. A background heartbeat thread (started by start(), joined by stop())
+ * runs mesh maintenance and gossip emission once per interval.
+ *
+ * Wire format (MessageType::Gossip payload), all integers big-endian. The first
+ * byte is the op; a 40-byte message id is origin(32 id-bytes) || seqno(u64 BE):
+ *   PUBLISH     [0][origin:32][seqno:u64][topic_len:u16][topic][data]
+ *   SUBSCRIBE   [1][topic_len:u16][topic]
+ *   UNSUBSCRIBE [2][topic_len:u16][topic]
+ *   GRAFT       [3][topic_len:u16][topic]          // "add me to your mesh for topic"
+ *   PRUNE       [4][topic_len:u16][topic]          // "drop me from your mesh for topic"
+ *   IHAVE       [5][topic_len:u16][topic][count:u16][id:40]*   // "I hold these message ids"
+ *   IWANT       [6][count:u16][id:40]*                          // "send me these message ids"
+ */
+
+#include "librats/util/rats_export.h"
+#include "librats/node/peer_network.h"
+#include "librats/peer/peer.h"
+#include "librats/core/bytes.h"
+#include "librats/peer/peer_id.h"
+
+#include <atomic>
+#include <chrono>
+#include <condition_variable>
+#include <cstdint>
+#include <deque>
+#include <functional>
+#include <mutex>
+#include <random>
+#include <string>
+#include <thread>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+namespace librats {
+
+/// Outcome of validating an inbound published message before it is delivered or
+/// forwarded. REJECT drops it (and would penalise the sender in a scored build);
+/// IGNORE drops it silently; ACCEPT delivers locally and forwards along the mesh.
+enum class ValidationResult { Accept, Reject, Ignore };
+
+class RATS_API PubSub final : public Subsystem {
+public:
+    using Handler   = std::function<void(const PeerId& from, const std::string& topic, ByteView data)>;
+    using Validator = std::function<ValidationResult(const PeerId& from, const std::string& topic, ByteView data)>;
+
+    /// GossipSub tuning. Defaults mirror the libp2p reference (D=6, D_low=4,
+    /// D_high=12) and are sensible for small-to-medium meshes.
+    struct Config {
+        int mesh_target   = 6;   ///< D     — desired mesh degree per topic
+        int mesh_low      = 4;   ///< D_low — graft more peers below this
+        int mesh_high     = 12;  ///< D_high— prune peers above this
+        int fanout_size   = 6;   ///< peers used to publish to a topic we are not subscribed to
+        int gossip_factor = 6;   ///< D_lazy— peers we emit IHAVE to per heartbeat per topic
+        std::chrono::milliseconds fanout_ttl{60000};         ///< drop a fanout set idle this long
+        std::chrono::milliseconds heartbeat_interval{1000};  ///< mesh maintenance + gossip cadence
+        int    history_length = 5;     ///< heartbeat windows the message cache keeps (for IWANT)
+        int    history_gossip = 3;     ///< of those, how many windows IHAVE advertises
+        size_t seen_limit     = 8192;  ///< dedup ids remembered
+    };
+
+    PubSub();
+    explicit PubSub(Config config);
+    ~PubSub() override;
+
+    /// Subscribe to a topic and deliver matching messages to `handler`.
+    void subscribe(const std::string& topic, Handler handler);
+    void unsubscribe(const std::string& topic);
+
+    /// Publish `data` on `topic`: along the mesh if we are subscribed, else via fanout.
+    void publish(const std::string& topic, ByteView data);
+
+    bool                     is_subscribed(const std::string& topic) const;
+    std::vector<std::string> subscribed_topics() const;
+    std::vector<PeerId>      peers_for_topic(const std::string& topic) const;  ///< known subscribers
+    std::vector<PeerId>      mesh_peers(const std::string& topic) const;       ///< our mesh for topic
+
+    /// Gate inbound messages for a topic; `topic == ""` installs a global validator
+    /// used when no per-topic validator is registered.
+    void set_validator(const std::string& topic, Validator validator);
+
+    // Subsystem.
+    void attach(NodeContext& ctx) override;
+    void start() override;  ///< launch the heartbeat thread
+    void stop() override;   ///< stop and join it
+
+private:
+    struct Topic {
+        std::unordered_set<PeerId, PeerId::Hash> subscribers;  ///< peers that announced interest
+        std::unordered_set<PeerId, PeerId::Hash> mesh;         ///< full-message peering (we subscribe)
+        std::unordered_set<PeerId, PeerId::Hash> fanout;       ///< publish targets when not subscribed
+        std::chrono::steady_clock::time_point    last_fanout{};
+    };
+
+    struct CachedMessage {
+        std::string topic;
+        Bytes       frame;  ///< the whole PUBLISH frame, resent verbatim to answer IWANT
+    };
+
+    // Inbound dispatch (all run on a reactor thread).
+    void on_new_peer(const Peer& peer);
+    void on_peer_gone(const PeerId& id);
+    void on_gossip(const Peer& peer, ByteView payload);
+
+    void recv_subscription(const PeerId& from, const std::string& topic, bool subscribe);
+    void recv_graft(const PeerId& from, const std::string& topic);
+    void recv_prune(const PeerId& from, const std::string& topic);
+    void recv_publish(const PeerId& from, ByteView frame, const PeerId& origin, uint64_t seqno,
+                      const std::string& topic, ByteView data);
+    void recv_ihave(const PeerId& from, const std::string& topic, const std::vector<std::string>& ids);
+    void recv_iwant(const PeerId& from, const std::vector<std::string>& ids);
+
+    // Heartbeat (background thread).
+    void heartbeat_loop();
+    void do_heartbeat();
+
+    /// Bring `topic`'s mesh toward [mesh_low, mesh_high] around mesh_target, queuing
+    /// the GRAFT/PRUNE control messages to send after the lock is released. Requires
+    /// `mutex_` held; a no-op unless we are subscribed to `topic`.
+    using CtrlList = std::vector<std::pair<PeerId, std::string>>;
+    void maintain_mesh_locked(const std::string& topic, CtrlList& grafts, CtrlList& prunes);
+
+    // Helpers.
+    void send_ctrl(const PeerId& to, uint8_t op, const std::string& topic);
+    void broadcast_ctrl(uint8_t op, const std::string& topic);
+    void deliver_local(const PeerId& from, const std::string& topic, ByteView data);
+    Handler          handler_for(const std::string& topic) const;
+    ValidationResult validate(const PeerId& from, const std::string& topic, ByteView data);
+    bool             mark_seen(const std::string& id);             ///< false if already seen
+    void             cache_message(const std::string& id, const std::string& topic, const Bytes& frame);
+    std::vector<PeerId> random_sample(std::vector<PeerId> in, int n);
+
+    PeerNetwork* network_ = nullptr;
+    Config       config_;
+
+    mutable std::mutex mutex_;  ///< guards subscriptions_, topics_, validators_, seqno_
+    uint64_t           seqno_ = 0;
+    std::unordered_map<std::string, Handler>   subscriptions_;  ///< topic -> local handler
+    std::unordered_map<std::string, Topic>     topics_;         ///< topic -> peering state
+    std::unordered_map<std::string, Validator> validators_;     ///< topic -> validator
+    Validator                                  global_validator_;
+
+    mutable std::mutex mcache_mutex_;  ///< guards mcache_, history_, seen_*
+    std::unordered_map<std::string, CachedMessage> mcache_;     ///< id -> cached frame (for IWANT)
+    std::deque<std::vector<std::string>>           history_;    ///< gossip windows of ids (front = newest)
+    std::unordered_set<std::string>                seen_;       ///< dedup set
+    std::deque<std::string>                        seen_order_; ///< FIFO eviction order for seen_
+
+    std::mutex   rng_mutex_;
+    std::mt19937 rng_;
+
+    std::atomic<bool>       running_{false};
+    std::thread             heartbeat_thread_;
+    std::mutex              hb_mutex_;
+    std::condition_variable hb_cv_;
+};
+
+} // namespace librats

package/native-src/src/librats/subsystems/reconnection.cpp

@@ -0,0 +1,239 @@
+#include "librats/subsystems/reconnection.h"
+#include "librats/node/node_context.h"
+#include "librats/util/logger.h"
+
+#include <algorithm>
+#include <chrono>
+#include <unordered_set>
+#include <vector>
+
+namespace librats {
+
+namespace {
+// Wall clock at the edge, kept out of PeerBook so the book stays pure/testable.
+uint64_t now_secs() {
+    return static_cast<uint64_t>(std::chrono::duration_cast<std::chrono::seconds>(
+        std::chrono::system_clock::now().time_since_epoch()).count());
+}
+} // namespace
+
+ReconnectionService::ReconnectionService() : ReconnectionService(Config()) {}
+
+ReconnectionService::ReconnectionService(Config config) : config_(std::move(config)) {
+    // Build the book up front so the pointer is fixed for the object's life (no race
+    // with reactor-thread reads in on_connected) and so add() before start() records
+    // into an already-loaded book rather than clobbering it.
+    if (!config_.store_path.empty()) {
+        book_ = std::make_unique<PeerBook>(config_.store_path);
+        book_->load();
+        // Age out the long tail and cap the archive immediately on load.
+        book_->prune(now_secs(), static_cast<uint64_t>(config_.archive_max_age.count()), config_.archive_max);
+        book_->save();
+    }
+}
+
+ReconnectionService::~ReconnectionService() { stop(); }
+
+void ReconnectionService::add(const Address& address) {
+    {
+        std::lock_guard<std::mutex> lock(mutex_);
+        if (targets_.count(address)) return;
+        if (targets_.size() >= config_.max_targets) {
+            LOG_WARN("reconnect", "Active-target cap (" << config_.max_targets
+                                  << ") reached; dropping " << address.to_string());
+            return;
+        }
+        Target& t = targets_[address];
+        t.address = address;
+        t.next_attempt = std::chrono::steady_clock::now();
+    }
+    if (book_) { book_->note_seen(address, now_secs()); book_->save(); }
+    wake_.notify_all();
+}
+
+void ReconnectionService::remove(const Address& address) {
+    bool erased;
+    {
+        std::lock_guard<std::mutex> lock(mutex_);
+        erased = targets_.erase(address) > 0;
+    }
+    if (book_ && book_->remove(address)) book_->save();
+    if (erased) LOG_DEBUG("reconnect", "Stopped reconnecting to " << address.to_string());
+}
+
+size_t ReconnectionService::target_count() const {
+    std::lock_guard<std::mutex> lock(mutex_);
+    return targets_.size();
+}
+
+std::vector<Address> ReconnectionService::known_peers(size_t n) const {
+    if (!book_) return {};
+    return book_->best(n, now_secs(), static_cast<uint64_t>(config_.archive_max_age.count()));
+}
+
+void ReconnectionService::attach(NodeContext& ctx) {
+    network_ = &ctx.network;
+    network_->on_peer_connected([this](const Peer& peer) { on_connected(peer); });
+    network_->on_peer_disconnected([this](const PeerId& id) { on_disconnected(id); });
+    network_->on_dial_failed([this](const Address& addr) { on_dial_failed(addr); });
+}
+
+void ReconnectionService::start() {
+    if (running_.exchange(true)) return;
+
+    // Seed the active set with the most promising peers from the book — the
+    // working set is "best N recent", the rest of the book stays a passive
+    // reserve pool reachable via known_peers().
+    if (book_) {
+        const auto recent = book_->best(config_.startup_targets, now_secs(),
+                                        static_cast<uint64_t>(config_.archive_max_age.count()));
+        std::lock_guard<std::mutex> lock(mutex_);
+        for (const Address& addr : recent) {
+            if (targets_.size() >= config_.max_targets) break;
+            auto [it, inserted] = targets_.try_emplace(addr);
+            if (inserted) { it->second.address = addr; it->second.next_attempt = std::chrono::steady_clock::now(); }
+        }
+    }
+    thread_ = std::thread(&ReconnectionService::loop, this);
+}
+
+void ReconnectionService::stop() {
+    if (!running_.exchange(false)) return;
+    wake_.notify_all();
+    if (thread_.joinable()) thread_.join();
+    if (book_) book_->save();
+}
+
+// A successful connection. Two jobs: record the address in the book, and — the
+// success-side mirror of on_dial_failed — clear this target's in-flight dial state
+// right now. We must NOT wait for the loop to reconcile from the live-peer snapshot:
+// a peer that connects and then drops again BETWEEN ticks would otherwise leave its
+// dial flagged in-flight until dial_timeout, needlessly stalling the redial for
+// seconds even though the address is plainly reachable. (For an inbound peer
+// info->addresses is still empty at this point — its dialable address arrives later
+// via identify — so an inbound link matches no target here and is reconciled by the
+// loop's live snapshot instead; inbound links are thus not auto-persisted either.)
+void ReconnectionService::on_connected(const Peer& peer) {
+    auto info = peer.info();
+    if (!info) return;
+
+    const uint64_t now       = now_secs();
+    const auto     steady_now = std::chrono::steady_clock::now();
+    bool book_changed = false;
+    {
+        std::lock_guard<std::mutex> lock(mutex_);
+        for (const Address& addr : info->addresses) {
+            auto it = targets_.find(addr);
+            if (it == targets_.end()) {
+                if (!config_.persist_discovered) continue;
+                if (targets_.size() >= config_.max_targets) continue;  // bound active growth
+                it = targets_.emplace(addr, Target{}).first;
+                it->second.address = addr;
+            }
+            it->second.mark_connected(steady_now);  // resolve the in-flight dial now
+            if (book_) { book_->note_connected(addr, peer.id(), now); book_changed = true; }
+        }
+    }
+    if (book_ && book_changed) book_->save();
+}
+
+// A peer dropped. The loop reconciles which targets are live from the directory
+// snapshot, so there is no per-target state to clear here — just wake it so the
+// re-dial happens at once rather than waiting out a tick.
+void ReconnectionService::on_disconnected(const PeerId& /*id*/) {
+    wake_.notify_all();
+}
+
+// An outbound dial we asked for closed before establishing. Clear the in-flight
+// flag so the target becomes eligible again — it then re-dials on its own backoff
+// schedule (next_attempt, set when we dispatched the dial). Without this signal
+// the target would stay "dialing" until the dial_deadline backstop, needlessly
+// stretching every retry to that timeout.
+void ReconnectionService::on_dial_failed(const Address& address) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    auto it = targets_.find(address);
+    if (it != targets_.end()) it->second.dialing = false;
+}
+
+std::chrono::milliseconds ReconnectionService::backoff_for(int attempts) const {
+    // base * 2^(attempts-1), capped at max.
+    auto delay = config_.base_backoff;
+    for (int i = 1; i < attempts && delay < config_.max_backoff; ++i) delay *= 2;
+    return (std::min)(delay, config_.max_backoff);  // parenthesized: dodge windows.h min macro
+}
+
+void ReconnectionService::loop() {
+    std::unordered_set<Address> live;  // reused across ticks to avoid re-allocating
+    while (running_.load()) {
+        const auto now = std::chrono::steady_clock::now();
+
+        // Ground truth for "this target is already connected": the union of every
+        // live peer's dialable addresses, identify-learned ones included. Matching
+        // on this — instead of a flag set only on the on_peer_connected event — is
+        // what lets us recognise a peer reached over a route we did not dial (an
+        // inbound link, or a cross-connect the directory resolved to the other
+        // connection). Without it such a peer is never seen as connected and we
+        // re-dial it forever, each dial torn down by the node as a duplicate.
+        live.clear();
+        for (const PeerInfo& p : network_->peers())
+            for (const Address& a : p.addresses)
+                live.insert(a);
+
+        std::vector<Address> to_dial, gave_up;
+        {
+            std::lock_guard<std::mutex> lock(mutex_);
+            for (auto it = targets_.begin(); it != targets_.end();) {
+                Target& target = it->second;
+
+                // Connected right now (over any route): keep the retry state armed
+                // so a later drop re-dials promptly, and never dial a live peer.
+                if (live.count(it->first)) {
+                    target.mark_connected(now);
+                    ++it;
+                    continue;
+                }
+
+                // A dial is already in flight: don't pile on a duplicate while the
+                // handshake completes. We clear `dialing` on dial failure; the next
+                // tick clears it once the peer shows up live; dial_deadline is only a
+                // backstop for the case where neither signal ever arrives.
+                if (target.dialing && now < target.dial_deadline) { ++it; continue; }
+                if (target.next_attempt > now) { ++it; continue; }
+
+                // Give up actively dialing a persistently-dead target: drop it from
+                // the active set (it stays in the book as history, ranked down by its
+                // failure streak, until it ages out). A reconnect resets attempts, so
+                // only addresses that never came back are reaped.
+                if (config_.max_attempts > 0 && target.attempts >= static_cast<int>(config_.max_attempts)) {
+                    gave_up.push_back(target.address);
+                    it = targets_.erase(it);
+                    continue;
+                }
+
+                to_dial.push_back(target.address);
+                target.attempts++;
+                target.dialing = true;
+                target.dial_deadline = now + config_.dial_timeout;
+                target.next_attempt = now + backoff_for(target.attempts);
+                ++it;
+            }
+        }
+        if (book_ && !gave_up.empty()) {
+            const uint64_t ts = now_secs();
+            for (const Address& addr : gave_up) book_->note_failure(addr, ts);
+            book_->save();
+        }
+        for (const Address& addr : gave_up)
+            LOG_DEBUG("reconnect", "Giving up on " << addr.to_string() << " after "
+                      << config_.max_attempts << " attempts");
+        for (const Address& addr : to_dial) {
+            LOG_DEBUG("reconnect", "Dialing " << addr.to_string());
+            network_->connect(addr);
+        }
+
+        std::unique_lock<std::mutex> lock(wait_mutex_);
+        wake_.wait_for(lock, config_.tick, [this] { return !running_.load(); });
+    }
+}
+
+} // namespace librats

package/native-src/src/librats/subsystems/reconnection.h

@@ -0,0 +1,126 @@
+#pragma once
+
+/**
+ * @file reconnection.h
+ * @brief Keeps connections to a set of peer addresses alive with backoff.
+ *
+ * A Subsystem that remembers peer addresses (optionally persisted via PeerBook)
+ * and re-dials them when they drop, using exponential backoff. Built only on
+ * PeerNetwork: it dials via connect() and, each tick, reconciles its targets
+ * against the set of currently-connected peers (PeerNetwork::peers(), whose
+ * addresses include the dialable endpoints learned via identify). A target whose
+ * address is served by a live peer — over ANY route, inbound or outbound — is
+ * left alone; only the rest are dialed. Reconciling against this snapshot, rather
+ * than trusting a flag toggled on the connect event, is what stops us from
+ * endlessly re-dialing a peer already connected over a link we did not initiate
+ * (an inbound link, or a cross-connect the directory resolved to the other side).
+ *
+ * A target is dropped when the app calls remove(), or — if Config::max_attempts
+ * is set — after that many consecutive failed dials without ever connecting (it
+ * "gives up", freeing memory and pruning the store). A successful connection
+ * resets the attempt counter, so only persistently-dead addresses are reaped.
+ */
+
+#include "librats/util/rats_export.h"
+#include "librats/node/peer_network.h"
+#include "librats/peer/peer.h"
+#include "librats/core/address.h"
+#include "librats/peer/peer_id.h"
+#include "librats/peer/peer_book.h"
+
+#include <atomic>
+#include <chrono>
+#include <condition_variable>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <thread>
+#include <unordered_map>
+
+namespace librats {
+
+class RATS_API ReconnectionService final : public Subsystem {
+public:
+    struct Config {
+        std::string               store_path = "";          ///< persist the peer book here (empty = memory only)
+        bool                      persist_discovered = true; ///< remember dialed peers automatically
+        size_t                    max_targets = 1024;        ///< cap on ACTIVE re-dial targets (bounds memory + dial fan-out)
+        size_t                    max_attempts = 0;          ///< give up actively dialing a target after this many consecutive failures; 0 = retry forever
+        size_t                    startup_targets = 32;      ///< on start, actively re-dial this many best peers from the book
+        size_t                    archive_max = 4096;        ///< cap on the persistent peer book (history of everyone we met)
+        std::chrono::seconds      archive_max_age{std::chrono::hours(24 * 30)};  ///< forget peers unseen this long (30 days)
+        std::chrono::milliseconds base_backoff{1000};
+        std::chrono::milliseconds max_backoff{60000};
+        std::chrono::milliseconds tick{1000};
+        std::chrono::milliseconds dial_timeout{15000};  ///< assume an in-flight dial is dead after this if no connect/fail event arrives (backstop only)
+    };
+
+    ReconnectionService();
+    explicit ReconnectionService(Config config);
+    ~ReconnectionService() override;
+
+    /// Register an address to keep connected. Persists it if a store is configured.
+    void add(const Address& address);
+
+    /// Stop reconnecting to an address: drops it as a target and from the store.
+    /// Use when the application intentionally parts with a peer and does not want
+    /// it re-dialed. (A later fresh connection to it may re-learn it if
+    /// persist_discovered is on.)
+    void remove(const Address& address);
+
+    size_t target_count() const;
+
+    /// The passive reserve pool: up to n best-known peer addresses from the book
+    /// (history of everyone we have connected to), most promising first. Not dialed
+    /// automatically beyond the startup seed — exposed for the app / discovery to
+    /// use as a bootstrap source when peer count is low.
+    std::vector<Address> known_peers(size_t n) const;
+
+    void attach(NodeContext& ctx) override;
+    void start() override;
+    void stop() override;
+
+private:
+    struct Target {
+        Address                               address;
+        bool                                  dialing = false;  ///< a connect() is in flight; don't pile on a duplicate
+        int                                   attempts = 0;     ///< failed dials since the target was last seen connected
+        std::chrono::steady_clock::time_point next_attempt;     ///< earliest time to (re)dial
+        std::chrono::steady_clock::time_point dial_deadline;    ///< backstop: give up waiting on the in-flight dial after this
+
+        /// Record that this target is connected as of `now`: no dial is in flight
+        /// any more and the failure streak resets, so a later drop re-dials at once
+        /// (an address that just worked earns no backoff penalty). Used by BOTH the
+        /// loop's live-snapshot reconciliation and the on_connected event, so the
+        /// two paths that clear an in-flight dial can never drift apart.
+        void mark_connected(std::chrono::steady_clock::time_point now) {
+            dialing      = false;
+            attempts     = 0;
+            next_attempt = now;
+        }
+    };
+
+    void on_connected(const Peer& peer);
+    void on_disconnected(const PeerId& id);
+    void on_dial_failed(const Address& address);
+    void loop();
+    std::chrono::milliseconds backoff_for(int attempts) const;
+
+    Config                      config_;
+    PeerNetwork*                network_ = nullptr;
+    // Built once in the constructor (when store_path is set) and never reassigned,
+    // so the pointer is safe to read from any thread; PeerBook is itself internally
+    // synchronized. (Creating it in start() raced reads from on_connected, which can
+    // fire on a reactor thread before this subsystem's start() returns.)
+    std::unique_ptr<PeerBook>   book_;
+
+    std::thread             thread_;
+    std::atomic<bool>       running_{false};
+    std::mutex              wait_mutex_;
+    std::condition_variable wake_;
+
+    mutable std::mutex                       mutex_;
+    std::unordered_map<Address, Target>      targets_;  ///< keyed by the peer Address
+};
+
+} // namespace librats