librats 1.0.2 → 2.1.4

package/native-src/src/librats/transport/link.h

@@ -0,0 +1,96 @@
+#pragma once
+
+/**
+ * @file link.h
+ * @brief The byte-stream a Connection runs on, independent of how it is carried.
+ *
+ * Everything above this line — block framing, the Noise handshake, the Session,
+ * backpressure, the establish deadline — only ever needs an ordered, reliable,
+ * non-blocking byte stream. A Link is exactly that and nothing more, so the same
+ * Connection code drives a kernel TCP socket (TcpLink) and the library's own
+ * reliability layer over datagrams (UdpStreamLink → UdpStream). Neither transport
+ * is a special case of the other: both are first-class, and a peer connected over
+ * either is indistinguishable to every layer above.
+ *
+ * Ownership and threading follow the rest of the transport layer: a Link belongs
+ * to exactly one Connection, which belongs to exactly one Reactor, and is touched
+ * only by that reactor's thread. It therefore holds no locks and no atomics.
+ *
+ * The read/write calls mirror non-blocking socket semantics deliberately, because
+ * that is what the reactor loop is built around:
+ *   - read()  fills the caller's span and reports WouldBlock once the link has no
+ *             more data ready, or Closed when the peer has finished sending.
+ *   - write() takes what it can and reports WouldBlock when it can take no more;
+ *             the Connection keeps the remainder queued and calls want_write(true)
+ *             so it is told when to retry.
+ */
+
+#include "librats/core/address.h"
+#include "librats/core/bytes.h"
+#include "librats/core/socket.h"
+#include "librats/core/types.h"
+
+#include <cstddef>
+#include <optional>
+
+namespace librats {
+
+class Link {
+public:
+    virtual ~Link() = default;
+
+    /// Outcome of a read/write attempt. `Closed` is orderly (the peer is done
+    /// sending); `Error` means the link is broken and must be torn down.
+    enum class Status { Ok, WouldBlock, Closed, Error };
+
+    struct IoResult {
+        size_t bytes  = 0;
+        Status status = Status::WouldBlock;
+    };
+
+    virtual TransportKind kind() const noexcept = 0;
+
+    /// Read up to `into.size()` bytes. A Closed or Error result always carries
+    /// zero bytes — data and end-of-stream are never reported together, so the
+    /// caller drains first and only then sees the close.
+    virtual IoResult read(ByteSpan into) = 0;
+
+    /// Hand `count` contiguous slices to the link in order, writing as many bytes
+    /// as it will take (a partial write is normal and expected).
+    virtual IoResult write(const ByteView* slices, size_t count) = 0;
+
+    /// Called once the link reports writable while the connection is still
+    /// connecting. Returns false if the connect attempt failed.
+    virtual bool connect_completed() = 0;
+
+    /// Ask to be woken (via Connection::on_writable) when more can be written.
+    /// Called only when the state actually changes, so an implementation may
+    /// treat it as an unconditional set.
+    virtual void want_write(bool on) = 0;
+
+    /// The peer's endpoint. For TCP the port is the peer's *ephemeral* source
+    /// port, so only the IP is meaningful (see identify); for UDP it is the
+    /// endpoint datagrams are exchanged with. nullopt if not yet known.
+    virtual std::optional<Address> remote_endpoint() const = 0;
+
+    /// The kernel socket backing this link, or RATS_INVALID_SOCKET when the link
+    /// has none of its own (a UDP stream shares the mux's socket with every other
+    /// stream). The Reactor uses it to decide whether the link is poller-driven.
+    virtual socket_t fd() const noexcept { return RATS_INVALID_SOCKET; }
+
+    /// Told the ConnId the Reactor assigned, right after adoption. A link that has
+    /// to be reachable from somewhere other than its Connection (a UDP stream, from
+    /// the datagram path) needs it; a plain socket does not.
+    virtual void attach(ConnId) {}
+
+    /// Why the link broke, consulted when read/write reported Error. Lets a
+    /// transport that knows more than "the socket failed" — a dial that was never
+    /// answered, a peer that went silent — say so.
+    virtual CloseReason error_reason() const { return CloseReason::PeerReset; }
+
+    /// Begin an orderly shutdown. Called by the Reactor as it tears the connection
+    /// down; the Link may still linger internally to flush what it owes the peer.
+    virtual void close(CloseReason reason) = 0;
+};
+
+} // namespace librats

package/native-src/src/librats/transport/reactor.cpp

@@ -0,0 +1,540 @@
+#include "librats/transport/reactor.h"
+#include "librats/transport/tcp_link.h"
+#include "librats/core/ip_address.h"
+#include "librats/util/logger.h"
+#include "librats/util/network_utils.h"
+
+#include <cstring>
+#include <utility>
+
+namespace librats {
+
+namespace {
+
+/// Turn a dial target into the numeric endpoint the datagram transport needs.
+/// A literal is used as-is; a hostname is resolved in the order this socket can
+/// actually use.
+///
+/// The order matters as much as the result. One mux socket serves every peer and
+/// is bound in one family, which the default configuration makes IPv4 — so asking
+/// for AAAA first, as TCP can afford to, would keep handing back addresses this
+/// socket cannot send to while the same host advertises a perfectly dialable A
+/// record. Answering with a reachable address is the whole job here.
+std::optional<Address> resolve_dial_target(const std::string& host, int port,
+                                           AddressFamily af) {
+    if (auto ip = IpAddress::parse(host))
+        return Address{*ip, static_cast<uint16_t>(port)};
+
+    if (af != AddressFamily::IPv4) {
+        if (auto ip = IpAddress::parse(network_utils::resolve_hostname_v6(host)))
+            return Address{*ip, static_cast<uint16_t>(port)};
+    }
+    if (af != AddressFamily::IPv6) {
+        if (auto ip = IpAddress::parse(network_utils::resolve_hostname(host)))
+            return Address{*ip, static_cast<uint16_t>(port)};
+    }
+    return std::nullopt;
+}
+
+} // namespace
+
+Reactor::Reactor(uint8_t index, ConnectionDelegate& delegate, SecurityProvider& security)
+    : index_(index), delegate_(delegate), security_(security), poller_(IOPoller::create()) {}
+
+Reactor::~Reactor() {
+    stop();
+}
+
+// ── Lifecycle ───────────────────────────────────────────────────────────────
+
+void Reactor::listen(socket_t server_socket) {
+    server_socket_ = server_socket;
+}
+
+void Reactor::listen_udp(socket_t udp_socket, AddressFamily family) {
+    mux_ = std::make_unique<UdpMux>(udp_socket, family, *this);
+}
+
+void Reactor::start() {
+    if (running_.exchange(true)) return;
+    thread_ = std::thread(&Reactor::run, this);
+}
+
+void Reactor::stop() {
+    if (!running_.exchange(false)) {
+        if (thread_.joinable()) thread_.join();
+        return;
+    }
+    wakeup_.signal();              // break the poll wait
+    if (thread_.joinable()) thread_.join();
+}
+
+// ── Work submission ─────────────────────────────────────────────────────────
+
+bool Reactor::on_reactor_thread() const noexcept {
+    return std::this_thread::get_id() == thread_id_.load(std::memory_order_acquire);
+}
+
+void Reactor::post(Task task) {
+    // After stop() the loop has exited and (apart from the brief stopping window
+    // caught by the final drain in run()) the task will never run. Enqueue anyway
+    // so its captures are released at destruction, but flag the dropped work.
+    if (!running_.load(std::memory_order_acquire))
+        LOG_DEBUG("reactor", "Reactor " << static_cast<int>(index_)
+                  << " post() after stop; task may not run");
+    tasks_.push(std::move(task));
+    wakeup_.signal();
+}
+
+void Reactor::execute(Task task) {
+    if (on_reactor_thread()) task();
+    else                     post(std::move(task));
+}
+
+ConnId Reactor::connect(std::string host, int port, TransportKind kind, DialProfile profile) {
+    if (kind == TransportKind::Udp && !mux_) {
+        LOG_DEBUG("reactor", "Reactor " << static_cast<int>(index_)
+                  << " has no datagram transport; refusing UDP dial to " << host << ":" << port);
+        return kInvalidConnId;
+    }
+
+    // Reserve the id here, on the calling thread, rather than inside the task. The
+    // caller can then cancel this exact attempt (close()) before the task has even
+    // run — which is what makes racing one transport against the other cancellable
+    // instead of a matter of luck.
+    const ConnId id = next_conn_id_.fetch_add(1, std::memory_order_relaxed);
+    post([this, host = std::move(host), port, kind, profile, id] {
+        start_dial(id, host, port, kind, profile);
+    });
+    return id;
+}
+
+void Reactor::start_dial(ConnId id, const std::string& host, int port, TransportKind kind,
+                         DialProfile profile) {
+    if (kind == TransportKind::Udp) {
+        const auto target = resolve_dial_target(host, port, mux_->family());
+        if (!target) {
+            LOG_DEBUG("reactor", "Cannot resolve " << host << " for a UDP dial");
+            abort_dial(id, host, port);
+            return;
+        }
+        // The one mux socket cannot leave its own address family, and that is
+        // knowable here rather than after the fact. Saying so now costs nothing —
+        // the dialer brings the other transport in at once. Opening the stream
+        // anyway would spend the Syn's whole give-up on datagrams the kernel
+        // refuses one by one, and refusals at that layer are per-packet and
+        // silent: a dial that never had a chance would be indistinguishable from
+        // a peer that is merely slow to answer.
+        if (!family_can_reach(mux_->family(), target->ip.is_v6())) {
+            LOG_DEBUG("reactor", "Datagram socket cannot reach " << target->to_string()
+                      << "; leaving " << host << ":" << port << " to the other transport");
+            abort_dial(id, host, port);
+            return;
+        }
+        std::unique_ptr<Link> link = mux_->connect(*target, profile);
+        if (!link) { abort_dial(id, host, port); return; }
+
+        Connection* conn = adopt(std::move(link), ConnRole::Outbound, id);
+        if (conn) conn->set_dial_address(host, static_cast<uint16_t>(port));
+        return;
+    }
+
+    socket_t sock = tcp_connect_start(host, port);
+    if (!is_valid_socket(sock)) {
+        LOG_DEBUG("reactor", "Outbound connect to " << host << ":" << port << " failed to start");
+        abort_dial(id, host, port);
+        return;
+    }
+    Connection* conn = adopt(std::make_unique<TcpLink>(sock, *this), ConnRole::Outbound, id);
+    if (!conn) { close_socket(sock); return; }   // the dial was cancelled while queued
+    conn->set_dial_address(host, static_cast<uint16_t>(port));
+}
+
+void Reactor::abort_dial(ConnId id, const std::string& host, int port) {
+    resolve_dial(id);  // the id is spent either way; release any cancellation slot
+    delegate_.on_dial_aborted(index_, id, host, static_cast<uint16_t>(port));
+}
+
+void Reactor::close(ConnId id, CloseReason reason) {
+    close_conn(id, reason, /*spare_established=*/false);
+}
+
+void Reactor::cancel_dial(ConnId id, CloseReason reason) {
+    close_conn(id, reason, /*spare_established=*/true);
+}
+
+void Reactor::close_conn(ConnId id, CloseReason reason, bool spare_established) {
+    execute([this, id, reason, spare_established] {
+        auto it = conns_.find(id);
+        if (it != conns_.end()) {
+            // It got there first: the dial it was is over, and what it left behind
+            // is a connection to a peer. Whether that one or another link survives
+            // is decided by the peer table on both ends alike — see peer_table.cpp.
+            if (spare_established && it->second->state() == ConnState::Established) return;
+            mark_for_close(id, reason);
+            return;
+        }
+        // Not adopted yet: its connect task is still queued behind this one. Leave
+        // a note so adopt() drops it on arrival. Ids at or below the watermark have
+        // already come and gone, so they need no slot (and cannot leak one).
+        if (id != kInvalidConnId && id > resolved_watermark_) cancelled_dials_.insert(id);
+    });
+}
+
+void Reactor::broadcast(FrameHeader header, std::shared_ptr<const Bytes> payload) {
+    execute([this, header, payload = std::move(payload)] {
+        for (auto& [id, conn] : conns_) {
+            if (conn->state() == ConnState::Established)
+                conn->send(header, ByteView(*payload));
+        }
+    });
+}
+
+// ── Timers ──────────────────────────────────────────────────────────────────
+
+TimerId Reactor::schedule(std::chrono::milliseconds delay, Task on_fire) {
+    return timers_.schedule(delay, std::move(on_fire));
+}
+
+void Reactor::cancel(TimerId id) {
+    timers_.cancel(id);
+}
+
+// ── Lookups / interest ──────────────────────────────────────────────────────
+
+Connection* Reactor::find(ConnId id) noexcept {
+    auto it = conns_.find(id);
+    return it == conns_.end() ? nullptr : it->second.get();
+}
+
+void Reactor::set_interest(socket_t sock, uint32_t events) {
+    poller_->modify(sock, events);
+}
+
+// ── Reactor loop ────────────────────────────────────────────────────────────
+
+void Reactor::run() {
+    thread_id_ = std::this_thread::get_id();
+    LOG_INFO("reactor", "Reactor " << static_cast<int>(index_)
+             << " started (backend: " << poller_->name() << ")");
+
+    set_socket_nonblocking(wakeup_.fd());
+    poller_->add(wakeup_.fd(), PollIn);
+    if (is_valid_socket(server_socket_)) {
+        set_socket_nonblocking(server_socket_);
+        poller_->add(server_socket_, PollIn);
+    }
+    if (mux_) {
+        set_socket_nonblocking(mux_->socket());
+        poller_->add(mux_->socket(), PollIn);
+    }
+
+    PollResult        events[kMaxEvents];
+    std::vector<Task> task_batch;
+
+    schedule_maintenance();
+
+    while (running_.load(std::memory_order_relaxed)) {
+        int timeout = timers_.next_timeout_ms(kMaxPollMs);
+        // The datagram transport keeps its own deadlines — retransmission timeouts,
+        // delayed acks, keep-alives — rather than being swept on a fixed cadence, so
+        // the wait has to cover the earliest of them too. With no stream due it
+        // contributes nothing, and an idle mux costs no wake-ups at all.
+        if (mux_) timeout = mux_->next_timeout_ms(timeout);
+        const int n = poller_->wait(events, kMaxEvents, timeout);
+
+        drain_tasks(task_batch);                         // connect/close/send-arm
+        for (int i = 0; i < n; ++i) handle_event(events[i]);
+        timers_.run_due();
+        // Service whatever datagram deadlines have come due. Deliberately not a
+        // TimerQueue entry: those deadlines move on nearly every packet, and
+        // re-scheduling a timer that often would cost far more than the one
+        // comparison this is when nothing is due. Before process_pending_close(),
+        // so a stream that dies here is torn down in the same turn.
+        if (mux_) mux_->tick();
+        // Everything queued by send() during this turn goes to the wire here, in
+        // one write per connection rather than one per frame. Deliberately after
+        // the mux tick — a handler it dispatched may well have sent something —
+        // and before the close pass, so a connection that dies on the write is
+        // torn down in the same turn.
+        flush_dirty();
+        process_pending_close();
+        // The mux batches its datagrams; the ones it collected inside a readable
+        // event or a tick have already gone out, but an application send reaches a
+        // UDP stream from a task, outside any batch of its own. Flushing once here
+        // covers those without giving any of them a timer to wait on.
+        if (mux_) mux_->flush_output();
+    }
+
+    // Graceful drain: run whatever was queued before/around stop() so in-flight
+    // sends flush and explicit closes settle, then tear everything down. (post()
+    // may still race in after this; such tasks are dropped — see post().)
+    drain_tasks(task_batch);
+    flush_dirty();               // in-flight sends leave before the door closes
+    process_pending_close();
+    shutdown_connections();
+    if (mux_) mux_->shutdown();   // after the connections, so no Link outlives it
+
+    // Timers outlive the loop that scheduled them unless the heap goes with it,
+    // and schedule_maintenance() re-arms itself: an entry left behind would start
+    // a second self-perpetuating chain on the next run(), so N restarts would cost
+    // N+1 sweeps per interval and N+1 timed wake-ups, for as long as the node
+    // lives. Whatever else is still pending — establish deadlines, backoffs —
+    // belongs to connections that no longer exist.
+    timers_.clear();
+
+    // Hand back the sockets this loop registered above. A registration belongs to a
+    // descriptor, and a stopped reactor can be started again on freshly opened ones
+    // — which the kernel is free to hand the very same descriptor numbers, and on
+    // the BSDs, which allocate the lowest free one, very nearly always does. The
+    // backends that keep a registration table of their own (kqueue, poll, IOCP)
+    // would then refuse to add a descriptor they believe is already watched, while
+    // the kernel forgot it the moment it was closed — leaving a restarted node
+    // polling nothing at all: no accepts, no datagrams, every dial to it timing out.
+    // epoll drops a closed descriptor by itself, which is why this only ever showed
+    // on macOS.
+    //
+    // The listen sockets themselves are the node's to close, and it does so after
+    // joining this thread — so they are still open here, which is exactly when a
+    // registration can still be withdrawn cleanly.
+    poller_->remove(wakeup_.fd());
+    if (is_valid_socket(server_socket_)) {
+        poller_->remove(server_socket_);
+        // Forgotten as well as unwatched: the next run() must not re-add a
+        // descriptor the node closed in between, which by then may name something
+        // else entirely. listen() hands over the new one before every start.
+        server_socket_ = RATS_INVALID_SOCKET;
+    }
+    if (mux_) {
+        poller_->remove(mux_->socket());
+        mux_.reset();
+    }
+
+    LOG_INFO("reactor", "Reactor " << static_cast<int>(index_) << " stopped");
+}
+
+void Reactor::drain_tasks(std::vector<Task>& scratch) {
+    tasks_.drain(scratch);
+    for (auto& task : scratch) task();
+}
+
+void Reactor::drain_wakeup() {
+    wakeup_.drain();
+}
+
+void Reactor::handle_event(const PollResult& ev) {
+    const socket_t fd = ev.fd;
+
+    if (fd == wakeup_.fd()) { drain_wakeup(); return; }
+    if (fd == server_socket_) { if (ev.events & PollIn) do_accept(); return; }
+    if (mux_ && fd == mux_->socket()) {
+        // Any event, not just PollIn. An error on a datagram socket belongs to one
+        // datagram — Windows reports an ICMP complaint about an earlier send on the
+        // *next* receive — and not to the socket, which stays usable; the cure is
+        // therefore to read, which surfaces or clears it and drains whatever queued
+        // up behind it. Treating PollErr as "nothing to do" would strand every
+        // datagram after it, since nothing else here ever re-arms this socket.
+        //
+        // Stopped short of draining: come straight back rather than waiting for the
+        // next datagram to re-arm an edge-triggered poller.
+        if (mux_->on_readable()) wakeup_.signal();
+        return;
+    }
+
+    auto it = fd_to_conn_.find(fd);
+    if (it == fd_to_conn_.end()) return;
+    dispatch_events(it->second, ev.events);
+}
+
+void Reactor::dispatch_events(ConnId id, uint32_t events) {
+    Connection* conn = find(id);
+    if (!conn) return;
+    if (conn->state() == ConnState::Closing || conn->state() == ConnState::Closed) return;
+    // Condemned earlier this turn, just not torn down yet — process_pending_close()
+    // runs at the end of the turn, so events polled alongside the one that closed it
+    // would otherwise still be delivered, and the connection could change state
+    // *after* the decision to close it was taken against the state it had.
+    //
+    // Which is not merely wasted work on bytes about to be dropped. cancel_dial()
+    // spares an attempt that has established; let one finish its handshake here,
+    // just after that check found it still handshaking, and it reaches the peer
+    // table as a duplicate and supersedes the sibling that beat it — while itself
+    // still standing condemned. Both are then gone at the end of the turn and the
+    // peer is lost, which is the very failure the cancel/close split exists to
+    // prevent. The map is empty on every turn that closes nothing, nearly all of them.
+    if (!pending_close_.empty() && pending_close_.count(id)) return;
+
+    bool keep = true;
+    if (events & (PollErr | PollHup)) {
+        keep = conn->on_error();
+    } else {
+        if (keep && (events & PollIn))  keep = conn->on_readable();
+        if (keep && (events & PollOut)) keep = conn->on_writable();
+    }
+
+    if (!keep) mark_for_close(id, conn->close_reason());
+}
+
+void Reactor::dispatch_link_events(ConnId id, uint32_t events) {
+    dispatch_events(id, events);
+}
+
+void Reactor::schedule_maintenance() {
+    timers_.schedule(kMaintenanceInterval, [this] {
+        // Connections being torn down this tick are still in conns_; the sweep is
+        // read-only with respect to the map, and on_maintenance_tick() never closes.
+        for (auto& [id, conn] : conns_) conn->on_maintenance_tick();
+        schedule_maintenance();
+    });
+}
+
+void Reactor::do_accept() {
+    // Level-triggered: drain all pending connections this tick. Use a raw,
+    // non-logging accept — accept_client() logs an error on every EWOULDBLOCK,
+    // which is the normal "no more pending" signal here.
+    while (true) {
+        socket_t client = ::accept(server_socket_, nullptr, nullptr);
+        if (!is_valid_socket(client)) break;  // EWOULDBLOCK / error → done this tick
+        suppress_sigpipe(client);  // not inherited from the listener; raw accept()
+
+        // Admission control: refuse new inbound peers when at capacity, before
+        // paying any handshake cost. Keep draining the backlog so the listener
+        // doesn't stay readable. The precise per-peer cap is enforced again at
+        // on_established (this coarse gate can't yet know the remote's identity).
+        if (!delegate_.admit_inbound()) {
+            close_socket(client);
+            continue;
+        }
+
+        Connection* conn = adopt(std::make_unique<TcpLink>(client, *this), ConnRole::Inbound,
+                                 next_conn_id_.fetch_add(1, std::memory_order_relaxed));
+        if (!conn) { close_socket(client); continue; }
+        conn->start_handshake();  // accepted sockets are already connected
+    }
+}
+
+ConnId Reactor::adopt_inbound_link(std::unique_ptr<Link> link) {
+    // Same coarse admission gate the TCP listener applies, for the same reason:
+    // turn a flood away before any handshake work is done for it.
+    if (!delegate_.admit_inbound()) return kInvalidConnId;
+
+    const ConnId id = next_conn_id_.fetch_add(1, std::memory_order_relaxed);
+    Connection* conn = adopt(std::move(link), ConnRole::Inbound, id);
+    if (!conn) return kInvalidConnId;
+    conn->start_handshake();  // an accepted stream is already connected
+    return id;
+}
+
+bool Reactor::resolve_dial(ConnId id) {
+    if (id > resolved_watermark_) resolved_watermark_ = id;
+    return cancelled_dials_.erase(id) > 0;
+}
+
+Connection* Reactor::adopt(std::unique_ptr<Link> link, ConnRole role, ConnId id) {
+    // A dial cancelled while its task sat in the queue never becomes a connection;
+    // dropping `link` here is what actually calls the attempt off.
+    if (resolve_dial(id)) return nullptr;
+
+    const socket_t fd = link->fd();
+    if (is_valid_socket(fd)) {
+        set_socket_nonblocking(fd);
+        // Only a TCP link has a socket of its own, so this reaches exactly the
+        // sockets Nagle applies to. Safe to do here and nowhere else because the
+        // send queue now aggregates on our side — see set_tcp_nodelay().
+        set_tcp_nodelay(fd);
+    }
+
+    auto conn = std::make_unique<Connection>(id, std::move(link), role, *this, delegate_);
+    Connection* raw = conn.get();
+    raw->link().attach(id);
+
+    conns_.emplace(id, std::move(conn));
+    conn_count_.fetch_add(1, std::memory_order_relaxed);
+
+    if (is_valid_socket(fd)) {
+        fd_to_conn_.emplace(fd, id);
+        // Inbound sockets are connected: watch for readable. Outbound sockets are
+        // still connecting: watch for writable, which signals connect completion.
+        poller_->add(fd, role == ConnRole::Inbound ? PollIn : PollOut);
+    }
+
+    // Reap connections that never reach Established (stuck connect or handshake).
+    TimerId timer = timers_.schedule(kEstablishTimeout, [this, id] {
+        auto it = conns_.find(id);
+        if (it == conns_.end()) return;
+        const ConnState st = it->second->state();
+        if (st != ConnState::Established && st != ConnState::Closing && st != ConnState::Closed) {
+            mark_for_close(id, st == ConnState::Connecting ? CloseReason::ConnectFailed
+                                                           : CloseReason::HandshakeFailed);
+        }
+    });
+    raw->set_establish_timer(timer);
+    return raw;
+}
+
+// ── Teardown ────────────────────────────────────────────────────────────────
+
+void Reactor::mark_for_close(ConnId id, CloseReason reason) {
+    pending_close_.emplace(id, reason);  // first reason wins
+}
+
+void Reactor::flush_dirty() {
+    if (dirty_.empty()) return;
+    // Swapped out first: a flush can close a connection, and a close can queue
+    // further work — none of which should extend the batch being walked.
+    std::vector<ConnId> batch;
+    batch.swap(dirty_);
+    for (const ConnId id : batch) {
+        Connection* conn = find(id);
+        if (!conn) continue;                                   // gone since it booked
+        if (conn->state() != ConnState::Established) continue;  // closing; nothing to owe
+        if (!conn->flush_pending()) mark_for_close(id, conn->close_reason());
+    }
+}
+
+void Reactor::process_pending_close() {
+    if (pending_close_.empty()) return;
+    auto batch = std::move(pending_close_);
+    pending_close_.clear();
+    for (const auto& [id, reason] : batch) remove(id, reason);
+}
+
+void Reactor::remove(ConnId id, CloseReason reason) {
+    auto it = conns_.find(id);
+    if (it == conns_.end()) return;
+
+    std::unique_ptr<Connection> conn = std::move(it->second);
+    conns_.erase(it);
+
+    const socket_t fd = conn->socket();
+    if (is_valid_socket(fd)) {
+        fd_to_conn_.erase(fd);
+        poller_->remove(fd);
+    }
+    conn->cancel_establish_timer();  // stop the reaper before this fd can be reused
+    conn_count_.fetch_sub(1, std::memory_order_relaxed);
+
+    LOG_DEBUG("reactor", "Connection " << id << " closed (" << to_string(reason) << ")");
+    conn->shutdown_link(reason);     // let the transport say goodbye / flush
+    delegate_.on_closed(*conn, reason);
+    if (is_valid_socket(fd)) close_socket(fd);
+    // ~Connection here releases the Link, which is what hands a UDP stream back to
+    // the mux (to linger briefly or go straight away).
+}
+
+void Reactor::shutdown_connections() {
+    for (auto& [id, conn] : conns_) {
+        const socket_t fd = conn->socket();
+        if (is_valid_socket(fd)) poller_->remove(fd);
+        conn->shutdown_link(CloseReason::ReactorShutdown);
+        delegate_.on_closed(*conn, CloseReason::ReactorShutdown);
+        if (is_valid_socket(fd)) close_socket(fd);
+    }
+    conns_.clear();
+    fd_to_conn_.clear();
+    pending_close_.clear();
+    cancelled_dials_.clear();
+    conn_count_.store(0, std::memory_order_relaxed);
+}
+
+} // namespace librats