librats 1.0.2 → 2.2.0

package/native-src/src/{io_poller.cpp → librats/core/io_poller.cpp}

@@ -1,8 +1,9 @@
-#include "io_poller.h"
-#include "logger.h"
+#include "librats/core/io_poller.h"
+#include "librats/util/logger.h"
 
 #include <cstring>
 #include <algorithm>
+#include <mutex>
 #include <set>
 
 //=============================================================================
@@ -37,8 +38,12 @@
 #elif defined(POLLER_USE_IOCP)
     #include <winsock2.h>
     #include <ws2tcpip.h>
+    #include "librats/core/notifier.h"   // wakes the readiness thread out of WSAPoll
+    #include <condition_variable>
     #include <mutex>
+    #include <thread>
     #include <unordered_map>
+    #include <unordered_set>
     #include <vector>
     #include <memory>
 #elif defined(POLLER_USE_POLL)
@@ -193,15 +198,21 @@ public:
     }
     
     bool add(socket_t fd, uint32_t events) override {
+        std::lock_guard<std::mutex> lock(mutex_);
+        // Already registered — answer as epoll does (EEXIST → false) rather than
+        // quietly re-arming under the caller. A second add is a caller bug, and
+        // the backends have to agree on what it does; see IocpPoller::add().
+        if (registered_.find(fd) != registered_.end()) return false;
         if (!apply_changes(fd, events, EV_ADD | EV_CLEAR))
             return false;
         registered_.insert(fd);
         return true;
     }
-    
+
     bool modify(socket_t fd, uint32_t events) override {
+        std::lock_guard<std::mutex> lock(mutex_);
         if (registered_.find(fd) == registered_.end()) return false;
-        
+
         // kqueue: adding a filter that already exists replaces it.
         // We also need to delete filters that are no longer wanted.
         struct kevent changes[4];
@@ -230,8 +241,9 @@ public:
     }
     
     bool remove(socket_t fd) override {
+        std::lock_guard<std::mutex> lock(mutex_);
         if (registered_.erase(fd) == 0) return false;
-        
+
         struct kevent changes[2];
         // Delete both read and write filters. Ignore errors (filter may not exist).
         EV_SET(&changes[0], fd, EVFILT_READ, EV_DELETE, 0, 0, nullptr);
@@ -308,8 +320,13 @@ public:
     
 private:
     int kqfd_ = -1;
+    /// Guards registered_ only. wait() never touches it, so the event loop is
+    /// never blocked by a control call — but add/modify/remove are documented as
+    /// callable from any thread (io_poller.h), and the concurrency tests take that
+    /// literally, so the set itself has to be safe against concurrent mutation.
+    std::mutex mutex_;
     std::set<socket_t> registered_;  ///< Track registered fds
-    
+
     bool apply_changes(socket_t fd, uint32_t events, uint16_t kq_flags) {
         struct kevent changes[2];
         int nchanges = 0;
@@ -343,10 +360,27 @@ private:
 //   readiness notifications through the IOCP completion port (proactor
 //   model adapted to reactor semantics).
 //
-//   Listen sockets and connecting sockets (where overlapped recv/send
-//   cannot be used) fall back to a non-blocking WSAPoll check done
-//   inside wait() — typically only 1 listen + ~30 connecting sockets,
-//   so this is negligible overhead.
+//   Listen sockets and connecting sockets cannot carry a zero-byte
+//   overlapped operation (WSARecv on either fails with WSAENOTCONN), so
+//   IOCP never has anything to tell us about them. They are checked with
+//   a non-blocking WSAPoll at the top of wait() instead.
+//
+//   That check ALONE is not enough, and getting this wrong is expensive:
+//   wait() then blocks in GQCS for the caller's full timeout, so readiness
+//   on one of those sockets is not noticed until wait() happens to return
+//   for some other reason — up to Reactor::kMaxPollMs (50 ms) late. Every
+//   outbound connect and every inbound accept paid that delay. So a small
+//   readiness thread (aux_loop) blocks in WSAPoll on exactly those sockets
+//   and posts a completion packet the moment one is ready, which is what
+//   GQCS is waiting for. The thread is only a WAKE source: the events
+//   themselves still come from check_wsapoll_sockets() under the mutex,
+//   against the live socket set — so a stale entry in the thread's
+//   snapshot can cost a spurious wakeup and nothing more.
+//
+//   wsapoll_mode_ indexes those sockets separately from active_. The check
+//   runs on every wait() and used to walk every registered socket to find
+//   the one or two that are not IOCP-driven; with the index it costs
+//   nothing at all when there are none (the common steady state).
 //
 //   When send_to_peer() on another thread arms a write via modify(),
 //   the zero-byte WSASend completes and wakes GQCS immediately —
@@ -365,14 +399,40 @@ public:
     IocpPoller() {
         iocp_ = CreateIoCompletionPort(INVALID_HANDLE_VALUE, nullptr, 0, 1);
         if (!iocp_) {
-            LOG_POLLER_ERROR("CreateIoCompletionPort failed: " + 
+            LOG_POLLER_ERROR("CreateIoCompletionPort failed: " +
                             std::to_string(GetLastError()));
-        } else {
-            LOG_POLLER_INFO("Created IOCP poller");
+            return;
+        }
+        LOG_POLLER_INFO("Created IOCP poller");
+
+        // The readiness thread for listen/connecting sockets. It parks on the
+        // condition variable whenever there is nothing in WSAPoll mode, which on a
+        // node whose connections are all established is most of the time.
+        //
+        // Its wakeup socket is built here rather than as a plain member because a
+        // poller is constructed with the Reactor, which happens before anything
+        // calls init_socket_library() — and a Notifier built before WSAStartup is a
+        // Notifier that silently does nothing. Idempotent, so asking again is free.
+        init_socket_library();
+        aux_wake_ = std::make_unique<Notifier>();
+        if (!is_valid_socket(aux_wake_->fd())) {
+            LOG_POLLER_WARN("No wakeup socket for the readiness thread; "
+                            "listen/connect readiness falls back to a short poll timeout");
         }
+        aux_thread_ = std::thread(&IocpPoller::aux_loop, this);
     }
-    
+
     ~IocpPoller() override {
+        // Stop the readiness thread FIRST: it touches active_ and posts to iocp_,
+        // so neither may be torn down while it is still running.
+        {
+            std::lock_guard<std::mutex> lock(mutex_);
+            aux_stop_ = true;
+        }
+        aux_cv_.notify_all();
+        if (aux_wake_) aux_wake_->signal();
+        if (aux_thread_.joinable()) aux_thread_.join();
+
         {
             std::lock_guard<std::mutex> lock(mutex_);
             // Cancel all pending I/O before destroying
@@ -382,6 +442,8 @@ public:
                 }
             }
             active_.clear();
+            wsapoll_mode_.clear();
+            demoted_.clear();
             all_states_.clear();
         }
         if (iocp_) {
@@ -392,16 +454,31 @@ public:
     
     bool add(socket_t fd, uint32_t events) override {
         std::lock_guard<std::mutex> lock(mutex_);
-        
+
+        // Already registered. Overwriting active_[fd] would orphan the old state:
+        // it stays in all_states_ with removed == false, so gc_removed_states()
+        // never frees it, and its outstanding zero-byte WSARecv keeps completing
+        // — reporting events for an fd this poller has since been told to forget,
+        // by which time Windows may have handed the descriptor number to a
+        // different socket. epoll answers EEXIST here; answer the same and let the
+        // caller reach for modify().
+        if (active_.find(fd) != active_.end()) {
+            LOG_POLLER_DEBUG("add: fd " + std::to_string(fd) +
+                             " is already registered; use modify()");
+            return false;
+        }
+
         auto state_ptr = std::make_unique<IocpSocketState>(fd);
         auto* state = state_ptr.get();
         state->desired_events = events;
         
         // Associate socket with IOCP (needed for overlapped I/O)
         CreateIoCompletionPort(reinterpret_cast<HANDLE>(fd), iocp_, 0, 0);
-        
+
+        state->is_datagram = is_datagram_socket(fd);
+
         // Determine socket mode:
-        // - Try zero-byte WSARecv to detect if socket is connected (IOCP mode)
+        // - Try a readiness probe to detect if socket is connected (IOCP mode)
         // - Listen sockets and connecting sockets use WSAPoll fallback
         if (events & PollIn) {
             if (arm_read(state)) {
@@ -419,10 +496,24 @@ public:
         if (state->mode == SocketMode::Iocp && (events & PollOut)) {
             arm_write(state);
         }
-        
+
         active_[fd] = state;
         all_states_.push_back(std::move(state_ptr));
-        
+
+        // Which of the two mechanisms a socket ended up on decides how it behaves
+        // under load and on error, and it is not visible from anywhere else — so say
+        // it once, here, rather than leave the next person to infer it.
+        LOG_POLLER_DEBUG("add: fd " + std::to_string(fd) + " → " +
+                         (state->mode == SocketMode::Iocp ? "IOCP" : "WSAPoll") +
+                         " mode" + (state->is_datagram ? " (datagram)" : ""));
+
+        // A socket IOCP cannot watch has to be picked up by the readiness thread,
+        // which is very likely blocked in WSAPoll on a set that predates this one.
+        if (state->mode == SocketMode::WsaPoll) {
+            wsapoll_mode_.insert(fd);
+            wake_aux();
+        }
+
         return true;
     }
     
@@ -441,9 +532,17 @@ public:
         if (state->mode == SocketMode::WsaPoll && (events & PollIn)) {
             if (arm_read(state)) {
                 state->mode = SocketMode::Iocp;
+                wsapoll_mode_.erase(fd);
+                demoted_.erase(fd);
+                wake_aux();   // one fewer socket to watch; re-snapshot
             }
         }
-        
+        // A socket that stays in WSAPoll mode may have changed which events it
+        // wants (a connecting socket disarming PollOut), and the readiness thread
+        // is holding the previous mask.
+        if (state->mode == SocketMode::WsaPoll) wake_aux();
+
+
         // Arm overlapped ops as needed for IOCP-mode sockets
         if (state->mode == SocketMode::Iocp) {
             if ((events & PollIn) && !state->read_pending) {
@@ -474,6 +573,12 @@ public:
         }
         
         active_.erase(it);
+        // Before the caller closes this socket. The readiness thread may still be
+        // polling it from an older snapshot — harmless (a closed handle reports
+        // POLLNVAL, and only the live set produces events), but waking it now
+        // means it stops looking at a descriptor that is about to be reused.
+        if (wsapoll_mode_.erase(fd) > 0) wake_aux();
+        demoted_.erase(fd);
         ++removed_count_;
         // Note: SocketState is NOT freed yet — it stays in all_states_ so
         // that the OVERLAPPED pointers remain valid until cancelled
@@ -487,13 +592,28 @@ public:
         
         //------------------------------------------------------------------
         // Step 1: Check WSAPoll-mode sockets (listen + connecting)
-        //         Non-blocking check (timeout=0), typically <= 31 sockets
+        //         Non-blocking check (timeout=0), and free when there are none
         //------------------------------------------------------------------
         {
             std::lock_guard<std::mutex> lock(mutex_);
+            // Re-arm the readiness thread here rather than at the end of the last
+            // wait(): the caller coming back for more is what says it has finished
+            // handling the previous batch. Re-arming any earlier would have the
+            // thread reporting sockets the caller has not dealt with yet.
+            if (wake_pending_) {
+                wake_pending_ = false;
+                aux_cv_.notify_one();
+            }
+            // Order matters: retry the older demotions first, so a socket demoted by
+            // arm_pending() below waits for the next wait() instead of being asked to
+            // arm twice in a row microseconds apart.
+            promote_demoted();
+            arm_pending();
             count = check_wsapoll_sockets(results, max_results);
         }
-        
+
+        const int wsapoll_count = count;
+
         //------------------------------------------------------------------
         // Step 2: Wait for IOCP completions (connected data sockets)
         //         If Step 1 already found events, don't block (timeout=0)
@@ -525,10 +645,16 @@ public:
         //------------------------------------------------------------------
         if (iocp_count > 0) {
             std::lock_guard<std::mutex> lock(mutex_);
-            
+            bool woken = false;
+
             for (ULONG i = 0; i < iocp_count && count < max_results; ++i) {
-                if (!entries[i].lpOverlapped) continue;
-                
+                // The readiness thread's wake packet: no overlapped, and the only
+                // completion key this poller ever posts by hand.
+                if (!entries[i].lpOverlapped) {
+                    if (entries[i].lpCompletionKey == kAuxWakeKey) woken = true;
+                    continue;
+                }
+
                 auto* io = reinterpret_cast<IocpOverlapped*>(entries[i].lpOverlapped);
                 auto* state = io->state;
                 if (!state) continue;
@@ -543,28 +669,42 @@ public:
                 // Check the NTSTATUS from the overlapped result
                 // 0 = STATUS_SUCCESS, non-zero = error (e.g. connection reset)
                 ULONG_PTR internal_status = io->overlapped.Internal;
-                
+
+                // A datagram readiness probe peeks at one byte of what is always a
+                // larger datagram, so it reports STATUS_BUFFER_OVERFLOW rather than
+                // success. That IS the "data is waiting" signal — the datagram is
+                // still queued (MSG_PEEK took nothing), ready for the real read.
+                constexpr ULONG_PTR kStatusBufferOverflow = 0x80000005ul;
+                if (state->is_datagram && internal_status == kStatusBufferOverflow)
+                    internal_status = 0;
+
                 uint32_t reported_events = 0;
                 if (internal_status == 0) {
                     // Success — report readiness for the event type we were watching
                     reported_events = io->event_type & state->desired_events;
-                    
-                    // Re-arm overlapped I/O for next notification.
-                    // Without this, the socket becomes deaf after the first
-                    // completion because sync_poller() only calls modify()
-                    // when desired_events change — which they don't for a
-                    // socket that stays PollIn-only.
-                    if ((io->event_type & PollIn) && (state->desired_events & PollIn)) {
-                        arm_read(state);
-                    }
-                    if ((io->event_type & PollOut) && (state->desired_events & PollOut)) {
-                        arm_write(state);
-                    }
                 } else {
                     // I/O error (connection reset, cancelled, etc.)
                     reported_events = PollErr;
                 }
-                
+
+                // This socket needs a fresh overlapped operation, or it goes deaf:
+                // nothing else arms it, since modify() is called only when the
+                // watched events change and for a socket that stays PollIn-only they
+                // never do. The work is queued rather than done here — see
+                // arm_pending(), which is also where a socket that cannot be armed
+                // gets handed to the readiness thread.
+                //
+                // Queued after an ERROR for a datagram socket but not for a stream
+                // one, and the asymmetry is the whole point. A stream error is
+                // terminal: the owner closes the connection on PollErr, so arming
+                // again would only race that close. A datagram error is not — it
+                // belongs to ONE datagram (Windows reports an ICMP complaint about an
+                // earlier send on the next receive) while the socket stays perfectly
+                // usable, and skipping it there costs every datagram that reactor
+                // would have received from then on.
+                if (internal_status == 0 || state->is_datagram)
+                    pending_arm_.push_back(state->fd);
+
                 if (reported_events == 0) continue;
                 
                 // Merge events for the same fd (read + write may fire together)
@@ -582,7 +722,20 @@ public:
                     ++count;
                 }
             }
-            
+
+            // The wake packet says a listen/connecting socket became ready while
+            // we were blocked, so look again now instead of making the caller come
+            // back for it. Only when step 1 found nothing there: the two modes are
+            // disjoint sets of sockets, so with wsapoll_count == 0 nothing this can
+            // add is already in `results`.
+            if (woken && wsapoll_count == 0 && count < max_results) {
+                if (wake_pending_) {
+                    wake_pending_ = false;
+                    aux_cv_.notify_one();
+                }
+                count += check_wsapoll_sockets(results + count, max_results - count);
+            }
+
             // GC removed states whose cancelled I/O has fully drained.
             // Threshold: at least 64 removed, or removed > half of total —
             // avoids running the sweep on every wait() call.
@@ -627,12 +780,15 @@ private:
         bool read_pending;              ///< Zero-byte WSARecv in flight
         bool write_pending;             ///< Zero-byte WSASend in flight
         bool removed;                   ///< Marked for removal
+        bool is_datagram;               ///< SOCK_DGRAM — readiness probed with MSG_PEEK
+        char peek_byte;                 ///< one-byte landing pad for that probe
         IocpOverlapped read_io;         ///< Overlapped for read notification
         IocpOverlapped write_io;        ///< Overlapped for write notification
-        
+
         explicit IocpSocketState(socket_t f)
             : fd(f), desired_events(0), mode(SocketMode::WsaPoll),
-              read_pending(false), write_pending(false), removed(false) {
+              read_pending(false), write_pending(false), removed(false),
+              is_datagram(false), peek_byte(0) {
             read_io.state = this;
             read_io.event_type = PollIn;
             write_io.state = this;
@@ -643,29 +799,188 @@ private:
     std::mutex mutex_;
     std::unordered_map<socket_t, IocpSocketState*> active_;     ///< fd → state lookup
     std::vector<std::unique_ptr<IocpSocketState>> all_states_;  ///< Owns all states
+    /// The subset of active_ that IOCP cannot watch (listen + connecting sockets).
+    /// Indexed separately so check_wsapoll_sockets() costs nothing when it is empty
+    /// instead of walking every registered socket to discover that.
+    std::unordered_set<socket_t> wsapoll_mode_;
+    /// The subset of wsapoll_mode_ that belongs on IOCP and is only there because
+    /// arming failed — see promote_demoted(). A listen socket is never in here.
+    std::unordered_set<socket_t> demoted_;
+    /// Sockets whose completion has been handed to the caller and which therefore
+    /// need a fresh overlapped operation — posted at the top of the next wait(),
+    /// once the caller has drained them. See arm_pending(). Holds fds rather than
+    /// state pointers so that a socket removed in the meantime is simply not found.
+    std::vector<socket_t> pending_arm_;
     std::vector<WSAPOLLFD> wsapoll_fds_;                    ///< Reusable WSAPoll buffer
     size_t removed_count_ = 0;                              ///< Number of removed states awaiting GC
-    
+
+    //----------------------------------------------------------------------
+    // Readiness thread for WSAPoll-mode sockets
+    //----------------------------------------------------------------------
+
+    /// Completion key of the packet the readiness thread posts. Sockets are
+    /// associated with key 0, so this cannot collide with a real completion.
+    static constexpr ULONG_PTR kAuxWakeKey = 1;
+
+    /// Backstop timeout for the thread's WSAPoll. Every change to the watched set
+    /// signals aux_wake_, so this only has to cover a wakeup that went missing;
+    /// without a wakeup socket at all it becomes the actual notification latency,
+    /// hence the much shorter degraded value.
+    static constexpr int kAuxPollMs         = 250;
+    static constexpr int kAuxPollDegradedMs = 5;
+
+    std::thread               aux_thread_;
+    std::condition_variable   aux_cv_;
+    std::unique_ptr<Notifier> aux_wake_;            ///< breaks the thread out of WSAPoll
+    bool                    aux_stop_     = false;  ///< guarded by mutex_
+    /// A wake packet is in the completion queue and wait() has not looked yet.
+    /// While it is set the thread stays parked, so one ready socket can only ever
+    /// produce one packet however long the caller takes to come back. Guarded by
+    /// mutex_.
+    bool                    wake_pending_ = false;
+
+    /// Tell the readiness thread its view of the watched set is out of date.
+    /// Called under mutex_; wakes it whether it is parked on the condition
+    /// variable or blocked inside WSAPoll.
+    void wake_aux() {
+        aux_cv_.notify_one();
+        if (aux_wake_) aux_wake_->signal();
+    }
+
+    /// The readiness thread's wakeup socket, or INVALID if it has none.
+    socket_t aux_wake_fd() const noexcept {
+        return aux_wake_ ? aux_wake_->fd() : RATS_INVALID_SOCKET;
+    }
+
+    /// Block in WSAPoll on the sockets IOCP cannot watch, and post a completion
+    /// packet the moment one is ready — nothing more. wait() still derives the
+    /// events themselves from the live set under the mutex, so everything this
+    /// thread knows is allowed to be slightly stale.
+    void aux_loop() {
+        const socket_t wake_fd     = aux_wake_fd();
+        const int      poll_timeout = is_valid_socket(wake_fd) ? kAuxPollMs
+                                                               : kAuxPollDegradedMs;
+        std::vector<WSAPOLLFD> fds;
+
+        for (;;) {
+            {
+                std::unique_lock<std::mutex> lock(mutex_);
+                // Park while there is nothing to watch, or while a wake packet is
+                // still outstanding — polling then would only rediscover a socket
+                // the caller has already been told about.
+                aux_cv_.wait(lock, [this] {
+                    return aux_stop_ || (!wake_pending_ && !wsapoll_mode_.empty());
+                });
+                if (aux_stop_) return;
+
+                fds.clear();
+                fds.reserve(wsapoll_mode_.size() + 1);
+                for (const socket_t fd : wsapoll_mode_) {
+                    const auto it = active_.find(fd);
+                    if (it == active_.end() || it->second->removed) continue;
+
+                    WSAPOLLFD pfd{};
+                    pfd.fd = fd;
+                    if (it->second->desired_events & PollIn)  pfd.events |= POLLIN;
+                    if (it->second->desired_events & PollOut) pfd.events |= POLLOUT;
+                    if (pfd.events == 0) continue;
+                    fds.push_back(pfd);
+                }
+            }
+
+            // Always last, so the set is never empty and WSAPoll always blocks.
+            if (is_valid_socket(wake_fd)) {
+                WSAPOLLFD pfd{};
+                pfd.fd     = wake_fd;
+                pfd.events = POLLIN;
+                fds.push_back(pfd);
+            }
+            if (fds.empty()) continue;   // no wakeup socket and nothing left to watch
+
+            const int n = WSAPoll(fds.data(), static_cast<ULONG>(fds.size()), poll_timeout);
+            if (n <= 0) continue;        // timed out, or the set went stale under us
+
+            // The wakeup itself only means "the set changed, look again", so drain
+            // it and drop it before asking whether a watched socket fired.
+            if (is_valid_socket(wake_fd)) {
+                if (fds.back().revents != 0) aux_wake_->drain();
+                fds.pop_back();
+            }
+
+            bool ready = false;
+            for (const WSAPOLLFD& pfd : fds) {
+                if (pfd.revents != 0) { ready = true; break; }
+            }
+            if (!ready) continue;   // only the wakeup fired: re-snapshot and poll again
+
+            {
+                std::lock_guard<std::mutex> lock(mutex_);
+                if (aux_stop_) return;
+                if (wake_pending_) continue;   // one packet is already owed
+                wake_pending_ = true;
+            }
+            if (!PostQueuedCompletionStatus(iocp_, 0, kAuxWakeKey, nullptr)) {
+                // Nothing will consume the packet we just promised, so take the
+                // promise back — otherwise this thread parks for good.
+                LOG_POLLER_ERROR("PostQueuedCompletionStatus failed: " +
+                                 std::to_string(GetLastError()));
+                std::lock_guard<std::mutex> lock(mutex_);
+                wake_pending_ = false;
+            }
+        }
+    }
+
+
     //----------------------------------------------------------------------
     // Zero-byte overlapped I/O: readiness notification via IOCP
     //----------------------------------------------------------------------
     
+    /// Is this a datagram socket?
+    ///
+    /// It matters because the zero-byte WSARecv this poller uses for readiness
+    /// notification is only safe on a stream socket. On a datagram socket the
+    /// operation completes with WSAEMSGSIZE — and Windows DISCARDS the datagram
+    /// that satisfied it, because it did not fit in the (empty) buffer, losing one
+    /// datagram per notification. arm_read() therefore probes a datagram socket
+    /// with a one-byte MSG_PEEK instead, which reports the same readiness without
+    /// taking anything off the queue.
+    static bool is_datagram_socket(socket_t fd) {
+        int type = 0;
+        int len  = static_cast<int>(sizeof(type));
+        if (getsockopt(fd, SOL_SOCKET, SO_TYPE, reinterpret_cast<char*>(&type), &len) != 0)
+            return false;
+        return type == SOCK_DGRAM;
+    }
+
     /// Post a zero-byte WSARecv. Completes when data is available to read.
     /// Returns false if the socket is not connected (listen socket).
     bool arm_read(IocpSocketState* state) {
         if (state->read_pending) return true;
         
         std::memset(&state->read_io.overlapped, 0, sizeof(OVERLAPPED));
-        
+
         WSABUF buf;
-        buf.buf = nullptr;
-        buf.len = 0;
-        DWORD flags = 0;
+        DWORD  flags = 0;
+        if (state->is_datagram) {
+            // Peek at one byte rather than receiving zero: MSG_PEEK leaves the
+            // datagram on the queue for the real recvfrom() that follows, where a
+            // zero-byte receive would consume and discard it (see
+            // is_datagram_socket). The completion carries STATUS_BUFFER_OVERFLOW
+            // because one byte is not the whole datagram — which is precisely the
+            // "something is there" signal we are after, and is treated as success
+            // when the completion is dequeued.
+            buf.buf = &state->peek_byte;
+            buf.len = 1;
+            flags   = MSG_PEEK;
+        } else {
+            buf.buf = nullptr;
+            buf.len = 0;
+        }
         DWORD bytes = 0;
-        
+
         int ret = WSARecv(state->fd, &buf, 1, &bytes, &flags,
                           &state->read_io.overlapped, nullptr);
-        
+
         if (ret == 0) {
             // Completed immediately — completion still posted to IOCP
             state->read_pending = true;
@@ -687,7 +1002,11 @@ private:
     /// Also wakes GQCS when called from another thread (via modify/send_to_peer).
     bool arm_write(IocpSocketState* state) {
         if (state->write_pending) return true;
-        
+        // A zero-byte WSASend on a datagram socket would put an empty datagram on
+        // the wire rather than probe writability. Nothing asks for PollOut on one
+        // (a datagram socket is writable essentially always), so simply decline.
+        if (state->is_datagram) return false;
+
         std::memset(&state->write_io.overlapped, 0, sizeof(OVERLAPPED));
         
         WSABUF buf;
@@ -737,46 +1056,181 @@ private:
     // WSAPoll fallback for listen + connecting sockets
     //----------------------------------------------------------------------
     
+    /// Has this socket's connect attempt already failed?
+    ///
+    /// MSDN lists "WSAPoll does not report a failed connection attempt" as a known
+    /// limitation: neither POLLERR nor POLLWRNORM raised, so a refused or
+    /// unreachable dial produces no event at all and sits until the reactor's
+    /// establish deadline reaps it fifteen seconds later. Measured on Windows 11
+    /// that no longer reproduces — a refused loopback dial comes back as
+    /// POLLERR|POLLHUP|POLLWRNORM, and a single WSAPoll spanning the failure sees
+    /// it too — so on a current build this is belt and braces. It stays because
+    /// the limitation is still documented, the affected builds are still out
+    /// there, and the fallback is one getsockopt: the socket carries the error
+    /// either way, so ask it directly rather than depend on which behaviour we got.
+    ///
+    /// Only ever consulted for a socket WSAPoll reported nothing about, so it costs
+    /// one getsockopt per *connecting* socket per wait() — there are only ever a
+    /// handful. A connect still in flight reads back 0, so there are no false
+    /// positives. (SO_ERROR is read-and-clear, but the caller that acts on the
+    /// PollErr never reads it again — Connection::on_error does not.)
+    static bool connect_failed(socket_t fd) {
+        int err = 0;
+        int len = static_cast<int>(sizeof(err));
+        if (getsockopt(fd, SOL_SOCKET, SO_ERROR, reinterpret_cast<char*>(&err), &len) != 0)
+            return false;
+        return err != 0;
+    }
+
+    /// Post the overlapped operations the last batch of completions made necessary.
+    /// Called under mutex, at the top of wait().
+    ///
+    /// Deliberately NOT done where the completion is processed. There the data that
+    /// triggered it is still sitting unread in the socket buffer — the caller only
+    /// reads after wait() has returned — so a fresh zero-byte WSARecv completes on
+    /// the spot and queues a second completion with nothing behind it. That is one
+    /// wasted wakeup and one empty read for every burst of data: measured at exactly
+    /// 2.00 readiness reports per message where 1.00 is achievable, half of them
+    /// finding an empty socket.
+    ///
+    /// Waiting until the caller comes back for more is what lets the arm find an
+    /// empty buffer and actually pend, which is the entire point of arming. It is
+    /// the same signal wake_pending_ already relies on: a caller asking for the next
+    /// batch has finished with the one it was given. Nothing is lost in between — if
+    /// data lands while no operation is posted, the arm below completes immediately
+    /// and step 2 of this very wait() dequeues it.
+    void arm_pending() {
+        if (pending_arm_.empty()) return;
+
+        for (const socket_t fd : pending_arm_) {
+            const auto it = active_.find(fd);
+            if (it == active_.end()) continue;      // removed while we were away
+            auto* state = it->second;
+            if (state->removed || state->mode != SocketMode::Iocp) continue;
+
+            if ((state->desired_events & PollIn)  && !state->read_pending)
+                arm_read(state);
+            if ((state->desired_events & PollOut) && !state->write_pending)
+                arm_write(state);
+
+            // Arming can fail outright: WSARecv hands back a queued error instead of
+            // pending (exactly what a datagram socket does while it still has ICMP
+            // reports to deliver), or the system is out of buffers. Nothing is then
+            // left in flight to bring us back to this socket and nobody arms it
+            // again, so hand it to the readiness thread — which polls instead of
+            // arming and therefore cannot be left holding nothing. Slower per event,
+            // and the only alternative is deaf.
+            if ((state->desired_events & PollIn) && !state->read_pending) {
+                state->mode = SocketMode::WsaPoll;
+                wsapoll_mode_.insert(fd);
+                demoted_.insert(fd);            // and try to get it back later
+                wake_aux();
+                LOG_POLLER_DEBUG("fd " + std::to_string(fd) +
+                                 " could not be armed; demoted to WSAPoll mode");
+            }
+        }
+        pending_arm_.clear();
+    }
+
+    /// Put demoted sockets back on the completion port. Called under mutex.
+    ///
+    /// A socket reaches WSAPoll mode two ways, and only one of them is permanent.
+    /// A listen socket can never carry overlapped I/O and belongs there for good;
+    /// a socket that was demoted because arming failed is there only until the
+    /// condition that failed it clears — a datagram socket with an ICMP report
+    /// still queued arms perfectly well once that queue drains. Nothing else would
+    /// ever move it back: modify() is the only other promotion path and the reactor
+    /// never calls it on the mux socket, so one transient ICMP would otherwise pin
+    /// the busiest socket in the process to the slow path for the rest of its life
+    /// (measured at ~1.3x per datagram, and ~2.7x per readiness notification).
+    ///
+    /// Costs one WSARecv attempt per demoted socket per wait(), and a socket leaves
+    /// the set the moment one succeeds. Sockets that were never on IOCP are not in
+    /// the set at all, so the steady-state cost is one empty-container check.
+    void promote_demoted() {
+        if (demoted_.empty()) return;
+
+        for (auto it = demoted_.begin(); it != demoted_.end(); ) {
+            const socket_t fd  = *it;
+            const auto     ait = active_.find(fd);
+            if (ait == active_.end() || ait->second->removed) {
+                it = demoted_.erase(it);
+                continue;
+            }
+
+            auto* state = ait->second;
+            if (!(state->desired_events & PollIn) || !arm_read(state)) {
+                ++it;             // still cannot be armed; try again next time
+                continue;
+            }
+
+            state->mode = SocketMode::Iocp;
+            wsapoll_mode_.erase(fd);
+            it = demoted_.erase(it);
+            wake_aux();           // one fewer socket for the readiness thread
+            LOG_POLLER_DEBUG("fd " + std::to_string(fd) +
+                             " armed again; back on IOCP mode");
+        }
+    }
+
     /// Non-blocking check of WSAPoll-mode sockets. Called under mutex.
+    ///
+    /// Walks wsapoll_mode_ rather than active_: these are the one listen socket and
+    /// however many dials are in flight, where active_ is every connection the
+    /// reactor owns. This runs on every wait(), so the difference is the whole cost
+    /// of the fallback at scale — and it is now zero when there is nothing in it.
     int check_wsapoll_sockets(PollResult* results, int max_results) {
+        if (wsapoll_mode_.empty() || max_results <= 0) return 0;
+
         wsapoll_fds_.clear();
-        
-        for (auto& [fd, state] : active_) {
+        wsapoll_fds_.reserve(wsapoll_mode_.size());
+
+        for (const socket_t fd : wsapoll_mode_) {
+            const auto it = active_.find(fd);
+            if (it == active_.end()) continue;
+            const auto* state = it->second;
             if (state->removed || state->mode != SocketMode::WsaPoll) continue;
-            
+
             WSAPOLLFD pfd;
             pfd.fd = fd;
             pfd.events = 0;
             if (state->desired_events & PollIn)  pfd.events |= POLLIN;
             if (state->desired_events & PollOut) pfd.events |= POLLOUT;
             pfd.revents = 0;
-            
+            if (pfd.events == 0) continue;
+
             wsapoll_fds_.push_back(pfd);
         }
-        
+
         if (wsapoll_fds_.empty()) return 0;
-        
+
         // Non-blocking poll (timeout = 0)
-        int n = WSAPoll(wsapoll_fds_.data(),
-                        static_cast<ULONG>(wsapoll_fds_.size()), 0);
-        if (n <= 0) return 0;
-        
+        const int n = WSAPoll(wsapoll_fds_.data(),
+                              static_cast<ULONG>(wsapoll_fds_.size()), 0);
+        if (n < 0) return 0;
+
         int count = 0;
         for (auto& pfd : wsapoll_fds_) {
             if (count >= max_results) break;
-            if (pfd.revents == 0) continue;
-            
+
             uint32_t events = 0;
             if (pfd.revents & POLLIN)  events |= PollIn;
             if (pfd.revents & POLLOUT) events |= PollOut;
             if (pfd.revents & POLLERR) events |= PollErr;
             if (pfd.revents & POLLHUP) events |= PollHup;
-            
+
+            // Silence from a socket waiting to become writable is the one case
+            // WSAPoll is documented not to speak for; see connect_failed().
+            if (events == 0) {
+                if (!(pfd.events & POLLOUT) || !connect_failed(pfd.fd)) continue;
+                events = PollErr;
+            }
+
             results[count].fd = pfd.fd;
             results[count].events = events;
             ++count;
         }
-        
+
         return count;
     }
 };
@@ -803,7 +1257,8 @@ public:
     
     bool add(socket_t fd, uint32_t events) override {
         std::lock_guard<std::mutex> lock(mutex_);
-        registered_[fd] = events;
+        // Reject a duplicate like every other backend does — see IocpPoller::add().
+        if (!registered_.emplace(fd, events).second) return false;
         dirty_ = true;
         return true;
     }