@encharm/cws 4.8.3 → 4.10.0

package/src/Socket.h

@@ -2,6 +2,9 @@
 #define SOCKET_CWS_H
 
 #include "Networking.h"
+#include "SendWorker.h"
+#include "Zlib.h"
+#include <vector>
 #ifndef _WIN32
 #include <sys/uio.h>
 #endif
@@ -42,6 +45,7 @@ protected:
     bool corkable = false;
     bool corkPending = false;
 
+public:
     // this is not needed by HttpSocket!
     struct Queue {
         struct Message {
@@ -54,12 +58,21 @@ protected:
             // -1: heap (new char[]). Messages are allocated as raw char arrays, so every
             // allocation site must set this explicitly (the initializers above never run).
             int poolIndex = -1;
+            // data lives in its own new[] buffer (not inside this block); freed with delete[].
+            bool ownsData = false;
+            // data is a raw, unframed payload still to be deflated and framed (by the send
+            // worker, or by materializeCb if a main-thread write path reaches it first).
+            bool compressPending = false;
+            unsigned char opCode = 0;
         };
 
         Message *head = nullptr, *tail = nullptr;
         size_t totalLength = 0;
 
         static void release(NodeData *nodeData, Message *message) {
+            if (message->ownsData) {
+                delete [] (char *) message->data;
+            }
             if (message->poolIndex >= 0) {
                 nodeData->freeSmallMemoryBlock((char *) message, message->poolIndex);
             } else {
@@ -98,6 +111,55 @@ protected:
         }
     } messageQueue;
 
+    // One in-flight worker-thread send per socket (see SendWorker.h). The frames it
+    // covers are moved out of messageQueue into the op, so nothing the worker reads
+    // can be freed underneath it; the main thread takes them back on completion.
+    // `socket` is nulled if the socket closes first, and the fd is then closed by the
+    // completion (never while the worker may still be inside a send on it).
+    struct SendOp {
+        Socket *socket;
+        NodeData *nodeData;
+        Queue::Message *head, *tail;
+        size_t bytes;
+        int count;
+        uv_os_sock_t fd;
+        ssize_t result;
+        int error;
+        bool closeFd;
+        void *deflateWindow;   // this socket's sliding window (nullptr: worker's shared compressor)
+        void *destroyWindow;   // set when the socket closed mid-flight: completion destroys it
+#ifdef _WIN32
+        WSABUF bufs[512];
+#else
+        struct iovec iov[512];
+#endif
+    };
+    static void performSend(SendOp *op);   // worker thread
+    static void sendComplete(SendOp *op);  // main thread
+    static void materializeOnMain(Socket *s, Queue::Message *m, cWS::zlib::Stream *stream, bool resetAfter, char *buffer, size_t bufferSize, std::string &dynamic);
+protected:
+    SendOp *sendOp = nullptr;
+    // Set by setState<STATE>(): STATE::onEnd, so the completion path can end a socket on a hard error.
+    void (*endCb)(Socket *) = nullptr;
+    // Per-socket deflate window handed to worker ops (WebSocket sets it); once an op has
+    // been orphaned by close, the op owns the window and the socket must not destroy it.
+    void *workerDeflateWindow = nullptr;
+    bool workerOwnsWindow = false;
+    // Deflates + frames a compressPending message on the main thread (set by WebSocket).
+    void (*materializeCb)(Socket *, Queue::Message *) = nullptr;
+
+    void materializePending() {
+        for (Queue::Message *m = messageQueue.front(); m; m = m->nextMessage) {
+            if (m->compressPending && materializeCb) {
+                materializeCb(this, m);
+            }
+        }
+    }
+
+    bool sendBusy() {
+        return sendOp != nullptr;
+    }
+
     int getPoll() {
         return state.poll;
     }
@@ -258,10 +320,13 @@ protected:
         }
 
         if (events & UV_WRITABLE) {
-            if (!socket->messageQueue.empty() && (events & UV_WRITABLE)) {
+            if (!socket->messageQueue.empty() && (events & UV_WRITABLE) && !socket->sendBusy()) {
                 socket->cork(true);
                 while (true) {
                     Queue::Message *messagePtr = socket->messageQueue.front();
+                    if (messagePtr->compressPending && socket->materializeCb) {
+                        socket->materializeCb(socket, messagePtr);
+                    }
                     ssize_t sent = ::send(socket->getFd(), messagePtr->data, messagePtr->length, MSG_NOSIGNAL);
                     if (sent == (ssize_t) messagePtr->length) {
                         // Pop before the callback: it may close/terminate this socket,
@@ -310,6 +375,7 @@ protected:
 
     template<class STATE>
     void setState() {
+        endCb = STATE::onEnd;
         if (ssl) {
             setCb(sslIoHandler<STATE>);
         } else {
@@ -334,6 +400,9 @@ protected:
         messagePtr->callbackData = nullptr;
         messagePtr->reserved = nullptr;
         messagePtr->poolIndex = -1;
+        messagePtr->ownsData = false;
+        messagePtr->compressPending = false;
+        messagePtr->opCode = 0;
 
         if (data) {
             memcpy((char *) messagePtr->data, data, messagePtr->length);
@@ -348,6 +417,12 @@ protected:
 
     bool write(Queue::Message *message, bool &wasTransferred) {
         ssize_t sent = 0;
+        if (sendBusy()) {
+            // a worker send is in flight: anything written now must follow it
+            messageQueue.push(message);
+            wasTransferred = true;
+            return true;
+        }
         if (messageQueue.empty()) {
 
             if (ssl) {
@@ -408,6 +483,9 @@ protected:
                 messagePtr = (Queue::Message *) nodeData->getSmallMemoryBlock(memoryIndex);
                 messagePtr->data = ((char *) messagePtr) + sizeof(Queue::Message);
                 messagePtr->poolIndex = memoryIndex;
+                messagePtr->ownsData = false;
+                messagePtr->compressPending = false;
+                messagePtr->opCode = 0;
             } else {
                 messagePtr = allocMessage(estimatedLength - sizeof(Queue::Message));
             }
@@ -435,6 +513,9 @@ protected:
                 messagePtr->nextMessage = nullptr;
                 messagePtr->reserved = nullptr;
                 messagePtr->poolIndex = memoryIndex;
+                messagePtr->ownsData = false;
+                messagePtr->compressPending = false;
+                messagePtr->opCode = 0;
 
                 bool wasTransferred;
                 if (write(messagePtr, wasTransferred)) {
@@ -486,7 +567,7 @@ protected:
 
 public:
     auto getBufferedAmount() {
-        return messageQueue.totalLength;
+        return messageQueue.totalLength + (sendOp ? sendOp->bytes : 0);
     }
 
     Socket(NodeData *nodeData, Loop *loop, uv_os_sock_t fd, SSL *ssl) : Poll(loop, fd), ssl(ssl), nodeData(nodeData) {
@@ -639,6 +720,20 @@ public:
             return;
         }
 
+        if (!ssl && SendWorker::active()) {
+            if (sendBusy()) {
+                return; // the completion submits what has queued up meanwhile
+            }
+            if (getPoll() & UV_WRITABLE) {
+                change(nodeData->loop, this, setPoll(UV_READABLE));
+            }
+            if (submitToWorker()) {
+                return;
+            }
+            // worker queue full: fall through and send synchronously this tick
+        }
+        materializePending();
+
         std::vector<PendingCallback> callbacks;
         bool stalled = false;
         while (!messageQueue.empty() && !stalled) {
@@ -675,9 +770,10 @@ public:
             return;
         }
         unregisterCork();
-        if (isClosed()) {
+        if (isClosed() || sendBusy()) {
             return;
         }
+        materializePending();
         while (!messageQueue.empty()) {
             ssize_t sent = writeQueueGathered();
             if (sent <= 0 || consumeSent(sent, nullptr)) {
@@ -701,13 +797,108 @@ public:
         }
     }
 
+    // Moves up to 512 queued frames into an op and hands it to the worker thread.
+    bool submitToWorker() {
+        if (sendOp || messageQueue.empty() || isClosed()) {
+            return true;
+        }
+        SendOp *op = new SendOp();
+        op->socket = this;
+        op->nodeData = nodeData;
+        op->fd = getFd();
+        op->head = messageQueue.head;
+        op->bytes = 0;
+        op->count = 0;
+        op->result = 0;
+        op->error = 0;
+        op->closeFd = false;
+        op->deflateWindow = workerDeflateWindow;
+        op->destroyWindow = nullptr;
+        Queue::Message *m = messageQueue.head, *last = nullptr;
+        while (m && op->count < 512) {
+            // iov entries are filled by the worker once pending messages are compressed
+            op->bytes += m->length;
+            op->count++;
+            last = m;
+            m = m->nextMessage;
+        }
+        op->tail = last;
+        last->nextMessage = nullptr;
+        messageQueue.head = m;
+        if (!m) {
+            messageQueue.tail = nullptr;
+            messageQueue.totalLength = 0;
+        } else {
+            messageQueue.totalLength -= op->bytes;
+        }
+        if (!SendWorker::submit(op)) {
+            requeueFront(op);
+            delete op;
+            return false;
+        }
+        sendOp = op;
+        return true;
+    }
+
+    // Queues a raw payload to be deflated and framed on the send worker (WebSocket::send
+    // uses this for compressed messages when the worker is active). The payload is copied
+    // because callers may reuse their buffer as soon as send() returns.
+    void enqueueCompressPending(const char *payload, size_t length, unsigned char opCode,
+                                void(*callback)(void *socket, void *data, bool cancelled, void *reserved), void *callbackData) {
+        int memoryIndex = nodeData->getMemoryBlockIndex(sizeof(Queue::Message));
+        Queue::Message *messagePtr = (Queue::Message *) nodeData->getSmallMemoryBlock(memoryIndex);
+        char *raw = new char[length ? length : 1];
+        memcpy(raw, payload, length);
+        messagePtr->data = raw;
+        messagePtr->length = length;
+        messagePtr->nextMessage = nullptr;
+        messagePtr->callback = callback;
+        messagePtr->callbackData = callbackData;
+        messagePtr->reserved = nullptr;
+        messagePtr->poolIndex = memoryIndex;
+        messagePtr->ownsData = true;
+        messagePtr->compressPending = true;
+        messagePtr->opCode = opCode;
+        enqueue(messagePtr);
+        if (!corkPending) {
+            corkPending = true;
+            nodeData->corkState->pending.push_back(this);
+        }
+    }
+
+    // Puts what is left of an op back at the head of the queue, in order.
+    void requeueFront(SendOp *op) {
+        if (!op->head) {
+            return;
+        }
+        op->tail->nextMessage = messageQueue.head;
+        messageQueue.head = op->head;
+        if (!messageQueue.tail) {
+            messageQueue.tail = op->tail;
+        }
+        messageQueue.totalLength += op->bytes;
+    }
+
     template <class T>
     void closeSocket() {
         unregisterCork();
         uv_os_sock_t fd = getFd();
         Context *netContext = nodeData->netContext;
         stop(nodeData->loop);
-        netContext->closeSocket(fd);
+        if (sendOp) {
+            // The worker may be inside a send on this fd: closing it now could let the
+            // number be reused by a new connection and receive our bytes. Orphan the op
+            // and let its completion close the fd.
+            sendOp->socket = nullptr;
+            sendOp->closeFd = true;
+            if (workerDeflateWindow) {
+                sendOp->destroyWindow = workerDeflateWindow; // the worker may still be using it
+                workerOwnsWindow = true;
+            }
+            sendOp = nullptr;
+        } else {
+            netContext->closeSocket(fd);
+        }
 
         if (ssl) {
             SSL_free(ssl);

package/src/WebSocket.cpp

@@ -14,6 +14,8 @@ WebSocket<isServer>::WebSocket(bool perMessageDeflate, cS::Socket *socket) : cS:
         Group<isServer> *group = Group<isServer>::from(this);
         slidingDeflateWindow = Hub::allocateDefaultCompressor(group->deflateLevel, group->deflateWindowBits, group->deflateMemLevel);
     }
+    workerDeflateWindow = slidingDeflateWindow;
+    materializeCb = WebSocket<isServer>::materialize;
 }
 
 /*
@@ -61,9 +63,25 @@ void WebSocket<isServer>::send(const char *message, size_t length, OpCode opCode
         }
     };
 
+    if (transformData.compress && !ssl && corkActive() && cS::SendWorker::active()) {
+        // deflate + framing happen on the send worker, not on the JS thread
+        enqueueCompressPending(message, length, (unsigned char) opCode, (void(*)(void *, void *, bool, void *)) callback, callbackData);
+        return;
+    }
+
     sendTransformed<WebSocketTransformer>((char *) message, length, (void(*)(void *, void *, bool, void *)) callback, callbackData, transformData);
 }
 
+// Main-thread fallback for a compressPending message that a synchronous write path reached
+// (same-tick terminate, full worker queue, drain loop after a short write).
+template <bool isServer>
+void WebSocket<isServer>::materialize(cS::Socket *s, cS::Socket::Queue::Message *m) {
+    WebSocket<isServer> *webSocket = static_cast<WebSocket<isServer> *>(s);
+    Hub *hub = Group<isServer>::from(webSocket)->hub;
+    zlib::Stream *stream = webSocket->slidingDeflateWindow ? (zlib::Stream *) webSocket->slidingDeflateWindow : hub->deflationStream;
+    cS::Socket::materializeOnMain(s, m, stream, !webSocket->slidingDeflateWindow, hub->zlibBuffer, Hub::LARGE_BUFFER_SIZE, hub->dynamicZlibBuffer);
+}
+
 /*
  * Prepares a single message for use with sendPrepared.
  *
@@ -320,8 +338,8 @@ void WebSocket<isServer>::onEnd(cS::Socket *s) {
 
     webSocket->nodeData->clearPendingPollChanges(webSocket);
 
-    // remove any per-websocket zlib memory
-    if (webSocket->slidingDeflateWindow) {
+    // remove any per-websocket zlib memory (unless an orphaned send op took it over)
+    if (webSocket->slidingDeflateWindow && !webSocket->workerOwnsWindow) {
         // this relates to Hub::allocateDefaultCompressor
         zlib::destroy((zlib::Stream *) webSocket->slidingDeflateWindow);
         webSocket->slidingDeflateWindow = nullptr;

package/src/WebSocket.h

@@ -24,6 +24,7 @@ protected:
     unsigned char controlTipLength = 0, hasOutstandingPong = false;
 
     void *slidingDeflateWindow = nullptr;
+    static void materialize(cS::Socket *s, cS::Socket::Queue::Message *m);
 
     WebSocket(bool perMessageDeflate, cS::Socket *socket);
 

package/src/Zlib.cpp

@@ -1,4 +1,7 @@
 #include "Zlib.h"
+#include "MicroDeflate.h"
+#include <cstdlib>
+#include <cstring>
 
 #ifdef CWS_ZLIB_NG
 #include "zlib-ng.h"
@@ -133,8 +136,37 @@ char *inflate(Stream *stream, char *data, size_t &length, size_t maxPayload, cha
     return buffer;
 }
 
+bool microDeflateEnabled() {
+    static int enabled = -1;
+    if (enabled < 0) {
+        const char *v = getenv("CWS_MICRO_DEFLATE");
+        enabled = !(v && (!strcmp(v, "0") || !strcmp(v, "false") || !strcmp(v, "off") || !strcmp(v, "no")));
+    }
+    return enabled == 1;
+}
+
+char *deflateIndependent(Stream *fallback, char *data, size_t &length, char *buffer, size_t bufferSize, std::string &dynamic) {
+    if (!microDeflateEnabled()) {
+        return deflate(fallback, data, length, buffer, bufferSize, dynamic, true);
+    }
+    static thread_local microdeflate::Encoder *encoder = nullptr;
+    if (!encoder) {
+        encoder = new microdeflate::Encoder();
+    }
+    size_t need = microdeflate::bound(length);
+    uint8_t *out;
+    if (need <= bufferSize) {
+        out = (uint8_t *) buffer;
+    } else {
+        dynamic.resize(need);
+        out = (uint8_t *) &dynamic[0];
+    }
+    length = encoder->compress((const uint8_t *) data, length, out);
+    return (char *) out;
+}
+
 const char *backend() {
-    return CWS_ZLIB_BACKEND;
+    return microDeflateEnabled() ? CWS_ZLIB_BACKEND " + microdeflate" : CWS_ZLIB_BACKEND;
 }
 
 }

package/src/Zlib.h

@@ -29,6 +29,12 @@ char *deflate(Stream *stream, char *data, size_t &length, char *buffer, size_t b
 // and length 0 on error or when the output would exceed maxPayload. Always resets.
 char *inflate(Stream *stream, char *data, size_t &length, size_t maxPayload, char *buffer, size_t bufferSize, std::string &dynamic);
 
+// Compresses one independent message (no context takeover). Uses the built-in
+// microdeflate encoder unless CWS_MICRO_DEFLATE=0, in which case `fallback` (a
+// shared zlib stream, reset after) is used. Same output contract as deflate().
+char *deflateIndependent(Stream *fallback, char *data, size_t &length, char *buffer, size_t bufferSize, std::string &dynamic);
+bool microDeflateEnabled();
+
 // e.g. "zlib-ng 2.2.4" or "zlib 1.3.1"
 const char *backend();