kuzzle 2.16.10 → 2.17.1

package/lib/core/network/httpRouter/routeHandler.js

@@ -38,7 +38,7 @@ const kerror = require('../../../kerror').wrap('network', 'http');
  *                           in JSON format
  */
 class RouteHandler {
-  constructor(url, query, message) {
+  constructor (url, query, message) {
     this.handler = null;
     this._request = null;
     this.url = url;
@@ -92,7 +92,7 @@ class RouteHandler {
    * @param {string} name
    * @param {string} value
    */
-  addArgument(name, value) {
+  addArgument (name, value) {
     this.data[name] = value;
   }
 
@@ -101,7 +101,7 @@ class RouteHandler {
    *
    * @param {Function} callback
    */
-  invokeHandler(callback) {
+  invokeHandler (callback) {
     this.handler(this.request, callback);
   }
 }

package/lib/core/network/httpRouter/routePart.js

@@ -32,7 +32,7 @@ const { has } = require('../../../util/safeObject');
  * @class RoutePart
  */
 class RoutePart {
-  constructor() {
+  constructor () {
     this.subparts = {};
     this.placeholders = null;
 
@@ -45,7 +45,7 @@ class RoutePart {
    * @param {string} part
    * @returns {boolean}
    */
-  exists(part) {
+  exists (part) {
     return this.subparts[part] !== undefined && this.subparts[part].handler !== null;
   }
 
@@ -56,8 +56,8 @@ class RoutePart {
    * @param {string} part
    * @returns {RoutePart}
    */
-  getNext(part) {
-    if (!has(this.subparts, part)) {
+  getNext (part) {
+    if (! has(this.subparts, part)) {
       this.subparts[part] = new RoutePart();
     }
 
@@ -71,7 +71,7 @@ class RoutePart {
    * @param {HttpMessage} message
    * @returns {RouteHandler} registered function handler
    */
-  getHandler(message) {
+  getHandler (message) {
     // Do not use WHATWG API yet, stick with the legacy (and deprecated) URL
     // There are two issues:
     //   - Heavy performance impact: https://github.com/nodejs/node/issues/30334

package/lib/core/network/protocolManifest.js

@@ -24,7 +24,7 @@
 const AbstractManifest = require('../shared/abstractManifest');
 
 class ProtocolManifest extends AbstractManifest {
-  constructor(protocolPath, protocol) {
+  constructor (protocolPath, protocol) {
     super(protocolPath);
 
     this.protocol = protocol;

package/lib/core/network/protocols/httpMessage.js

@@ -29,7 +29,7 @@ class HttpMessage {
    * @param {ClientConnection} connection
    * @param {uWS.HttpRequest} request
    */
-  constructor(connection, request) {
+  constructor (connection, request) {
     this.connection = connection;
     this._content = null;
     this.ips = connection.ips;
@@ -54,7 +54,7 @@ class HttpMessage {
   }
 
   set content (value) {
-    if (!value || value.length === 0) {
+    if (! value || value.length === 0) {
       this._content = null;
     }
     else {

package/lib/core/network/protocols/httpwsProtocol.js

@@ -27,6 +27,7 @@ const zlib = require('zlib');
 
 const uWS = require('uWebSockets.js');
 
+const { HttpStream } = require('../../../types');
 const { Request } = require('../../../api/request');
 const { KuzzleError } = require('../../../kerror/errors');
 const Protocol = require('./protocol');
@@ -76,11 +77,18 @@ const HTTP_ALLOWED_CONTENT_TYPES = [
   'application/x-www-form-urlencoded',
   'multipart/form-data',
 ];
+const HTTP_SKIPPED_HEADERS = [
+  'content-length',
+];
+const HTTP_HEADER_CONNECTION = Buffer.from('Connection');
 const HTTP_HEADER_CONTENT_LENGTH = Buffer.from('Content-Length');
 const HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN = Buffer.from('Access-Control-Allow-Origin');
 const HTTP_HEADER_VARY = Buffer.from('Vary');
+const HTTP_HEADER_TRANSFER_ENCODING = Buffer.from('Transfer-Encoding');
+const CHUNKED = Buffer.from('chunked');
 const WILDCARD = Buffer.from('*');
 const ORIGIN = Buffer.from('Origin');
+const CLOSE = Buffer.from('close');
 const CHARSET_REGEX = /charset=([\w-]+)/i;
 
 /**
@@ -119,7 +127,7 @@ class HttpWsProtocol extends Protocol {
     this.wsConfig = this.parseWebSocketOptions();
     this.httpConfig = this.parseHttpOptions();
 
-    if (!this.wsConfig.enabled && !this.httpConfig.enabled) {
+    if (! this.wsConfig.enabled && ! this.httpConfig.enabled) {
       return false;
     }
 
@@ -135,7 +143,7 @@ class HttpWsProtocol extends Protocol {
     }
 
     this.server.listen(entrypoint.config.port, socket => {
-      if (!socket) {
+      if (! socket) {
         throw new Error(`[http/websocket] fatal: unable to listen to port ${entrypoint.config.port}`);
       }
     });
@@ -197,7 +205,7 @@ class HttpWsProtocol extends Protocol {
     const socket = this.socketByConnectionId.get(data.connectionId);
     debugWS('notify: %a', data);
 
-    if (!socket) {
+    if (! socket) {
       return;
     }
 
@@ -212,7 +220,7 @@ class HttpWsProtocol extends Protocol {
   joinChannel (channel, connectionId) {
     const socket = this.socketByConnectionId.get(connectionId);
 
-    if (!socket) {
+    if (! socket) {
       return;
     }
 
@@ -223,7 +231,7 @@ class HttpWsProtocol extends Protocol {
   leaveChannel (channel, connectionId) {
     const socket = this.socketByConnectionId.get(connectionId);
 
-    if (!socket) {
+    if (! socket) {
       return;
     }
 
@@ -237,7 +245,7 @@ class HttpWsProtocol extends Protocol {
 
     const socket = this.socketByConnectionId.get(connectionId);
 
-    if (!socket) {
+    if (! socket) {
       return;
     }
 
@@ -260,7 +268,7 @@ class HttpWsProtocol extends Protocol {
 
   wsOnOpenHandler (socket) {
     const ip = Buffer.from(socket.getRemoteAddressAsText()).toString();
-    const connection = new ClientConnection(this.name, [ip], {cookie: socket.cookie});
+    const connection = new ClientConnection(this.name, [ip], { cookie: socket.cookie });
 
     this.entryPoint.newConnection(connection);
     this.connectionBySocket.set(socket, connection);
@@ -271,7 +279,7 @@ class HttpWsProtocol extends Protocol {
   wsOnCloseHandler (socket, code, message) {
     const connection = this.connectionBySocket.get(socket);
 
-    if (!connection) {
+    if (! connection) {
       return;
     }
 
@@ -289,7 +297,7 @@ class HttpWsProtocol extends Protocol {
   }
 
   wsOnMessageHandler (socket, data) {
-    if (!data || data.byteLength === 0) {
+    if (! data || data.byteLength === 0) {
       return;
     }
 
@@ -346,7 +354,7 @@ class HttpWsProtocol extends Protocol {
       request = new Request(parsed, { connection });
     }
     catch (e) {
-      this.wsSendError(socket, connection, e);
+      this.wsSendError(socket, connection, e, parsed.requestId);
       return;
     }
 
@@ -387,9 +395,14 @@ class HttpWsProtocol extends Protocol {
    * @param  {uWS.WebSocket} socket
    * @param  {ClientConnection} connection
    * @param  {Error} error
+   * @param  {String} requestId @optional
    */
-  wsSendError (socket, connection, error) {
+  wsSendError (socket, connection, error, requestId) {
     const request = new Request({}, { connection, error });
+
+    // If a requestId is provided we use it instead of the generated one
+    request.id = requestId || request.id;
+
     const sanitized = removeErrorStack(request.response.toJSON()).content;
 
     this.wsSend(socket, Buffer.from(JSON.stringify(sanitized)));
@@ -403,7 +416,7 @@ class HttpWsProtocol extends Protocol {
    * @param  {Buffer} payload
    */
   wsSend (socket, payload) {
-    if (!this.connectionBySocket.has(socket)) {
+    if (! this.connectionBySocket.has(socket)) {
       return;
     }
 
@@ -442,7 +455,7 @@ class HttpWsProtocol extends Protocol {
     const contentType = message.headers['content-type'];
 
     if ( contentType
-      && !HTTP_ALLOWED_CONTENT_TYPES.some(allowed => contentType.includes(allowed))
+      && ! HTTP_ALLOWED_CONTENT_TYPES.some(allowed => contentType.includes(allowed))
     ) {
       this.httpSendError(
         message,
@@ -499,7 +512,7 @@ class HttpWsProtocol extends Protocol {
         return;
       }
 
-      if (!isLast) {
+      if (! isLast) {
         return;
       }
 
@@ -582,6 +595,13 @@ class HttpWsProtocol extends Protocol {
         return;
       }
 
+      if (request.result instanceof HttpStream) {
+        this.httpSendStream(request, response, request.result, message);
+        return;
+      }
+
+      // Send the response in one go
+
       const data = this.httpRequestToResponse(request, message);
 
       this.httpCompress(message, data, result => {
@@ -592,36 +612,13 @@ class HttpWsProtocol extends Protocol {
         request.response.setHeader('Content-Encoding', result.encoding);
 
         response.cork(() => {
-          response.writeStatus(Buffer.from(request.response.status.toString()));
-
-          for (const header of this.httpConfig.headers) {
-            // If header is missing, add the default one
-            if (request.response.headers[header[2]] === undefined) {
-              response.writeHeader(header[0], header[1]);
-            }
-          }
-
-          // Access-Control-Allow-Origin Logic
-          if ( request.response.headers['Access-Control-Allow-Origin'] === undefined
-            && message.headers
-            && message.headers.origin
-          ) {
-            response.writeHeader(
-              HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN,
-              Buffer.from(message.headers.origin));
-
-            response.writeHeader(HTTP_HEADER_VARY, ORIGIN);
-          }
-
-          for (const [key, value] of Object.entries(request.response.headers)) {
-            response.writeHeader(Buffer.from(key), Buffer.from(value.toString()));
-          }
+          this.httpWriteRequestHeaders(request, response, message);
 
           const [ success ] = response.tryEnd(
             result.compressed,
             result.compressed.length);
 
-          if (!success) {
+          if (! success) {
             response.onWritable(offset => {
               const retryData = result.compressed.subarray(offset);
               const [ retrySuccess ] = response.tryEnd(
@@ -636,6 +633,182 @@ class HttpWsProtocol extends Protocol {
     });
   }
 
+  /**
+   * 
+   * @param {uWS.HttpRequest} request 
+   * @param {uWS.HttpResponse} response 
+   * @param {HttpMessage} message 
+   */
+  httpWriteRequestHeaders (request, response, message) {
+    response.writeStatus(Buffer.from(request.response.status.toString()));
+    
+    response.writeHeader(HTTP_HEADER_CONNECTION, CLOSE);
+
+    for (const header of this.httpConfig.headers) {
+      // If header is missing, add the default one
+      if (request.response.headers[header[2]] === undefined) {
+        response.writeHeader(header[0], header[1]);
+      }
+    }
+
+    // Access-Control-Allow-Origin Logic
+    if ( request.response.headers['Access-Control-Allow-Origin'] === undefined
+      && message.headers
+      && message.headers.origin
+    ) {
+      response.writeHeader(
+        HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN,
+        Buffer.from(message.headers.origin));
+
+      response.writeHeader(HTTP_HEADER_VARY, ORIGIN);
+    }
+
+    for (const [key, value] of Object.entries(request.response.headers)) {
+      // Skip some headers that are not allowed to be sent or modified
+      if (HTTP_SKIPPED_HEADERS.includes(key.toLowerCase())) {
+        continue;
+      }
+
+      response.writeHeader(Buffer.from(key), Buffer.from(value.toString()));
+    }
+  }
+
+  /**
+   * Send stream data thorugh the HTTP response
+   * @param {HttpRequest} request
+   * @param {HttpStream} httpStream - The data stream to send
+   * @param {uWS.HttpResponse} response
+   * @param {HttpMessage} message
+   */
+  httpSendStream (request, response, httpStream, message) {
+    const streamSizeFixed = typeof httpStream.totalBytes === 'number' && httpStream.totalBytes > 0;
+
+    if (httpStream.errored) {
+      this.httpSendError(
+        message,
+        response,
+        kerror.get('network', 'http', 'stream_errored', httpStream.error.message)
+      );
+      return;
+    }
+
+    if (httpStream.destroyed) {
+      this.httpSendError(
+        message,
+        response,
+        kerror.get('network', 'http', 'stream_closed')
+      );
+      return;
+    }
+
+    // Remove Content-Length header to avoid conflic with Transfer-Encoding header
+    request.response.setHeader('Content-Length', null);
+
+    // Send Headers in one go
+    response.cork(() => {
+      this.httpWriteRequestHeaders(request, response, message);
+
+      if (streamSizeFixed) {
+        response.writeHeader(HTTP_HEADER_CONTENT_LENGTH, Buffer.from(httpStream.totalBytes.toString()));
+      }
+      else {
+        response.writeHeader(HTTP_HEADER_TRANSFER_ENCODING, CHUNKED);
+      }
+    });
+
+    httpStream.stream.on('data', chunk => {
+      // Make a copy of the array buffer
+      const arrayBuffer = chunk.buffer.slice(chunk.byteOffset, chunk.byteOffset + chunk.byteLength);
+
+      const arrayBufferOffset = response.getWriteOffset();
+
+      let backpressure = false;
+
+      /**
+       * Switch method of writing data to the response
+       * based on the size of the stream.
+       * If the stream has a fixed size we can use the tryEnd method
+       */
+      if (streamSizeFixed) {
+        const [ success, done ] = response.tryEnd(arrayBuffer, httpStream.totalBytes);
+        backpressure = ! success;
+
+        if (done) {
+          httpStream.destroy();
+          return;
+        }
+      }
+      else {
+        const success = response.write(arrayBuffer);
+
+        backpressure = ! success;
+      }
+
+      // In case of backpressure we need to wait for drainage before sending more data
+      if (backpressure) {
+        response.arrayBufferOffset = arrayBufferOffset;
+        response.arrayBuffer = arrayBuffer;
+        httpStream.stream.pause();
+
+        /**
+         * When the stream is drained we can resume sending data,
+         * only if the is no backpressure after we wrote the missing chunk data.
+         */
+        response.onWritable(offset => {
+          if (! streamSizeFixed && offset - response.arrayBufferOffset === 0) {
+            httpStream.stream.resume();
+            return true;
+          }
+
+          let retryBackpressure = false;
+          const remainingChunkData = response.arrayBuffer.slice(offset - response.arrayBufferOffset);
+
+          if (streamSizeFixed) {
+            const [ success ] = response.tryEnd(remainingChunkData, httpStream.totalBytes);
+
+            retryBackpressure = ! success;
+          }
+          else {
+            const success = response.write(remainingChunkData);
+
+            retryBackpressure = ! success;
+          }
+
+          if (! retryBackpressure) {
+            httpStream.stream.resume();
+          }
+
+          return ! retryBackpressure;
+        });
+      }
+    });
+
+    httpStream.stream.on('end', () => {
+      if (httpStream.destroy()) {
+        response.end();
+      }
+    });
+
+    httpStream.stream.on('close', () => {
+      if (httpStream.destroy()) {
+        response.end();
+      }
+    });
+
+    httpStream.stream.on('error', err => {
+      if (httpStream.destroy()) {
+        response.end();
+      }
+
+      debugHTTP('[%s] httpSendStream: %s', httpStream.connection.id, err.message);
+    });
+
+    response.onAborted(() => {
+      httpStream.destroy();
+    });
+
+  }
+
   /**
    * Forward an error response to the client
    *
@@ -676,7 +849,8 @@ class HttpWsProtocol extends Protocol {
       if (message.headers && message.headers.origin) {
         if (global.kuzzle.config.internal.allowAllOrigins) {
           response.writeHeader(HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN, WILDCARD);
-        } else {
+        }
+        else {
           response.writeHeader(HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN, Buffer.from(message.headers.origin));
           response.writeHeader(HTTP_HEADER_VARY, ORIGIN);
         }
@@ -699,13 +873,13 @@ class HttpWsProtocol extends Protocol {
    * @param {HttpMessage} message
    * @returns {Buffer}
    */
-  httpRequestToResponse(request, message) {
+  httpRequestToResponse (request, message) {
     let data = removeErrorStack(request.response.toJSON());
 
     if (message.requestId !== data.requestId) {
       data.requestId = message.requestId;
 
-      if (!data.raw) {
+      if (! data.raw) {
         data.content.requestId = message.requestId;
       }
     }
@@ -759,7 +933,7 @@ class HttpWsProtocol extends Protocol {
    * @param  {Buffer} data
    * @param  {Function} callback
    */
-  httpCompress(message, data, callback) {
+  httpCompress (message, data, callback) {
     if (message.headers['accept-encoding']) {
       const encodings = message.headers['accept-encoding']
         .split(',')
@@ -788,15 +962,15 @@ class HttpWsProtocol extends Protocol {
 
       if (algorithm === 'gzip') {
         zlib.gzip(data, (err, compressed) => callback({
-          compressed: !err ? compressed : data,
-          encoding: !err ? 'gzip' : 'identity',
+          compressed: ! err ? compressed : data,
+          encoding: ! err ? 'gzip' : 'identity',
         }));
         return;
       }
       else if (algorithm === 'deflate') {
         zlib.deflate(data, (err, compressed) => callback({
-          compressed: !err ? compressed : data,
-          encoding: !err ? 'deflate' : 'identity',
+          compressed: ! err ? compressed : data,
+          encoding: ! err ? 'deflate' : 'identity',
         }));
         return;
       }
@@ -810,7 +984,7 @@ class HttpWsProtocol extends Protocol {
 
   httpUncompress (message, payload, cb) {
     let encodings = message.headers['content-encoding'];
-    if (!encodings) {
+    if (! encodings) {
       cb(null, payload);
       return;
     }

package/lib/core/network/protocols/mqttProtocol.js

@@ -99,7 +99,7 @@ class MqttProtocol extends Protocol {
     const payload = JSON.stringify(data.payload);
 
     for (const channel of data.channels) {
-      this.aedes.publish({payload, topic: channel}, this.publishCallback);
+      this.aedes.publish({ payload, topic: channel }, this.publishCallback);
     }
   }
 
@@ -126,14 +126,14 @@ class MqttProtocol extends Protocol {
 
     const client = this.connectionsById.get(data.connectionId);
 
-    if (!client) {
+    if (! client) {
       return;
     }
 
     const payload = Buffer.from(JSON.stringify(data.payload));
 
     data.channels.forEach(topic => {
-      client.publish({payload, topic}, this.publishCallback);
+      client.publish({ payload, topic }, this.publishCallback);
     });
   }
 

package/lib/core/network/protocols/protocol.js

@@ -46,7 +46,7 @@ class Protocol {
    *
    * @returns {Promise<boolean>}
    */
-  async init(name, entryPoint) {
+  async init (name, entryPoint) {
     this.entryPoint = entryPoint;
 
     // name should be passed in the constructor
@@ -83,12 +83,12 @@ class Protocol {
 
   joinChannel (channel, connectionId) {
     // do nothing by default
-    return {channel, connectionId};
+    return { channel, connectionId };
   }
 
   leaveChannel (channel, connectionId) {
     // do nothing by default
-    return {channel, connectionId};
+    return { channel, connectionId };
   }
 
   notify () {

package/lib/core/network/router.js

@@ -31,7 +31,7 @@ const HttpRouter = require('./httpRouter');
  * @param {Kuzzle} kuzzle
  */
 class Router {
-  constructor() {
+  constructor () {
     this.connections = new Map();
     this.http = new HttpRouter();
   }
@@ -42,13 +42,14 @@ class Router {
    * @param {RequestContext} requestContext
    */
   newConnection (requestContext) {
-    if (!requestContext.connection.id || !requestContext.connection.protocol) {
+    if (! requestContext.connection.id || ! requestContext.connection.protocol) {
       global.kuzzle.log.error(kerror.get(
         'protocol',
         'runtime',
         'invalid_connection',
         JSON.stringify(requestContext)));
-    } else {
+    }
+    else {
       this.connections.set(requestContext.connection.id, requestContext);
       global.kuzzle.statistics.newConnection(requestContext);
     }
@@ -62,7 +63,7 @@ class Router {
   removeConnection (requestContext) {
     const connId = requestContext.connection.id;
 
-    if (!connId || !requestContext.connection.protocol) {
+    if (! connId || ! requestContext.connection.protocol) {
       global.kuzzle.log.error(kerror.get(
         'protocol',
         'runtime',
@@ -71,7 +72,7 @@ class Router {
       return;
     }
 
-    if (!this.connections.has(connId)) {
+    if (! this.connections.has(connId)) {
       global.kuzzle.log.error(kerror.get(
         'protocol',
         'runtime',
@@ -131,7 +132,7 @@ class Router {
         request.input.action = route.action;
 
         if (route.deprecated) {
-          const { deprecated : { since, message } } = route;
+          const { deprecated: { since, message } } = route;
           request.addDeprecation(since, message);
         }