kuzzle 2.16.11 → 2.17.0

package/lib/core/network/protocols/httpwsProtocol.js

@@ -27,6 +27,7 @@ const zlib = require('zlib');
 
 const uWS = require('uWebSockets.js');
 
+const { HttpStream } = require('../../../types');
 const { Request } = require('../../../api/request');
 const { KuzzleError } = require('../../../kerror/errors');
 const Protocol = require('./protocol');
@@ -80,6 +81,8 @@ const HTTP_HEADER_CONNECTION = Buffer.from('Connection');
 const HTTP_HEADER_CONTENT_LENGTH = Buffer.from('Content-Length');
 const HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN = Buffer.from('Access-Control-Allow-Origin');
 const HTTP_HEADER_VARY = Buffer.from('Vary');
+const HTTP_HEADER_TRANSFER_ENCODING = Buffer.from('Transfer-Encoding');
+const CHUNKED = Buffer.from('chunked');
 const WILDCARD = Buffer.from('*');
 const ORIGIN = Buffer.from('Origin');
 const CLOSE = Buffer.from('close');
@@ -121,7 +124,7 @@ class HttpWsProtocol extends Protocol {
     this.wsConfig = this.parseWebSocketOptions();
     this.httpConfig = this.parseHttpOptions();
 
-    if (!this.wsConfig.enabled && !this.httpConfig.enabled) {
+    if (! this.wsConfig.enabled && ! this.httpConfig.enabled) {
       return false;
     }
 
@@ -137,7 +140,7 @@ class HttpWsProtocol extends Protocol {
     }
 
     this.server.listen(entrypoint.config.port, socket => {
-      if (!socket) {
+      if (! socket) {
         throw new Error(`[http/websocket] fatal: unable to listen to port ${entrypoint.config.port}`);
       }
     });
@@ -199,7 +202,7 @@ class HttpWsProtocol extends Protocol {
     const socket = this.socketByConnectionId.get(data.connectionId);
     debugWS('notify: %a', data);
 
-    if (!socket) {
+    if (! socket) {
       return;
     }
 
@@ -214,7 +217,7 @@ class HttpWsProtocol extends Protocol {
   joinChannel (channel, connectionId) {
     const socket = this.socketByConnectionId.get(connectionId);
 
-    if (!socket) {
+    if (! socket) {
       return;
     }
 
@@ -225,7 +228,7 @@ class HttpWsProtocol extends Protocol {
   leaveChannel (channel, connectionId) {
     const socket = this.socketByConnectionId.get(connectionId);
 
-    if (!socket) {
+    if (! socket) {
       return;
     }
 
@@ -239,7 +242,7 @@ class HttpWsProtocol extends Protocol {
 
     const socket = this.socketByConnectionId.get(connectionId);
 
-    if (!socket) {
+    if (! socket) {
       return;
     }
 
@@ -262,7 +265,7 @@ class HttpWsProtocol extends Protocol {
 
   wsOnOpenHandler (socket) {
     const ip = Buffer.from(socket.getRemoteAddressAsText()).toString();
-    const connection = new ClientConnection(this.name, [ip], {cookie: socket.cookie});
+    const connection = new ClientConnection(this.name, [ip], { cookie: socket.cookie });
 
     this.entryPoint.newConnection(connection);
     this.connectionBySocket.set(socket, connection);
@@ -273,7 +276,7 @@ class HttpWsProtocol extends Protocol {
   wsOnCloseHandler (socket, code, message) {
     const connection = this.connectionBySocket.get(socket);
 
-    if (!connection) {
+    if (! connection) {
       return;
     }
 
@@ -291,7 +294,7 @@ class HttpWsProtocol extends Protocol {
   }
 
   wsOnMessageHandler (socket, data) {
-    if (!data || data.byteLength === 0) {
+    if (! data || data.byteLength === 0) {
       return;
     }
 
@@ -405,7 +408,7 @@ class HttpWsProtocol extends Protocol {
    * @param  {Buffer} payload
    */
   wsSend (socket, payload) {
-    if (!this.connectionBySocket.has(socket)) {
+    if (! this.connectionBySocket.has(socket)) {
       return;
     }
 
@@ -444,7 +447,7 @@ class HttpWsProtocol extends Protocol {
     const contentType = message.headers['content-type'];
 
     if ( contentType
-      && !HTTP_ALLOWED_CONTENT_TYPES.some(allowed => contentType.includes(allowed))
+      && ! HTTP_ALLOWED_CONTENT_TYPES.some(allowed => contentType.includes(allowed))
     ) {
       this.httpSendError(
         message,
@@ -501,7 +504,7 @@ class HttpWsProtocol extends Protocol {
         return;
       }
 
-      if (!isLast) {
+      if (! isLast) {
         return;
       }
 
@@ -584,6 +587,13 @@ class HttpWsProtocol extends Protocol {
         return;
       }
 
+      if (request.result instanceof HttpStream) {
+        this.httpSendStream(request, response, request.result, message);
+        return;
+      }
+
+      // Send the response in one go
+
       const data = this.httpRequestToResponse(request, message);
 
       this.httpCompress(message, data, result => {
@@ -594,38 +604,13 @@ class HttpWsProtocol extends Protocol {
         request.response.setHeader('Content-Encoding', result.encoding);
 
         response.cork(() => {
-          response.writeStatus(Buffer.from(request.response.status.toString()));
-
-          response.writeHeader(HTTP_HEADER_CONNECTION, CLOSE);
-
-          for (const header of this.httpConfig.headers) {
-            // If header is missing, add the default one
-            if (request.response.headers[header[2]] === undefined) {
-              response.writeHeader(header[0], header[1]);
-            }
-          }
-
-          // Access-Control-Allow-Origin Logic
-          if ( request.response.headers['Access-Control-Allow-Origin'] === undefined
-            && message.headers
-            && message.headers.origin
-          ) {
-            response.writeHeader(
-              HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN,
-              Buffer.from(message.headers.origin));
-
-            response.writeHeader(HTTP_HEADER_VARY, ORIGIN);
-          }
-
-          for (const [key, value] of Object.entries(request.response.headers)) {
-            response.writeHeader(Buffer.from(key), Buffer.from(value.toString()));
-          }
+          this.httpWriteRequestHeaders(request, response, message);
 
           const [ success ] = response.tryEnd(
             result.compressed,
             result.compressed.length);
 
-          if (!success) {
+          if (! success) {
             response.onWritable(offset => {
               const retryData = result.compressed.subarray(offset);
               const [ retrySuccess ] = response.tryEnd(
@@ -640,6 +625,177 @@ class HttpWsProtocol extends Protocol {
     });
   }
 
+  /**
+   * 
+   * @param {uWS.HttpRequest} request 
+   * @param {uWS.HttpResponse} response 
+   * @param {HttpMessage} message 
+   */
+  httpWriteRequestHeaders (request, response, message) {
+    response.writeStatus(Buffer.from(request.response.status.toString()));
+    
+    response.writeHeader(HTTP_HEADER_CONNECTION, CLOSE);
+
+    for (const header of this.httpConfig.headers) {
+      // If header is missing, add the default one
+      if (request.response.headers[header[2]] === undefined) {
+        response.writeHeader(header[0], header[1]);
+      }
+    }
+
+    // Access-Control-Allow-Origin Logic
+    if ( request.response.headers['Access-Control-Allow-Origin'] === undefined
+      && message.headers
+      && message.headers.origin
+    ) {
+      response.writeHeader(
+        HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN,
+        Buffer.from(message.headers.origin));
+
+      response.writeHeader(HTTP_HEADER_VARY, ORIGIN);
+    }
+
+    for (const [key, value] of Object.entries(request.response.headers)) {
+      response.writeHeader(Buffer.from(key), Buffer.from(value.toString()));
+    }
+  }
+
+  /**
+   * Send stream data thorugh the HTTP response
+   * @param {HttpRequest} request
+   * @param {HttpStream} httpStream - The data stream to send
+   * @param {uWS.HttpResponse} response
+   * @param {HttpMessage} message
+   */
+  httpSendStream (request, response, httpStream, message) {
+    const streamSizeFixed = typeof httpStream.totalBytes === 'number' && httpStream.totalBytes > 0;
+
+    if (httpStream.errored) {
+      this.httpSendError(
+        message,
+        response,
+        kerror.get('network', 'http', 'stream_errored', httpStream.error.message)
+      );
+      return;
+    }
+
+    if (httpStream.destroyed) {
+      this.httpSendError(
+        message,
+        response,
+        kerror.get('network', 'http', 'stream_closed')
+      );
+      return;
+    }
+
+    // Remove Content-Length header to avoid conflic with Transfer-Encoding header
+    request.response.setHeader('Content-Length', null);
+
+    // Send Headers in one go
+    response.cork(() => {
+      this.httpWriteRequestHeaders(request, response, message);
+
+      if (streamSizeFixed) {
+        response.writeHeader(HTTP_HEADER_CONTENT_LENGTH, Buffer.from(httpStream.totalBytes.toString()));
+      }
+      else {
+        response.writeHeader(HTTP_HEADER_TRANSFER_ENCODING, CHUNKED);
+      }
+    });
+
+    httpStream.stream.on('data', chunk => {
+      // Make a copy of the array buffer
+      const arrayBuffer = chunk.buffer.slice(chunk.byteOffset, chunk.byteOffset + chunk.byteLength);
+
+      const arrayBufferOffset = response.getWriteOffset();
+
+      let backpressure = false;
+
+      /**
+       * Switch method of writing data to the response
+       * based on the size of the stream.
+       * If the stream has a fixed size we can use the tryEnd method
+       */
+      if (streamSizeFixed) {
+        const [ success, done ] = response.tryEnd(arrayBuffer, httpStream.totalBytes);
+        backpressure = ! success;
+
+        if (done) {
+          httpStream.destroy();
+          return;
+        }
+      }
+      else {
+        const success = response.write(arrayBuffer);
+
+        backpressure = ! success;
+      }
+
+      // In case of backpressure we need to wait for drainage before sending more data
+      if (backpressure) {
+        response.arrayBufferOffset = arrayBufferOffset;
+        response.arrayBuffer = arrayBuffer;
+        httpStream.stream.pause();
+
+        /**
+         * When the stream is drained we can resume sending data,
+         * only if the is no backpressure after we wrote the missing chunk data.
+         */
+        response.onWritable(offset => {
+          if (! streamSizeFixed && offset - response.arrayBufferOffset === 0) {
+            httpStream.stream.resume();
+            return true;
+          }
+
+          let retryBackpressure = false;
+          const remainingChunkData = response.arrayBuffer.slice(offset - response.arrayBufferOffset);
+
+          if (streamSizeFixed) {
+            const [ success ] = response.tryEnd(remainingChunkData, httpStream.totalBytes);
+
+            retryBackpressure = ! success;
+          }
+          else {
+            const success = response.write(remainingChunkData);
+
+            retryBackpressure = ! success;
+          }
+
+          if (! retryBackpressure) {
+            httpStream.stream.resume();
+          }
+
+          return ! retryBackpressure;
+        });
+      }
+    });
+
+    httpStream.stream.on('end', () => {
+      if (httpStream.destroy()) {
+        response.end();
+      }
+    });
+
+    httpStream.stream.on('close', () => {
+      if (httpStream.destroy()) {
+        response.end();
+      }
+    });
+
+    httpStream.stream.on('error', err => {
+      if (httpStream.destroy()) {
+        response.end();
+      }
+
+      debugHTTP('[%s] httpSendStream: %s', httpStream.connection.id, err.message);
+    });
+
+    response.onAborted(() => {
+      httpStream.destroy();
+    });
+
+  }
+
   /**
    * Forward an error response to the client
    *
@@ -680,7 +836,8 @@ class HttpWsProtocol extends Protocol {
       if (message.headers && message.headers.origin) {
         if (global.kuzzle.config.internal.allowAllOrigins) {
           response.writeHeader(HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN, WILDCARD);
-        } else {
+        }
+        else {
           response.writeHeader(HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN, Buffer.from(message.headers.origin));
           response.writeHeader(HTTP_HEADER_VARY, ORIGIN);
         }
@@ -703,13 +860,13 @@ class HttpWsProtocol extends Protocol {
    * @param {HttpMessage} message
    * @returns {Buffer}
    */
-  httpRequestToResponse(request, message) {
+  httpRequestToResponse (request, message) {
     let data = removeErrorStack(request.response.toJSON());
 
     if (message.requestId !== data.requestId) {
       data.requestId = message.requestId;
 
-      if (!data.raw) {
+      if (! data.raw) {
         data.content.requestId = message.requestId;
       }
     }
@@ -763,7 +920,7 @@ class HttpWsProtocol extends Protocol {
    * @param  {Buffer} data
    * @param  {Function} callback
    */
-  httpCompress(message, data, callback) {
+  httpCompress (message, data, callback) {
     if (message.headers['accept-encoding']) {
       const encodings = message.headers['accept-encoding']
         .split(',')
@@ -792,15 +949,15 @@ class HttpWsProtocol extends Protocol {
 
       if (algorithm === 'gzip') {
         zlib.gzip(data, (err, compressed) => callback({
-          compressed: !err ? compressed : data,
-          encoding: !err ? 'gzip' : 'identity',
+          compressed: ! err ? compressed : data,
+          encoding: ! err ? 'gzip' : 'identity',
         }));
         return;
       }
       else if (algorithm === 'deflate') {
         zlib.deflate(data, (err, compressed) => callback({
-          compressed: !err ? compressed : data,
-          encoding: !err ? 'deflate' : 'identity',
+          compressed: ! err ? compressed : data,
+          encoding: ! err ? 'deflate' : 'identity',
         }));
         return;
       }
@@ -814,7 +971,7 @@ class HttpWsProtocol extends Protocol {
 
   httpUncompress (message, payload, cb) {
     let encodings = message.headers['content-encoding'];
-    if (!encodings) {
+    if (! encodings) {
       cb(null, payload);
       return;
     }

package/lib/core/network/protocols/mqttProtocol.js

@@ -99,7 +99,7 @@ class MqttProtocol extends Protocol {
     const payload = JSON.stringify(data.payload);
 
     for (const channel of data.channels) {
-      this.aedes.publish({payload, topic: channel}, this.publishCallback);
+      this.aedes.publish({ payload, topic: channel }, this.publishCallback);
     }
   }
 
@@ -126,14 +126,14 @@ class MqttProtocol extends Protocol {
 
     const client = this.connectionsById.get(data.connectionId);
 
-    if (!client) {
+    if (! client) {
       return;
     }
 
     const payload = Buffer.from(JSON.stringify(data.payload));
 
     data.channels.forEach(topic => {
-      client.publish({payload, topic}, this.publishCallback);
+      client.publish({ payload, topic }, this.publishCallback);
     });
   }
 

package/lib/core/network/protocols/protocol.js

@@ -46,7 +46,7 @@ class Protocol {
    *
    * @returns {Promise<boolean>}
    */
-  async init(name, entryPoint) {
+  async init (name, entryPoint) {
     this.entryPoint = entryPoint;
 
     // name should be passed in the constructor
@@ -83,12 +83,12 @@ class Protocol {
 
   joinChannel (channel, connectionId) {
     // do nothing by default
-    return {channel, connectionId};
+    return { channel, connectionId };
   }
 
   leaveChannel (channel, connectionId) {
     // do nothing by default
-    return {channel, connectionId};
+    return { channel, connectionId };
   }
 
   notify () {

package/lib/core/network/router.js

@@ -31,7 +31,7 @@ const HttpRouter = require('./httpRouter');
  * @param {Kuzzle} kuzzle
  */
 class Router {
-  constructor() {
+  constructor () {
     this.connections = new Map();
     this.http = new HttpRouter();
   }
@@ -42,13 +42,14 @@ class Router {
    * @param {RequestContext} requestContext
    */
   newConnection (requestContext) {
-    if (!requestContext.connection.id || !requestContext.connection.protocol) {
+    if (! requestContext.connection.id || ! requestContext.connection.protocol) {
       global.kuzzle.log.error(kerror.get(
         'protocol',
         'runtime',
         'invalid_connection',
         JSON.stringify(requestContext)));
-    } else {
+    }
+    else {
       this.connections.set(requestContext.connection.id, requestContext);
       global.kuzzle.statistics.newConnection(requestContext);
     }
@@ -62,7 +63,7 @@ class Router {
   removeConnection (requestContext) {
     const connId = requestContext.connection.id;
 
-    if (!connId || !requestContext.connection.protocol) {
+    if (! connId || ! requestContext.connection.protocol) {
       global.kuzzle.log.error(kerror.get(
         'protocol',
         'runtime',
@@ -71,7 +72,7 @@ class Router {
       return;
     }
 
-    if (!this.connections.has(connId)) {
+    if (! this.connections.has(connId)) {
       global.kuzzle.log.error(kerror.get(
         'protocol',
         'runtime',
@@ -131,7 +132,7 @@ class Router {
         request.input.action = route.action;
 
         if (route.deprecated) {
-          const { deprecated : { since, message } } = route;
+          const { deprecated: { since, message } } = route;
           request.addDeprecation(since, message);
         }
 

package/lib/core/plugin/plugin.js

@@ -24,7 +24,6 @@
 const path = require('path');
 const fs = require('fs');
 const semver = require('semver');
-const enforcer = require('openapi-enforcer');
 
 const { PluginContext } = require('./pluginContext');
 const PrivilegedPluginContext = require('./privilegedContext');
@@ -34,7 +33,6 @@ const Manifest = require('./pluginManifest');
 const { has, isPlainObject } = require('../../util/safeObject');
 
 const assertionError = kerror.wrap('plugin', 'assert');
-const controllerError = kerror.wrap('plugin', 'controller');
 const runtimeError = kerror.wrap('plugin', 'runtime');
 
 const PLUGIN_NAME_REGEX = /^[a-z-\d]+$/;
@@ -43,7 +41,7 @@ const HTTP_VERBS = ['get', 'head', 'post', 'put', 'delete', 'patch', 'options'];
 class Plugin {
   constructor (
     instance,
-    { name, application=false, deprecationWarning=true } = {}
+    { name, application = false, deprecationWarning = true } = {}
   ) {
     this._instance = instance;
 
@@ -154,24 +152,46 @@ class Plugin {
 
   // getters/setters ===========================================================
 
-  get instance () { return this._instance; }
+  get instance () {
+    return this._instance;
+  }
 
-  get context () { return this._context; }
+  get context () {
+    return this._context;
+  }
 
-  get application () { return this._application; }
+  get application () {
+    return this._application;
+  }
 
-  get logPrefix () { return `[${this.name}]`; }
+  get logPrefix () {
+    return `[${this.name}]`;
+  }
 
-  get config () { return this._config; }
-  set config (config) { this._config = config; }
+  get config () {
+    return this._config;
+  }
+  set config (config) {
+    this._config = config;
+  }
 
-  get version () { return this._version; }
-  set version (version) { this._version = version; }
+  get version () {
+    return this._version;
+  }
+  set version (version) {
+    this._version = version;
+  }
 
-  get deprecationWarning () { return this._deprecationWarning; }
-  set deprecationWarning (value) { this._deprecationWarning = value; }
+  get deprecationWarning () {
+    return this._deprecationWarning;
+  }
+  set deprecationWarning (value) {
+    this._deprecationWarning = value;
+  }
 
-  get name () { return this._name; }
+  get name () {
+    return this._name;
+  }
   set name (name) {
     if (! this.constructor.checkName(name)) {
       this.printDeprecation('Plugin names should be in kebab-case. This behavior will be enforced in futur versions of Kuzzle.');
@@ -180,7 +200,9 @@ class Plugin {
     this._name = name;
   }
 
-  get manifest () { return this._manifest; }
+  get manifest () {
+    return this._manifest;
+  }
   set manifest (manifest) {
     this._manifest = manifest;
   }
@@ -264,7 +286,7 @@ class Plugin {
     return PLUGIN_NAME_REGEX.test(name);
   }
 
-  static async checkControllerDefinition (name, definition, { application=false } = {}) {
+  static checkControllerDefinition (name, definition, { application = false } = {}) {
     if (typeof name !== 'string') {
       throw assertionError.get(
         'invalid_controller_definition',
@@ -342,10 +364,6 @@ class Plugin {
               name,
               `action "${action}" has invalid http properties: ${routeProperties.join(', ')}`);
           }
-
-          if (route.openapi) {
-            await checkOpenAPISpecification(name, action, route);
-          }
         }
       }
     }
@@ -365,48 +383,4 @@ function checkHttpRouteProperties (route, action, name, application) {
   }
 }
 
-async function checkOpenAPISpecification (controller, action, route) {
-  if (! isPlainObject(route.openapi)) {
-    throw assertionError.get(
-      'invalid_controller_definition',
-      controller,
-      'The "openapi" property of the route definition must be an object');
-  }
-
-  const routePath = route.path.charAt(0) === '/'
-    ? route.path
-    : `/_/${route.path}`;
-
-  // Set :param notation to {param}
-  const formattedPath = routePath.replace(/\/:([^/]*)/g,'/{$1}');
-
-  const { error, warning } = await enforcer({
-    /* eslint-disable sort-keys */
-    openapi: '3.0.0',
-    info: {
-      title: 'Kuzzle API',
-      version: require('../../../package').version
-    },
-    paths: {
-      [formattedPath]: {
-        [route.verb]: route.openapi
-      }
-    },
-    /* eslint-enable sort-keys */
-  });
-
-  if (warning) {
-    global.kuzzle.log.warn(`Warning for OpenAPI specification in "${controller}:${action}", ${route.openapi} : ${warning}`);
-  }
-
-  if (error) {
-    throw controllerError.get(
-      'invalid_openapi_schema',
-      controller,
-      action,
-      route.openapi,
-      error);
-  }
-}
-
 module.exports = Plugin;

package/lib/core/plugin/pluginManifest.js

@@ -27,7 +27,7 @@ const kerror = require('../../kerror').wrap('plugin', 'manifest');
 const AbstractManifest = require('../shared/abstractManifest');
 
 class PluginManifest extends AbstractManifest {
-  constructor(pluginPath) {
+  constructor (pluginPath) {
     super(pluginPath);
     this.privileged = false;
   }
@@ -36,11 +36,11 @@ class PluginManifest extends AbstractManifest {
     super.load();
 
     // Ensure ES will accept the plugin name as index
-    if (!/^[\w-]+$/.test(this.name)) {
+    if (! /^[\w-]+$/.test(this.name)) {
       throw kerror.get('invalid_name', this.path);
     }
 
-    if (!_.isNil(this.raw) && !_.isNil(this.raw.privileged)) {
+    if (! _.isNil(this.raw) && ! _.isNil(this.raw.privileged)) {
       if (typeof this.raw.privileged !== 'boolean') {
         throw kerror.get(
           'invalid_privileged',