kuzzle 2.49.1 → 2.50.0-beta.3

package/dist/lib/core/cache/cacheEngine.js

@@ -0,0 +1,205 @@
+/*
+ * Kuzzle, a backend software, self-hostable and ready to use
+ * to power modern apps
+ *
+ * Copyright 2015-2022 Kuzzle
+ * mailto: support AT kuzzle.io
+ * website: http://kuzzle.io
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+"use strict";
+const Bluebird = require("bluebird");
+const Redis = require("../../service/cache/redis");
+class CacheEngine {
+    constructor() {
+        const config = global.kuzzle.config.services;
+        this.public = new Redis(config.memoryStorage, "public_adapter");
+        this.internal = new Redis(config.internalCache, "internal_adapter");
+        this.logger = global.kuzzle.log.child("core:cache:cacheEngine");
+    }
+    /**
+     * Initializes the redis clients
+     *
+     * @returns {Promise}
+     */
+    async init() {
+        await Bluebird.all([this.public.init(), this.internal.init()]);
+        this.registerInternalEvents();
+        this.registerPublicEvents();
+        this.logger.info("[✔] Cache initialized");
+    }
+    registerInternalEvents() {
+        /**
+         * Deletes on or multiple keys
+         * @param  {string|Array.<string>} keys
+         */
+        global.kuzzle.onAsk("core:cache:internal:del", (keys) => this.internal.commands.del(keys));
+        /**
+         * Asks a key to expire in the provided delay
+         * @param {string} key
+         * @param {number} ttl (in seconds)
+         */
+        global.kuzzle.onAsk("core:cache:internal:expire", (key, ttl) => this.internal.commands.expire(key, ttl));
+        /**
+         * Wipes the database clean
+         */
+        global.kuzzle.onAsk("core:cache:internal:flushdb", () => this.internal.commands.flushdb());
+        /**
+         * Returns basic information about the internal cache service
+         * @returns {Promise.<Object>}
+         */
+        global.kuzzle.onAsk("core:cache:internal:info:get", () => this.internal.info());
+        /**
+         * Fetches a single value
+         * @param {string} key
+         * @return {string}
+         */
+        global.kuzzle.onAsk("core:cache:internal:get", (key) => this.internal.commands.get(key));
+        /**
+         * Fetches multiple values in one go
+         * @param  {Array.<string>} ids
+         * @return {Array.<string|null>}
+         */
+        global.kuzzle.onAsk("core:cache:internal:mget", (keys) => {
+            // redis throws an error if trying to mget without arguments
+            if (keys.length === 0) {
+                return [];
+            }
+            return this.internal.commands.mget(keys);
+        });
+        /**
+         * Makes a key persistent (disable its expiration delay, if there is one)
+         * @param {string} key
+         */
+        global.kuzzle.onAsk("core:cache:internal:persist", (key) => this.internal.commands.persist(key));
+        /**
+         * Asks a key to expire in the provided delay
+         * @param {string} key
+         * @param {number} ttl (in milliseconds)
+         */
+        global.kuzzle.onAsk("core:cache:internal:pexpire", (key, ttl) => this.internal.commands.pexpire(key, ttl));
+        /**
+         * Fetches all keys matching the provided pattern
+         * @param {string} pattern
+         */
+        global.kuzzle.onAsk("core:cache:internal:searchKeys", (pattern) => this.internal.searchKeys(pattern));
+        /**
+         * Add a custom LUA script to the internal client
+         * The script can then be executed using core:cache:internal:script:execute
+         * @see {@link https://www.npmjs.com/package/ioredis#lua-scripting}
+         *
+         * @param {string} name of the script
+         * @param {number} keys -- number of keys
+         * @param {string} script
+         */
+        global.kuzzle.onAsk("core:cache:internal:script:define", (name, keys, script) => {
+            return this.internal.client.defineCommand(name, {
+                lua: script,
+                numberOfKeys: keys,
+            });
+        });
+        /**
+         * Execute a script previously defined with core:cache:internal:script:define
+         *
+         * @param {string} name of the script to execute
+         * @param {...string} args -- script arguments
+         * @return {*} script result (if any)
+         */
+        global.kuzzle.onAsk("core:cache:internal:script:execute", (name, ...args) => this.internal.client[name](...args));
+        /**
+         * Convenience method for easy access to options NX and PX of the "set"
+         * command.
+         * Deliberately not named after an existing Redis command to prevent
+         * confusion with Redis' API.
+         *
+         * Options:
+         *   - ttl: key expiration TTL in milliseconds
+         *   - onlyIfNew: if true, does not write the key if it already exists
+         *
+         * @param {string} key
+         * @param {string} value
+         * @param {{ttl: number, onlyIfNew: boolean}} [opts]
+         * @returns {Promise.<boolean>} true if the key was set, false otherwise
+         */
+        global.kuzzle.onAsk("core:cache:internal:store", (key, value, opts) => this.internal.store(key, value, opts));
+        /**
+         * Executes an arbitrary NATIVE cache command directly
+         * @param {string} command
+         * @param {Array} args -- command arguments
+         */
+        global.kuzzle.onAsk("core:cache:internal:execute", (command, ...args) => this.internal.exec(command, ...args));
+    }
+    registerPublicEvents() {
+        /**
+         * Deletes on or multiple keys
+         * @param  {string|Array.<string>} keys
+         */
+        global.kuzzle.onAsk("core:cache:public:del", (keys) => this.public.commands.del(keys));
+        /**
+         * Executes an arbitrary NATIVE cache command directly
+         * @param {string} command
+         * @param {Array} args -- command arguments
+         */
+        global.kuzzle.onAsk("core:cache:public:execute", (command, ...args) => this.public.exec(command, ...args));
+        /**
+         * Asks a key to expire in the provided delay
+         * @param {string} key
+         * @param {number} ttl (in seconds)
+         */
+        global.kuzzle.onAsk("core:cache:public:expire", (key, ttl) => this.public.commands.expire(key, ttl));
+        /**
+         * Wipes the database clean
+         */
+        global.kuzzle.onAsk("core:cache:public:flushdb", () => this.public.commands.flushdb());
+        /**
+         * Returns basic information about the internal cache service
+         * @returns {Promise.<Object>}
+         */
+        global.kuzzle.onAsk("core:cache:public:info:get", () => this.public.info());
+        /**
+         * Fetches a single value
+         * @param {string} key
+         * @return {string}
+         */
+        global.kuzzle.onAsk("core:cache:public:get", (key) => this.public.commands.get(key));
+        /**
+         * Executes multiple cache commands in one go, as a single transaction
+         * @param {Array} commands to execute
+         */
+        global.kuzzle.onAsk("core:cache:public:mExecute", (commands) => this.public.mExecute(commands));
+        /**
+         * Makes a key persistent (disable its expiration delay, if there is one)
+         * @param {string} key
+         */
+        global.kuzzle.onAsk("core:cache:public:persist", (key) => this.public.commands.persist(key));
+        /**
+         * Convenience method for easy access to options NX and PX of the "set"
+         * command.
+         * Deliberately not named after an existing Redis command to prevent
+         * confusion with Redis' API.
+         *
+         * Options:
+         *   - ttl: key expiration TTL in milliseconds
+         *   - onlyIfNew: if true, does not write the key if it already exists
+         *
+         * @param {string} key
+         * @param {string} value
+         * @param {{ttl: number, onlyIfNew: boolean}} [opts]
+         * @returns {Promise.<boolean>} true if the key was set, false otherwise
+         */
+        global.kuzzle.onAsk("core:cache:public:store", (key, value, opts) => this.public.store(key, value, opts));
+    }
+}
+module.exports = CacheEngine;
+//# sourceMappingURL=cacheEngine.js.map

package/dist/lib/core/network/accessLogger.d.ts

@@ -0,0 +1,29 @@
+/// <reference types="node" />
+export class AccessLogger {
+    isActive: boolean;
+    worker: Worker;
+    logger: import("../../kuzzle/Logger").Logger;
+    init(): Promise<void>;
+    log(connection: any, request: any, extra: any): void;
+}
+export class AccessLoggerWorker {
+    constructor(config: any, anonymousUserId: any);
+    config: any;
+    logger: pino.Logger<never, boolean>;
+    anonymousUserId: any;
+    init(): void;
+    initTransport(): void;
+    /**
+     * @param {ClientConnection} connection
+     * @param {Request} request
+     * @param {String} size - response size, in bytes
+     * @param {Object} [extra]
+     */
+    logAccess(connection: ClientConnection, request: Request, size: string, extra?: any): void;
+    /**
+     * @param  {ClientConnection} connection
+     */
+    getIP(connection: ClientConnection): any;
+}
+import { Worker } from "node:worker_threads";
+import pino = require("pino");

package/dist/lib/core/network/accessLogger.js

@@ -0,0 +1,250 @@
+/*
+ * Kuzzle, a backend software, self-hostable and ready to use
+ * to power modern apps
+ *
+ * Copyright 2015-2022 Kuzzle
+ * mailto: support AT kuzzle.io
+ * website: http://kuzzle.io
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+"use strict";
+// winston is CPU-hungry: isolating it in a worker thread allows for a more
+// efficient CPU resources management, and more performances in the end
+const { Worker, isMainThread, parentPort, workerData, } = require("worker_threads");
+const pino = require("pino");
+const moment = require("moment");
+const { KuzzleRequest } = require("../../api/request");
+const ALLOWED_TRANSPORTS = ["console", "elasticsearch", "file", "syslog"];
+class AccessLogger {
+    constructor() {
+        this.isActive = false;
+        this.worker = null;
+        this.logger = global.kuzzle.log.child("core:network:accessLogger");
+    }
+    async init() {
+        const config = global.kuzzle.config.server;
+        for (const out of config.logs.transports) {
+            if (out.transport && !ALLOWED_TRANSPORTS.includes(out.transport)) {
+                this.logger.error(`Failed to initialize logger transport "${out.transport}": unsupported transport. Skipped.`);
+            }
+            else {
+                this.isActive = this.isActive || !out.silent;
+            }
+        }
+        if (!this.isActive) {
+            return;
+        }
+        const anonymous = await global.kuzzle.ask("core:security:user:anonymous:get");
+        this.worker = new Worker(__filename, {
+            workerData: {
+                anonymousUserId: anonymous._id,
+                config,
+                kuzzleId: global.nodeId,
+            },
+        });
+    }
+    log(connection, request, extra) {
+        if (!this.isActive) {
+            return;
+        }
+        const serialized = request.serialize();
+        // Users can set anything in the request response, as long as it can
+        // be stringified. Problem is: not all that can be stringified is
+        // serializable to worker threads (e.g. functions).
+        serialized.options.result = undefined;
+        // Since we won't pass the response to the worker thread, we need to
+        // compute its size beforehand, as this information is needed for access
+        // logs
+        const size = request.response
+            ? Buffer.byteLength(JSON.stringify(request.response)).toString()
+            : "-";
+        try {
+            this.worker.postMessage({
+                connection,
+                extra,
+                request: serialized,
+                size,
+            });
+        }
+        catch (error) {
+            this.logger.error(`Failed to write access log for request "${request.id}": ${error.message}`);
+        }
+    }
+}
+class AccessLoggerWorker {
+    constructor(config, anonymousUserId) {
+        this.config = config;
+        this.logger = null;
+        this.anonymousUserId = anonymousUserId;
+    }
+    init() {
+        this.initTransport();
+        parentPort.on("message", ({ connection, extra, request, size }) => {
+            this.logAccess(connection, new KuzzleRequest(request.data, request.options), size, extra);
+        });
+    }
+    initTransport() {
+        const transports = { targets: [] };
+        for (const conf of this.config.logs.transports) {
+            if (conf.silent === true) {
+                continue;
+            }
+            // Guarantee default transport is 'console' and retro compatibility with winston options
+            switch (conf.transport || conf.preset || "console") {
+                case "console":
+                    transports.targets.push({
+                        level: conf.level || "info",
+                        options: {
+                            destination: 1,
+                        },
+                        target: "pino/file",
+                    });
+                    break;
+                case "elasticsearch":
+                    transports.targets.push({
+                        level: conf.level || "info",
+                        options: Object.assign({}, conf.options),
+                        target: "pino-elasticsearch",
+                    });
+                    break;
+                case "file":
+                    transports.targets.push({
+                        level: conf.level || "info",
+                        options: {
+                            append: conf.options?.append ?? true,
+                            destination: conf.options?.destination ?? "./kuzzle.access.log",
+                            mkdir: conf.options?.mkdir ?? true,
+                        },
+                        target: "pino/file",
+                    });
+                    break;
+                default:
+                // do nothing
+            }
+            // If a pino transport configuration is used, we'll try to use it as-is and
+            // assume the user installed the necessary dependencies in his Kuzzle application
+            if (typeof conf.target === "string" && conf.target !== "") {
+                transports.targets.push({
+                    level: conf.level || "info",
+                    options: conf.options || {},
+                    target: conf.target,
+                });
+            }
+        }
+        this.logger = pino.pino(pino.transport(transports));
+    }
+    /**
+     * @param {ClientConnection} connection
+     * @param {Request} request
+     * @param {String} size - response size, in bytes
+     * @param {Object} [extra]
+     */
+    logAccess(connection, request, size, extra = null) {
+        if (this.config.logs.accessLogFormat === "logstash") {
+            // custom kuzzle logs to be exported to logstash
+            this.logger.info({
+                connection,
+                error: request.error,
+                extra,
+                namespace: "kuzzle:accessLogs",
+                nodeId: global.kuzzle.id,
+                request: request.input,
+                status: request.status,
+            });
+            return;
+        }
+        // user init: prioritize the already decoded and verified token stored in
+        // the request
+        // If not available, then that may mean that we didn't verify the user yet,
+        // so we have to decode any provided token
+        let user = null;
+        if (request.context.token !== null) {
+            user =
+                request.context.token.userId === this.anonymousUserId
+                    ? "(anonymous)"
+                    : request.context.token.userId;
+        }
+        // = apache combined
+        const protocol = connection.protocol.toUpperCase();
+        let url;
+        let verb = "DO";
+        if (connection.protocol.indexOf("HTTP/") === 0) {
+            verb = extra.method;
+            url = extra.url;
+        }
+        // for other protocols than http, we rebuild a pseudo url
+        else {
+            url = `/${request.input.controller}/${request.input.action}`;
+            if (request.input.args.index) {
+                url += `/${request.input.args.index}`;
+            }
+            if (request.input.args.collection) {
+                url += `/${request.input.args.collection}`;
+            }
+            if (request.input.args._id) {
+                url += `/${request.input.args._id}`;
+            }
+            let queryString = "";
+            for (const k of Object.keys(request.input.args)) {
+                if (k === "_id" || k === "index" || k === "collection") {
+                    continue;
+                }
+                const val = request.input.args[k];
+                if (queryString.length > 0) {
+                    queryString += "&";
+                }
+                queryString += `${k}=${typeof val === "object" ? JSON.stringify(val) : val}`;
+            }
+            if (queryString.length > 0) {
+                url += `?${queryString}`;
+            }
+        }
+        if (user === null) {
+            user = "(unknown)";
+        }
+        const ip = this.getIP(connection);
+        const when = moment().format("DD/MMM/YYYY:HH:mm:ss ZZ");
+        const status = request.status || "-";
+        const referer = connection.headers.referer
+            ? `"${connection.headers.referer}"`
+            : "-";
+        const agent = connection.headers["user-agent"]
+            ? `"${connection.headers["user-agent"]}"`
+            : "-";
+        this.logger.info({ namespace: "kuzzle:accessLogs", nodeId: global.kuzzle.id }, `${ip} - ${user} [${when}] "${verb} ${url} ${protocol}" ${status} ${size} ${referer} ${agent}`);
+    }
+    /**
+     * @param  {ClientConnection} connection
+     */
+    getIP(connection) {
+        const { ips } = connection;
+        if (ips.length === 0) {
+            return "-";
+        }
+        const idx = Math.max(0, ips.length - 1 - this.config.logs.accessLogIpOffset);
+        return ips[idx];
+    }
+}
+if (!isMainThread) {
+    // Needed for instantiating a serialized KuzzleRequest object
+    global.kuzzle = { id: workerData.kuzzleId };
+    const worker = new AccessLoggerWorker(workerData.config, workerData.anonymousUserId);
+    worker.init();
+}
+// Exposing the worker isn't necessary for production code: this is only
+// useful to make this class testable. I usually don't like it when tests have a
+// say in how the code should be written, but in this particular case, I see
+// no other way to correctly test this.
+module.exports = { AccessLogger, AccessLoggerWorker };
+//# sourceMappingURL=accessLogger.js.map

package/dist/lib/core/network/clientConnection.d.ts

@@ -0,0 +1,15 @@
+export = ClientConnection;
+/**
+ * @class ClientConnection
+ * @param {string} protocol - The protocol used (http, websocket, mqtt etc)
+ * @param {Array.<string>} ips - The list of forwarded ips (= X-Forwarded-For http header + the final ip, i.e. client, proxy1, proxy2, etc.)
+ * @param {object} [headers] - Optional extra key-value object. I.e., for http, will receive the request headers
+ */
+declare class ClientConnection {
+    constructor(protocol: any, ips: any, headers?: any, internal?: any);
+    id: string;
+    protocol: any;
+    headers: any;
+    internal: any;
+    ips: any[];
+}

package/{lib → dist/lib}/core/network/clientConnection.js

@@ -18,12 +18,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 "use strict";
-
 const { isPlainObject } = require("../../util/safeObject");
 const uuid = require("uuid");
-
 /**
  * @class ClientConnection
  * @param {string} protocol - The protocol used (http, websocket, mqtt etc)
@@ -31,27 +28,23 @@ const uuid = require("uuid");
  * @param {object} [headers] - Optional extra key-value object. I.e., for http, will receive the request headers
  */
 class ClientConnection {
-  constructor(protocol, ips, headers = null, internal = null) {
-    this.id = uuid.v4();
-    this.protocol = protocol;
-    this.headers = {};
-    this.internal = {};
-
-    if (!Array.isArray(ips)) {
-      throw new TypeError(`Expected ips to be an Array, got ${typeof ips}`);
+    constructor(protocol, ips, headers = null, internal = null) {
+        this.id = uuid.v4();
+        this.protocol = protocol;
+        this.headers = {};
+        this.internal = {};
+        if (!Array.isArray(ips)) {
+            throw new TypeError(`Expected ips to be an Array, got ${typeof ips}`);
+        }
+        this.ips = ips;
+        if (isPlainObject(headers)) {
+            this.headers = headers;
+        }
+        if (isPlainObject(internal)) {
+            this.internal = internal;
+        }
+        Object.freeze(this);
     }
-    this.ips = ips;
-
-    if (isPlainObject(headers)) {
-      this.headers = headers;
-    }
-
-    if (isPlainObject(internal)) {
-      this.internal = internal;
-    }
-
-    Object.freeze(this);
-  }
 }
-
 module.exports = ClientConnection;
+//# sourceMappingURL=clientConnection.js.map

package/dist/lib/core/network/context.d.ts

@@ -0,0 +1,42 @@
+export = Context;
+declare class Context {
+    ClientConnection: typeof ClientConnection;
+    debug: any;
+    errors: {
+        BadRequestError: typeof BadRequestError;
+        ExternalServiceError: typeof ExternalServiceError;
+        ForbiddenError: typeof ForbiddenError;
+        GatewayTimeoutError: typeof GatewayTimeoutError;
+        InternalError: typeof KuzzleInternalError;
+        KuzzleError: typeof KuzzleError;
+        NotFoundError: typeof NotFoundError;
+        PartialError: typeof PartialError;
+        PluginImplementationError: typeof PluginImplementationError;
+        PreconditionError: typeof PreconditionError;
+        ServiceUnavailableError: typeof ServiceUnavailableError;
+        SizeLimitError: typeof SizeLimitError;
+        TooManyRequestsError: typeof TooManyRequestsError;
+        UnauthorizedError: typeof UnauthorizedError;
+    };
+    Protocol: typeof ProtocolBase;
+    Request: any;
+    RequestContext: any;
+    RequestInput: any;
+    log: {};
+}
+import ClientConnection = require("./clientConnection");
+import { BadRequestError } from "../../kerror/errors/badRequestError";
+import { ExternalServiceError } from "../../kerror/errors/externalServiceError";
+import { ForbiddenError } from "../../kerror/errors/forbiddenError";
+import { GatewayTimeoutError } from "../../kerror/errors/gatewayTimeoutError";
+import { InternalError as KuzzleInternalError } from "../../kerror/errors/internalError";
+import { KuzzleError } from "../../kerror/errors/kuzzleError";
+import { NotFoundError } from "../../kerror/errors/notFoundError";
+import { PartialError } from "../../kerror/errors/partialError";
+import { PluginImplementationError } from "../../kerror/errors/pluginImplementationError";
+import { PreconditionError } from "../../kerror/errors/preconditionError";
+import { ServiceUnavailableError } from "../../kerror/errors/serviceUnavailableError";
+import { SizeLimitError } from "../../kerror/errors/sizeLimitError";
+import { TooManyRequestsError } from "../../kerror/errors/tooManyRequestsError";
+import { UnauthorizedError } from "../../kerror/errors/unauthorizedError";
+import ProtocolBase = require("./protocols/protocol");

package/dist/lib/core/network/context.js

@@ -0,0 +1,57 @@
+/*
+ * Kuzzle, a backend software, self-hostable and ready to use
+ * to power modern apps
+ *
+ * Copyright 2015-2022 Kuzzle
+ * mailto: support AT kuzzle.io
+ * website: http://kuzzle.io
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+"use strict";
+const { BadRequestError, ExternalServiceError, ForbiddenError, GatewayTimeoutError, InternalError: KuzzleInternalError, KuzzleError, NotFoundError, PartialError, PluginImplementationError, PreconditionError, ServiceUnavailableError, SizeLimitError, TooManyRequestsError, UnauthorizedError, Request, RequestContext, RequestInput, } = require("../../kerror/errors");
+const ClientConnection = require("./clientConnection");
+const ProtocolBase = require("./protocols/protocol");
+const debug = require("../../util/debug")("kuzzle:network:protocols");
+class Context {
+    constructor() {
+        this.ClientConnection = ClientConnection;
+        this.debug = debug;
+        this.errors = {
+            BadRequestError,
+            ExternalServiceError,
+            ForbiddenError,
+            GatewayTimeoutError,
+            InternalError: KuzzleInternalError,
+            KuzzleError,
+            NotFoundError,
+            PartialError,
+            PluginImplementationError,
+            PreconditionError,
+            ServiceUnavailableError,
+            SizeLimitError,
+            TooManyRequestsError,
+            UnauthorizedError,
+        };
+        this.Protocol = ProtocolBase;
+        this.Request = Request;
+        this.RequestContext = RequestContext;
+        this.RequestInput = RequestInput;
+        this.log = {};
+        for (const type of ["silly", "debug", "verbose", "info", "warn", "error"]) {
+            this.log[type] = (...args) => global.kuzzle.log[type](...args);
+        }
+    }
+}
+module.exports = Context;
+//# sourceMappingURL=context.js.map