kuzzle 2.16.8 → 2.17.0

package/lib/util/bufferedPassThrough.js

@@ -0,0 +1,161 @@
+"use strict";
+/*
+ * Kuzzle, a backend software, self-hostable and ready to use
+ * to power modern apps
+ *
+ * Copyright 2015-2022 Kuzzle
+ * mailto: support AT kuzzle.io
+ * website: http://kuzzle.io
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BufferedPassThrough = void 0;
+const stream_1 = __importDefault(require("stream"));
+/**
+ * This streams accumulate chunks data into a buffer until the amount of data is equal or exceed  the buffer size.
+ * Then, it emits a single chunk with the accumulated data.
+ *
+ * This stream is useful when you want to reduce the amount of chunk produced by a stream.
+ * This can helps reducing the number of syscall made by the stream consumer.
+ */
+class BufferedPassThrough extends stream_1.default.Duplex {
+    constructor(options = { highWaterMark: 8196 }) {
+        super(options);
+        this.bufferSize = options.highWaterMark;
+        this.buffer = Buffer.alloc(options.highWaterMark);
+        this.offset = 0;
+    }
+    /**
+     * Writes a string or buffer to the internal buffer starting at the internal buffer offset.
+     * Data will be copied from the given start to the end position.
+     *
+     * @param data
+     * @param start Where to start copying/writing data from
+     * @param end Where to end copying/writing data from
+     * @param encoding Type of encoding to use
+     * @returns How many bytes have been copied / written to the internal buffer.
+     */
+    writeToBuffer(data, start, end, encoding) {
+        if (encoding === 'buffer') {
+            return data.copy(this.buffer, this.offset, start, end);
+        }
+        if (start === 0 && end === data.length) {
+            return this.buffer.write(data, this.offset, end - start, encoding);
+        }
+        return this.buffer.write(data.slice(start, end), this.offset, end - start, encoding);
+    }
+    async writeChunks(chunks) {
+        for (const chunk of chunks) {
+            await this.writeChunkData(chunk.chunk, chunk.encoding);
+        }
+    }
+    /**
+     * Writes a chunk of data to the internal buffer.
+     * If the internal buffer is full, it will be pushed to the stream.
+     * @param chunk
+     * @param encoding
+     */
+    async writeChunkData(chunk, encoding) {
+        let chunkOffset = 0;
+        while (chunkOffset < chunk.length) {
+            const remainingChunkSize = chunk.length - chunkOffset;
+            const remainingBufferSize = this.bufferSize - this.offset;
+            // If the remaining bytes in the chunk exceed the remaining bytes in the internal buffer
+            if (remainingChunkSize >= remainingBufferSize) {
+                // Write as much as the remaining bytes as possible to the buffer  from t
+                chunkOffset += this.writeToBuffer(chunk, chunkOffset, chunkOffset + remainingBufferSize, encoding);
+                // Sends the whole buffer to the stream since it is full
+                if (!this.push(this.buffer)) {
+                    await new Promise(res => {
+                        this.once('_drained', res);
+                    });
+                }
+                this.offset = 0; // Reset the offset
+            }
+            else {
+                // Write the whole chunk to the buffer
+                const size = this.writeToBuffer(chunk, chunkOffset, chunk.length, encoding);
+                // Increase the internal buffer offset
+                this.offset += size;
+                break;
+            }
+        }
+    }
+    /**
+     * @override from stream.Duplex
+     * Internal method called when data needs to be written.
+     * @param chunkData
+     * @param encoding
+     * @param callback
+     */
+    _write(chunkData, encoding, callback) {
+        this.writeChunkData(chunkData, encoding).then(() => {
+            callback();
+        });
+    }
+    /**
+     * @override from stream.Duplex
+     * Internal method called when multiple chunks were stored and needs to be written.
+     * @param chunks
+     * @param callback
+     */
+    _writev(chunks, callback) {
+        this.writeChunks(chunks).then(() => {
+            callback();
+        });
+    }
+    /**
+     * @override from stream.Duplex
+     * Internal method called when the stream is drained, to resume pushing data.
+     * @param size
+     */
+    _read() {
+        // Emits an internal _drained event when the internal buffer is drained.
+        this.emit('_drained');
+    }
+    /**
+     * @override from stream.Duplex
+     * Internal method called when the stream is ended.
+     * @param callback
+     */
+    _final(callback) {
+        if (this.buffer && this.offset > 0) {
+            // Push last bit of data
+            this.push(this.buffer.slice(0, this.offset));
+        }
+        this.push(null); // Close the stream
+        this.buffer = null;
+        callback();
+    }
+    /**
+     * @override from stream.Duplex
+     * Internal method called when stream is destroyed.
+     * @param err
+     * @param callback
+     */
+    _destroy(err, callback) {
+        if (this.buffer && this.offset > 0) {
+            // Push last bit of data
+            this.push(this.buffer.slice(0, this.offset));
+        }
+        this.push(null); // Close the stream
+        this.buffer = null;
+        callback(err);
+    }
+}
+exports.BufferedPassThrough = BufferedPassThrough;
+//# sourceMappingURL=bufferedPassThrough.js.map

package/lib/util/deprecate.js

@@ -35,15 +35,17 @@ const warnIfDeprecated = (logger, deprecations, member) => {
       .find(deprecated => deprecated === member),
     alternative = deprecations[deprecatedProperty];
 
-  if (!deprecatedProperty) {
+  if (! deprecatedProperty) {
     return;
   }
 
   if (alternative && typeof alternative.message === 'string') {
     deprecationWarning(logger, alternative.message);
-  } else if (typeof alternative === 'string') {
+  }
+  else if (typeof alternative === 'string') {
     deprecationWarning(logger, `Use of '${deprecatedProperty}' property is deprecated. Please, use '${alternative}' instead.`);
-  } else {
+  }
+  else {
     deprecationWarning(logger, `Use of '${deprecatedProperty}' property is deprecated.`);
   }
 };
@@ -54,7 +56,7 @@ const deprecateProperties = (logger, target, deprecations = {}) => {
   }
 
   return new Proxy(target, {
-    get(object, key, ...rest) {
+    get (object, key, ...rest) {
       if (key === '__isProxy') {
         return true;
       }
@@ -64,7 +66,7 @@ const deprecateProperties = (logger, target, deprecations = {}) => {
       return Reflect.get(object, key, ...rest);
     },
 
-    set(object, key, ...rest) {
+    set (object, key, ...rest) {
       warnIfDeprecated(logger, deprecations, key);
 
       return Reflect.set(object, key, ...rest);

package/lib/util/didYouMean.js

@@ -23,7 +23,7 @@
 
 const didYouMean = require('didyoumean');
 
-function printDidYouMean(...args) {
+function printDidYouMean (...args) {
   if (global.NODE_ENV !== 'development') {
     return '';
   }

package/lib/util/dump-collection.d.ts

@@ -0,0 +1,3 @@
+import { JSONObject } from '../../index';
+import { HttpStream } from '../types';
+export declare function dumpCollectionDocuments(index: string, collection: string, query?: any, format?: string, fields?: string[], options?: JSONObject): HttpStream;

package/lib/util/dump-collection.js

@@ -0,0 +1,265 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.dumpCollectionDocuments = void 0;
+const ndjson_1 = __importDefault(require("ndjson"));
+const get_1 = __importDefault(require("lodash/get"));
+const isObject_1 = __importDefault(require("lodash/isObject"));
+const kerror_1 = __importDefault(require("../kerror"));
+const bufferedPassThrough_1 = require("./bufferedPassThrough");
+const types_1 = require("../types");
+/**
+ * Flatten an object transform:
+ * {
+ *  title: "kuzzle",
+ *  info : {
+ *    tag: "news"
+ *  }
+ * }
+ *
+ * Into an object like:
+ * {
+ *  title: "kuzzle",
+ *  info.tag: news
+ * }
+ *
+ * @param {Object} target the object we have to flatten
+ * @returns {Object} the flattened object
+ */
+function flattenObject(target) {
+    const output = {};
+    flattenStep(output, target);
+    return output;
+}
+function flattenStep(output, object, prev = null) {
+    const keys = Object.keys(object);
+    for (let i = 0; i < keys.length; i++) {
+        const key = keys[i];
+        const value = object[key];
+        const newKey = prev ? prev + '.' + key : key;
+        if (Object.prototype.toString.call(value) === '[object Object]') {
+            output[newKey] = value;
+            flattenStep(output, value, newKey);
+        }
+        output[newKey] = value;
+    }
+}
+/**
+ * Extract fields from mapping by removing the properties from es mapping
+ *
+ * @param mapping
+ * @returns
+ */
+function extractMappingFields(mapping) {
+    const newMapping = {};
+    for (const key of Object.keys(mapping)) {
+        if (key === 'properties' && (0, isObject_1.default)(mapping[key])) {
+            newMapping[key] = extractMappingFields(mapping[key]);
+        }
+        else if ((0, isObject_1.default)(mapping[key]) && mapping[key].type) {
+            newMapping[key] = mapping[key].type;
+        }
+        else {
+            newMapping[key] = mapping[key];
+        }
+    }
+    return newMapping;
+}
+/**
+ * An iteration-order-safe version of lodash.values
+ *
+ * @param object The object containing the values
+ * @param fields The field names to pick in the right order
+ * @returns The values in the same order as the fields
+ * @see https://lodash.com/docs/4.17.15#values
+ */
+function pickValues(object, fields) {
+    return fields.map(f => formatValueForCSV((0, get_1.default)(object, f)));
+}
+/**
+ * Formats the value for correct CSV output, avoiding to return
+ * values that would badly serialize in CSV.
+ *
+ * @param value The value to format
+ * @returns The value or a string telling the value is not scalar
+ */
+function formatValueForCSV(value) {
+    if ((0, isObject_1.default)(value)) {
+        return '[OBJECT]';
+    }
+    return value;
+}
+class AbstractDumper {
+    constructor(index, collection, query = {}, writeStream, options = {
+        fieldsName: {},
+        scroll: '5s',
+        separator: ',',
+        size: 10,
+    }) {
+        this.index = index;
+        this.collection = collection;
+        this.query = query;
+        this.writeStream = writeStream;
+        this.options = options;
+        if (!writeStream) {
+            throw kerror_1.default.get('api', 'assert', 'missing_argument', 'writeStream');
+        }
+    }
+    /**
+     * One-shot call before the dump. Can be used to
+     * perform setup operations before dumping.
+     *
+     * @returns void
+     */
+    // eslint-disable-next-line @typescript-eslint/no-empty-function
+    async setup() { }
+    /**
+     * One-shot call before iterating over the data. Can be
+     * used to write the header of the dumped output.
+     */
+    // eslint-disable-next-line @typescript-eslint/no-empty-function
+    async writeHeader() { }
+    // eslint-disable-next-line @typescript-eslint/no-empty-function
+    async tearDown() { }
+    async scroll(scrollId) {
+        if (!scrollId) {
+            return null;
+        }
+        try {
+            return await global.kuzzle.ask('core:storage:public:document:scroll', scrollId, { scrollTTL: this.options.scroll });
+        }
+        catch {
+            return null;
+        }
+    }
+    /**
+     * The loop that iterates over the documents of the collection and
+     * calls all the other hooks.
+     *
+     * @returns a promise resolving when the dump is finished.
+     */
+    async dump() {
+        const waitWrite = new Promise((resolve, reject) => this.writeStream ? this.writeStream.on('finish', resolve) : reject());
+        this.writeStream.on('error', error => {
+            throw error;
+        });
+        await this.setup();
+        await this.writeHeader();
+        let results = await global.kuzzle.ask('core:storage:public:document:search', this.index, this.collection, this.query, {
+            lang: this.options.lang,
+            scroll: this.options.scroll,
+            size: this.options.size,
+        });
+        do {
+            for (const hit of results.hits) {
+                await this.onResult({
+                    _id: hit._id,
+                    _source: hit._source
+                });
+            }
+        } while ((results = await this.scroll(results.scrollId)));
+        await this.tearDown();
+        this.writeStream.end();
+        return waitWrite;
+    }
+}
+class JSONLDumper extends AbstractDumper {
+    constructor() {
+        super(...arguments);
+        this.ndjsonStream = ndjson_1.default.stringify();
+    }
+    async setup() {
+        this.ndjsonStream.on('data', (line) => {
+            this.writeStream.write(line);
+        });
+    }
+    async writeHeader() {
+        await this.writeLine({
+            collection: this.collection,
+            index: this.index,
+            type: 'collection',
+        });
+    }
+    writeLine(content) {
+        return new Promise(resolve => {
+            if (this.ndjsonStream.write(content)) {
+                resolve();
+            }
+            else {
+                this.ndjsonStream.once('drain', resolve);
+            }
+        });
+    }
+    onResult(document) {
+        return this.writeLine({
+            _id: document._id,
+            body: document._source,
+        });
+    }
+    get fileExtension() {
+        return 'jsonl';
+    }
+}
+class CSVDumper extends AbstractDumper {
+    constructor(index, collection, query = {}, writeStream, options, fields) {
+        super(index, collection, query, writeStream, options);
+        this.fields = fields;
+    }
+    get fileExtension() {
+        return 'csv';
+    }
+    async setup() {
+        if (!this.fields.length) {
+            // If no field has been selected, then all fields are selected.
+            const mappings = await global.kuzzle.ask('core:storage:public:mappings:get', this.index, this.collection);
+            if (!mappings.properties) {
+                return;
+            }
+            this.fields = Object.keys(flattenObject(extractMappingFields(mappings.properties)));
+        }
+        else if (this.fields.includes('_id')) {
+            // Delete '_id' from the selected fields, since IDs are
+            // _always_ exported.
+            this.fields.splice(this.fields.indexOf('_id'), 1);
+        }
+    }
+    writeHeader() {
+        const mappedFieldsName = ['_id', ...this.fields].map(field => {
+            return this.options.fieldsName[field] || field;
+        });
+        return this.writeLine(mappedFieldsName.join(this.options.separator));
+    }
+    writeLine(content) {
+        return new Promise(resolve => {
+            if (this.writeStream.write(`${content}\n`)) {
+                resolve();
+            }
+            else {
+                this.writeStream.once('drain', resolve);
+            }
+        });
+    }
+    onResult(document) {
+        const values = [document._id, ...pickValues(document._source, this.fields)];
+        return this.writeLine(values.join(this.options.separator));
+    }
+}
+function dumpCollectionDocuments(index, collection, query = {}, format = 'jsonl', fields = [], options = {}) {
+    let dumper;
+    const writableStream = new bufferedPassThrough_1.BufferedPassThrough({ highWaterMark: 16384 });
+    switch (format.toLowerCase()) {
+        case 'csv':
+            dumper = new CSVDumper(index, collection, query, writableStream, options, fields);
+            dumper.dump();
+            break;
+        default:
+            dumper = new JSONLDumper(index, collection, query, writableStream, options);
+            dumper.dump();
+            break;
+    }
+    return new types_1.HttpStream(writableStream);
+}
+exports.dumpCollectionDocuments = dumpCollectionDocuments;
+//# sourceMappingURL=dump-collection.js.map

package/lib/util/extractFields.js

@@ -32,7 +32,7 @@ const { isPlainObject } = require('./safeObject');
 *
 * @returns { Array<String> | Array<{ key: String, value: any }> }
 */
-function extractFields(document, {
+function extractFields (document, {
   fieldsToIgnore = [],
   alsoExtractValues = false,
 } = {}, {
@@ -54,7 +54,7 @@ function extractFields(document, {
         { extractedFields, path: currentPath });
     }
     else if (alsoExtractValues) {
-      extractedFields.push({key: currentPath, value});
+      extractedFields.push({ key: currentPath, value });
     }
     else {
       extractedFields.push(currentPath);

package/lib/util/inflector.d.ts

@@ -14,4 +14,12 @@ export declare class Inflector {
      * @param string String to transform the first letter in uppercase
      */
     static upFirst(string: string): string;
+    /**
+     * Converts a string to PascalCase
+     */
+    static pascalCase(string: string): string;
+    /**
+     * Converts a string to camelCase
+     */
+    static camelCase(string: any): string;
 }

package/lib/util/inflector.js

@@ -19,8 +19,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Inflector = void 0;
+const lodash_1 = __importDefault(require("lodash"));
 class Inflector {
     /**
      * Converts a string to kebab-case
@@ -46,6 +50,18 @@ class Inflector {
     static upFirst(string) {
         return string.charAt(0).toUpperCase() + string.slice(1);
     }
+    /**
+     * Converts a string to PascalCase
+     */
+    static pascalCase(string) {
+        return lodash_1.default.startCase(string).replace(/ /g, '');
+    }
+    /**
+     * Converts a string to camelCase
+     */
+    static camelCase(string) {
+        return lodash_1.default.camelCase(string);
+    }
 }
 exports.Inflector = Inflector;
 //# sourceMappingURL=inflector.js.map

package/lib/util/requestAssertions.js

@@ -44,7 +44,7 @@ module.exports = {
    * @param {string} attribute
    */
   assertBodyAttributeType: (request, attribute, type) => {
-    switch(type) {
+    switch (type) {
       case 'number':
       case 'boolean':
       case 'string':
@@ -86,7 +86,7 @@ module.exports = {
    * @param {string} attribute
    */
   assertBodyHasNotAttribute: (request, attribute) => {
-    if (!_.isNil(get(request.input.body, attribute))) {
+    if (! _.isNil(get(request.input.body, attribute))) {
       throw assertionError.get('forbidden_argument', `body.${attribute}`);
     }
   },
@@ -108,7 +108,7 @@ module.exports = {
    * @param {Request} request
    */
   assertHasId: request => {
-    if (!request.input.args._id) {
+    if (! request.input.args._id) {
       throw assertionError.get('missing_argument', '_id');
     }
   },
@@ -117,7 +117,7 @@ module.exports = {
    * @param {Request} request
    */
   assertHasIndex: request => {
-    if (!request.input.args.index) {
+    if (! request.input.args.index) {
       throw assertionError.get('missing_argument', 'index');
     }
   },
@@ -126,11 +126,11 @@ module.exports = {
    * @param {Request} request
    */
   assertHasIndexAndCollection: request => {
-    if (!request.input.args.index) {
+    if (! request.input.args.index) {
       throw assertionError.get('missing_argument', 'index');
     }
 
-    if (!request.input.args.collection) {
+    if (! request.input.args.collection) {
       throw assertionError.get('missing_argument', 'collection');
     }
   },
@@ -150,7 +150,7 @@ module.exports = {
    * @param {Request} request
    */
   assertIsObject: value => {
-    if (!isPlainObject(value)) {
+    if (! isPlainObject(value)) {
       throw assertionError.get('invalid_argument', value, 'object');
     }
   }

package/lib/util/wildcard.js

@@ -0,0 +1,55 @@
+/*
+ * Kuzzle, a backend software, self-hostable and ready to use
+ * to power modern apps
+ *
+ * Copyright 2015-2020 Kuzzle
+ * mailto: support AT kuzzle.io
+ * website: http://kuzzle.io
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+const _ = require('lodash');
+
+function match (pattern, list) {
+  // Match everything
+  if (pattern === '*') {
+    return list;
+  }
+  /**
+   * Reduces repeating "*" to one ".*"
+   * One of the fastest and most readable way to do this
+   */
+  const wildCardPattern = pattern.split('*')
+    .filter((patternPart, index, array) => 
+      patternPart !== ''
+      || index === 0
+      || index === array.length - 1
+    )
+    .map(patternPart => _.escapeRegExp(patternPart)) // escape special regex characters
+    .join('.*');
+  
+  // Match everything
+  if (wildCardPattern === '.*') {
+    return list;
+  }
+  
+  const regex = new RegExp(`^${wildCardPattern}$`);
+
+  // Keep only matching elements
+  return list.filter(item => ! regex.test(item));
+}
+
+module.exports = { match };