kourou 0.19.2 → 0.20.0

package/lib/hooks/command_not_found/api-action.js

@@ -1,8 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const tslib_1 = require("tslib");
-const chalk_1 = tslib_1.__importDefault(require("chalk"));
-const query_1 = tslib_1.__importDefault(require("../../commands/sdk/query"));
+const chalk_1 = (0, tslib_1.__importDefault)(require("chalk"));
+const query_1 = (0, tslib_1.__importDefault)(require("../../commands/sdk/query"));
 /**
  * Hooks that run the corresponding API action with sdk:query.
  *

package/lib/support/dump-collection.d.ts

@@ -1,2 +1,2 @@
-export declare function dumpCollectionData(sdk: any, index: string, collection: string, batchSize: number, destPath: string, query?: any, format?: string): Promise<unknown>;
+export declare function dumpCollectionData(sdk: any, index: string, collection: string, batchSize: number, destPath: string, query?: any, format?: string, fields?: string[]): Promise<void>;
 export declare function dumpCollectionMappings(sdk: any, index: string, collection: string, destPath: string, format?: string): Promise<void>;

package/lib/support/dump-collection.js

@@ -2,76 +2,226 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.dumpCollectionMappings = exports.dumpCollectionData = void 0;
 const tslib_1 = require("tslib");
-const fs_1 = tslib_1.__importDefault(require("fs"));
-const path_1 = tslib_1.__importDefault(require("path"));
-const cli_ux_1 = tslib_1.__importDefault(require("cli-ux"));
-const ndjson_1 = tslib_1.__importDefault(require("ndjson"));
-async function dumpCollectionData(sdk, index, collection, batchSize, destPath, query = {}, format = 'jsonl') {
-    const collectionDir = path_1.default.join(destPath, collection);
-    const filename = path_1.default.join(collectionDir, (format.toLowerCase() === 'jsonl' ? 'documents.jsonl' : 'documents.json'));
-    const writeStream = fs_1.default.createWriteStream(filename);
-    const waitWrite = new Promise(resolve => writeStream.on('finish', resolve));
-    const ndjsonStream = ndjson_1.default.stringify();
-    const options = {
-        scroll: '1m',
-        size: batchSize
-    };
-    const writeLine = (content) => {
+const fs_1 = (0, tslib_1.__importDefault)(require("fs"));
+const path_1 = (0, tslib_1.__importDefault)(require("path"));
+const cli_ux_1 = (0, tslib_1.__importDefault)(require("cli-ux"));
+const ndjson_1 = (0, tslib_1.__importDefault)(require("ndjson"));
+const common_1 = require("../common");
+/**
+ * Flatten an object transform:
+ * {
+ *  title: "kuzzle",
+ *  info : {
+ *    tag: "news"
+ *  }
+ * }
+ *
+ * Into an object like:
+ * {
+ *  title: "kuzzle",
+ *  info.tag: news
+ * }
+ *
+ * @param {Object} target the object we have to flatten
+ * @returns {Object} the flattened object
+ */
+function flattenObject(target) {
+    const output = {};
+    flattenStep(output, target);
+    return output;
+}
+function flattenStep(output, object, prev = null) {
+    const keys = Object.keys(object);
+    for (let i = 0; i < keys.length; i++) {
+        const key = keys[i];
+        const value = object[key];
+        const newKey = prev ? prev + '.' + key : key;
+        if (Object.prototype.toString.call(value) === '[object Object]') {
+            output[newKey] = value;
+            flattenStep(output, value, newKey);
+        }
+        output[newKey] = value;
+    }
+}
+class AbstractDumper {
+    constructor(sdk, index, collection, batchSize, destPath, query = {}) {
+        this.sdk = sdk;
+        this.index = index;
+        this.collection = collection;
+        this.batchSize = batchSize;
+        this.destPath = destPath;
+        this.query = query;
+        this.collectionDir = path_1.default.join(this.destPath, this.collection);
+        this.options = {
+            scroll: '2s',
+            size: batchSize
+        };
+    }
+    /**
+     * One-shot call before the dump. Can be used to
+     * perform setup operations before dumping.
+     *
+     * @returns void
+     */
+    // eslint-disable-next-line @typescript-eslint/no-empty-function
+    async setup() { }
+    /**
+     * One-shot call before iterating over the data. Can be
+     * used to write the header of the dumped output.
+     */
+    // eslint-disable-next-line @typescript-eslint/no-empty-function
+    async writeHeader() { }
+    // eslint-disable-next-line @typescript-eslint/no-empty-function
+    async tearDown() { }
+    /**
+     * The loop that iterates over the documents of the collection and
+     * calls all the other hooks.
+     *
+     * @returns a promise resolving when the dump is finished.
+     */
+    async dump() {
+        this.filename = path_1.default.join(this.collectionDir, `documents.${this.fileExtension}`);
+        this.writeStream = fs_1.default.createWriteStream(this.filename);
+        const waitWrite = new Promise((resolve, reject) => this.writeStream ? this.writeStream.on('finish', resolve) : reject());
+        this.writeStream.on('error', error => {
+            throw error;
+        });
+        await this.setup();
+        await this.writeHeader();
+        let results = await this.sdk.document.search(this.index, this.collection, { query: this.query }, this.options);
+        const progressBar = cli_ux_1.default.progress({
+            format: `Dumping ${this.collection} |{bar}| {percentage}% || {value}/{total} documents`
+        });
+        progressBar.start(results.total, 0);
+        do {
+            progressBar.update(results.fetched);
+            for (const hit of results.hits) {
+                await this.onResult({
+                    _id: hit._id,
+                    _source: hit._source
+                });
+            }
+        } while ((results = await results.next()));
+        await this.tearDown();
+        progressBar.stop();
+        this.writeStream.end();
+        return waitWrite;
+    }
+}
+class JSONLDumper extends AbstractDumper {
+    constructor() {
+        super(...arguments);
+        this.ndjsonStream = ndjson_1.default.stringify();
+    }
+    async setup() {
+        this.ndjsonStream.on('data', (line) => {
+            if (!this.writeStream) {
+                throw new Error('Cannot write data: WriteStream is not initialized.');
+            }
+            this.writeStream.write(line);
+        });
+        return;
+    }
+    async writeHeader() {
+        await this.writeLine({ type: 'collection', index: this.index, collection: this.collection });
+    }
+    writeLine(content) {
         return new Promise(resolve => {
-            if (ndjsonStream.write(content)) {
+            if (this.ndjsonStream.write(content)) {
                 resolve(undefined);
             }
             else {
-                ndjsonStream.once('drain', resolve);
+                this.ndjsonStream.once('drain', resolve);
             }
         });
-    };
-    writeStream.on('error', error => {
-        throw error;
-    });
-    ndjsonStream.on('data', (line) => writeStream.write(line));
-    fs_1.default.mkdirSync(collectionDir, { recursive: true });
-    if (format.toLowerCase() === 'jsonl') {
-        await writeLine({ type: 'collection', index, collection });
-    }
-    let results = await sdk.document.search(index, collection, { query }, options);
-    const progressBar = cli_ux_1.default.progress({
-        format: `Dumping ${collection} |{bar}| {percentage}% || {value}/{total} documents`
-    });
-    progressBar.start(results.total, 0);
-    const rawDocuments = {
-        [index]: {
-            [collection]: []
-        }
-    };
-    do {
-        progressBar.update(results.fetched);
-        for (const hit of results.hits) {
-            let document = null;
-            if (format.toLocaleLowerCase() === 'jsonl') {
-                document = {
-                    _id: hit._id,
-                    body: hit._source
-                };
-                await writeLine(document);
+    }
+    onResult(document) {
+        return this.writeLine({ _id: document._id, body: document._source });
+    }
+    get fileExtension() {
+        return 'jsonl';
+    }
+}
+class KuzzleDumper extends JSONLDumper {
+    constructor() {
+        super(...arguments);
+        this.rawDocuments = {
+            [this.index]: {
+                [this.collection]: []
             }
-            else {
-                rawDocuments[index][collection].push({
-                    index: {
-                        _id: hit._id
-                    }
-                });
-                rawDocuments[index][collection].push(hit._source);
+        };
+    }
+    get fileExtension() {
+        return 'json';
+    }
+    async onResult(document) {
+        this.rawDocuments[this.index][this.collection].push({
+            index: {
+                _id: document._id
+            }
+        });
+        this.rawDocuments[this.index][this.collection].push(document._source);
+        return;
+    }
+    tearDown() {
+        return this.writeLine(this.rawDocuments);
+    }
+}
+class CSVDumper extends AbstractDumper {
+    constructor(sdk, index, collection, batchSize, destPath, query = {}, fields, separator = ',') {
+        super(sdk, index, collection, batchSize, destPath, query);
+        this.fields = fields;
+        this.separator = separator;
+    }
+    get fileExtension() {
+        return 'csv';
+    }
+    async setup() {
+        if (!this.fields.length) {
+            // If no field has been selected, then all fields are selected.
+            const mappings = await this.sdk.collection.getMapping(this.index, this.collection);
+            if (!mappings.properties) {
+                return;
             }
+            this.fields = Object.keys(flattenObject(mappings.properties));
+        }
+        else {
+            // Delete '_id' from the selected fields, since IDs are
+            // _always_ exported.
+            if (this.fields.includes('_id')) {
+                this.fields.splice(this.fields.indexOf('_id'), 1);
+            }
+        }
+    }
+    writeHeader() {
+        return this.writeLine(['_id', ...this.fields].join(this.separator));
+    }
+    async writeLine(line) {
+        if (!this.writeStream) {
+            throw new Error('Cannot write data: WriteStream is not initialized.');
         }
-    } while ((results = await results.next()));
-    if (format.toLocaleLowerCase() === 'kuzzle') {
-        await writeLine(rawDocuments);
-    }
-    progressBar.stop();
-    ndjsonStream.end();
-    writeStream.end();
-    return waitWrite;
+        this.writeStream.write(line);
+        this.writeStream.write('\n');
+        return;
+    }
+    onResult(document) {
+        const values = [document._id, ...(0, common_1.pickValues)(document._source, this.fields)];
+        return this.writeLine(values.join(this.separator));
+    }
+}
+async function dumpCollectionData(sdk, index, collection, batchSize, destPath, query = {}, format = 'jsonl', fields = []) {
+    let dumper;
+    switch (format.toLowerCase()) {
+        case 'jsonl':
+            dumper = new JSONLDumper(sdk, index, collection, batchSize, destPath, query);
+            return dumper.dump();
+        case 'csv':
+            dumper = new CSVDumper(sdk, index, collection, batchSize, destPath, query, fields);
+            return dumper.dump();
+        default:
+            dumper = new KuzzleDumper(sdk, index, collection, batchSize, destPath, query);
+            return dumper.dump();
+    }
 }
 exports.dumpCollectionData = dumpCollectionData;
 async function dumpCollectionMappings(sdk, index, collection, destPath, format = 'jsonl') {

package/lib/support/editor.js

@@ -2,9 +2,9 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Editor = void 0;
 const tslib_1 = require("tslib");
-const fs_1 = tslib_1.__importDefault(require("fs"));
+const fs_1 = (0, tslib_1.__importDefault)(require("fs"));
 const child_process_1 = require("child_process");
-const tmp_1 = tslib_1.__importDefault(require("tmp"));
+const tmp_1 = (0, tslib_1.__importDefault)(require("tmp"));
 class Editor {
     constructor(content = '', options = {}) {
         this._content = content;
@@ -29,7 +29,7 @@ class Editor {
     run() {
         process.env.EDITOR = process.env.EDITOR || this.defaultEditor;
         const editor = process.env.EDITOR;
-        const response = child_process_1.spawnSync(editor, [this.tmpFile.name], { stdio: 'inherit' });
+        const response = (0, child_process_1.spawnSync)(editor, [this.tmpFile.name], { stdio: 'inherit' });
         if (response.status !== 0) {
             // eslint-disable-next-line
             console.error(response.stdout);

package/lib/support/execute.js

@@ -26,7 +26,7 @@ function execute(...args) {
         options = args.splice(args.length - 1)[0];
     }
     const [command, ...commandArgs] = args;
-    const process = child_process_1.spawn(command, commandArgs, options);
+    const process = (0, child_process_1.spawn)(command, commandArgs, options);
     let stdout = '';
     let stderr = '';
     // eslint-disable-next-line

package/lib/support/kuzzle.d.ts

@@ -40,7 +40,7 @@ export declare class KuzzleSDK {
     get document(): import("kuzzle-sdk").DocumentController;
     get collection(): import("kuzzle-sdk").CollectionController;
     get index(): import("kuzzle-sdk").IndexController;
-    get security(): any;
+    get security(): import("kuzzle-sdk/src/controllers/Security").SecurityController;
     get auth(): import("kuzzle-sdk").AuthController;
     get server(): any;
     get realtime(): import("kuzzle-sdk").RealtimeController;

package/lib/support/kuzzle.js

@@ -69,6 +69,7 @@ class KuzzleSDK {
         this.sdk = new kuzzle_sdk_1.Kuzzle(new ProtocolClass(this.host, {
             port: this.port,
             sslConnection: this.ssl,
+            pingInterval: 5000,
         }));
         this.sdk.volatile = {
             client: `${this.appName}@${this.appVersion}`

package/lib/support/restore-collection.js

@@ -2,14 +2,14 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.restoreCollectionMappings = exports.restoreCollectionData = void 0;
 const tslib_1 = require("tslib");
-const chalk_1 = tslib_1.__importDefault(require("chalk"));
-const fs_1 = tslib_1.__importDefault(require("fs"));
-const ndjson = require("ndjson");
+const chalk_1 = (0, tslib_1.__importDefault)(require("chalk"));
+const fs_1 = (0, tslib_1.__importDefault)(require("fs"));
+const ndjson_1 = (0, tslib_1.__importDefault)(require("ndjson"));
 function handleError(log, dumpFile, error) {
     if (error.status === 206) {
         const errorFile = `${dumpFile.split('.').slice(0, -1).join('.')}-errors.jsonl`;
         const writeStream = fs_1.default.createWriteStream(errorFile, { flags: 'a' });
-        const serialize = ndjson.stringify().pipe(writeStream);
+        const serialize = ndjson_1.default.stringify().pipe(writeStream);
         serialize.on('data', (line) => (writeStream.write(line)));
         for (const partialError of error.errors) {
             serialize.write(partialError);
@@ -37,7 +37,7 @@ async function restoreCollectionData(sdk, log, batchSize, dumpFile, index, colle
         let headerSkipped = false;
         let documents = [];
         const readStream = fs_1.default.createReadStream(dumpFile)
-            .pipe(ndjson.parse())
+            .pipe(ndjson_1.default.parse())
             .on('data', (obj) => {
             if (headerSkipped) {
                 documents.push(obj);

package/lib/support/restore-securities.js

@@ -2,8 +2,8 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.restoreUsers = exports.restoreProfiles = exports.restoreRoles = void 0;
 const tslib_1 = require("tslib");
-const lodash_1 = tslib_1.__importDefault(require("lodash"));
-const bluebird_1 = tslib_1.__importDefault(require("bluebird"));
+const lodash_1 = (0, tslib_1.__importDefault)(require("lodash"));
+const bluebird_1 = (0, tslib_1.__importDefault)(require("bluebird"));
 const sleep = (seconds) => new Promise((resolve) => setTimeout(resolve, seconds * 1000));
 async function restoreRoles(kommand, dump, preserveAnonymous = false) {
     if (dump.type !== 'roles') {