kourou 0.25.0 → 0.25.2

package/README.md

@@ -26,7 +26,7 @@ $ npm install -g kourou
 $ kourou COMMAND
 running command...
 $ kourou (-v|--version|version)
-kourou/0.25.0 linux-x64 node-v16.15.0
+kourou/0.25.2 linux-x64 node-v14.20.0
 $ kourou --help [COMMAND]
 USAGE
   $ kourou COMMAND
@@ -365,7 +365,7 @@ ARGUMENTS
   DESTINATION  Directory to scaffold the app
 
 OPTIONS
-  --flavor=flavor  [default: generic] Template flavor ("generic", "iot-platform", "iot-console", "iot-platform").
+  --flavor=flavor  [default: generic] Template flavor ("generic", "iot-backend", "iot-console").
                    Those can be found here: https://github.com/kuzzleio/project-templates
 
   --help           show CLI help
@@ -506,6 +506,9 @@ OPTIONS
   --ssl
       Use SSL to connect to Kuzzle
 
+  --type=type
+      [default: all] Type of the export: all, mappings, data
+
   --username=username
       [default: anonymous] Kuzzle username (local strategy)
 
@@ -547,7 +550,7 @@ _See code: [src/commands/collection/import.ts](src/commands/collection/import.ts
 
 ## `kourou collection:migrate SCRIPT PATH`
 
-Migrate a collection
+Migrate a collection by transforming documents from a dump file and importing them into Kuzzle
 
 ```
 USAGE
@@ -938,6 +941,8 @@ OPTIONS
 
   --ssl                    Use SSL to connect to Kuzzle
 
+  --type=type              [default: all] Type of the export: all, mappings, data
+
   --username=username      [default: anonymous] Kuzzle username (local strategy)
 
 EXAMPLES

package/lib/commands/app/scaffold.js

@@ -35,7 +35,7 @@ AppScaffold.flags = {
     help: command_1.flags.help(),
     flavor: command_1.flags.string({
         default: "generic",
-        description: `Template flavor ("generic", "iot-platform", "iot-console", "iot-platform").
+        description: `Template flavor ("generic", "iot-backend", "iot-console").
     Those can be found here: https://github.com/kuzzleio/project-templates`,
     }),
 };

package/lib/commands/collection/export.d.ts

@@ -20,6 +20,7 @@ export default class CollectionExport extends Kommand {
         format: flags.IOptionFlag<string>;
         fields: flags.IOptionFlag<string | undefined>;
         scrollTTL: flags.IOptionFlag<string>;
+        type: flags.IOptionFlag<string>;
     };
     static args: {
         name: string;

package/lib/commands/collection/export.js

@@ -22,10 +22,16 @@ class CollectionExport extends common_1.Kommand {
         }
         const countAll = await this.sdk.document.count(this.args.index, this.args.collection);
         const count = await this.sdk.document.count(this.args.index, this.args.collection, { query });
-        this.logInfo(`Dumping ${count} of ${countAll} documents from collection "${this.args.index}:${this.args.collection}" in ${exportPath} ...`);
+        this.logInfo(`Dumping collection "${this.args.index}:${this.args.collection}" in ${exportPath} ...`);
         fs_1.default.mkdirSync(exportPath, { recursive: true });
-        await (0, dump_collection_1.dumpCollectionMappings)(this.sdk, this.args.index, this.args.collection, exportPath, this.flags.format);
-        await (0, dump_collection_1.dumpCollectionData)(this.sdk, this.args.index, this.args.collection, Number(this.flags["batch-size"]), exportPath, query, this.flags.format, fields, this.flags.scrollTTL);
+        if (this.flags.type === "all" || this.flags.type === "mappings") {
+            this.logInfo(`Dumping collection mappings ...`);
+            await (0, dump_collection_1.dumpCollectionMappings)(this.sdk, this.args.index, this.args.collection, exportPath, this.flags.format);
+        }
+        if (this.flags.type === "all" || this.flags.type === "data") {
+            this.logInfo(`Dumping collection ${count} of ${countAll} documents ...`);
+            await (0, dump_collection_1.dumpCollectionData)(this.sdk, this.args.index, this.args.collection, Number(this.flags["batch-size"]), exportPath, query, this.flags.format, fields, this.flags.scrollTTL);
+        }
         this.logOk(`Collection ${this.args.index}:${this.args.collection} dumped`);
     }
 }
@@ -60,6 +66,9 @@ exportable fields in the mapping will be exported.`,
         description: `The scroll TTL option to pass to the dump operation (which performs a document.search under the hood),
 expressed in ms format, e.g. '2s', '1m', '3h'.`,
         default: "20s",
+    }), type: command_1.flags.string({
+        description: "Type of the export: all, mappings, data",
+        default: "all",
     }) }, kuzzle_1.kuzzleFlags), { protocol: command_1.flags.string({
         description: "Kuzzle protocol (http or websocket)",
         default: "ws",

package/lib/commands/collection/migrate.js

@@ -25,7 +25,7 @@ class CollectionMigrate extends common_1.Kommand {
 }
 exports.default = CollectionMigrate;
 CollectionMigrate.keepAuth = true;
-CollectionMigrate.description = "Migrate a collection";
+CollectionMigrate.description = "Migrate a collection by transforming documents from a dump file and importing them into Kuzzle";
 CollectionMigrate.flags = Object.assign(Object.assign({ help: command_1.flags.help({}), "batch-size": command_1.flags.string({
         description: "Maximum batch size (see limits.documentsWriteCount config)",
         default: "200",

package/lib/commands/index/export.d.ts

@@ -5,6 +5,7 @@ export default class IndexExport extends Kommand {
     static description: string;
     static flags: {
         protocol: flags.IOptionFlag<string>;
+        type: flags.IOptionFlag<string>;
         host: flags.IOptionFlag<string>;
         port: flags.IOptionFlag<string>;
         ssl: import("@oclif/parser/lib/flags").IBooleanFlag<boolean>;

package/lib/commands/index/export.js

@@ -19,11 +19,18 @@ class IndexExport extends common_1.Kommand {
         const { collections } = await this.sdk.collection.list(this.args.index);
         for (const collection of collections) {
             try {
-                if (collection.type !== "realtime") {
+                if (collection.type === "realtime") {
+                    continue;
+                }
+                if (this.flags.type === "all" || this.flags.type === "mappings") {
+                    this.logInfo(`Dumping collection "${collection.name}" mappings ...`);
                     await (0, dump_collection_1.dumpCollectionMappings)(this.sdk, this.args.index, collection.name, exportPath, this.flags.format);
+                }
+                if (this.flags.type === "all" || this.flags.type === "data") {
+                    this.logInfo(`Dumping collection "${collection.name}" documents ...`);
                     await (0, dump_collection_1.dumpCollectionData)(this.sdk, this.args.index, collection.name, Number(this.flags["batch-size"]), exportPath, query, this.flags.format, this.flags.scrollTTL);
-                    cli_ux_1.default.action.stop();
                 }
+                cli_ux_1.default.action.stop();
             }
             catch (error) {
                 this.logKo(`Error when exporting collection "${collection.name}": ${error}`);
@@ -53,6 +60,9 @@ expressed in ms format, e.g. '2s', '1m', '3h'.`,
     }) }, kuzzle_1.kuzzleFlags), { protocol: command_1.flags.string({
         description: "Kuzzle protocol (http or websocket)",
         default: "ws",
+    }), type: command_1.flags.string({
+        description: "Type of the export: all, mappings, data",
+        default: "all",
     }) });
 IndexExport.args = [{ name: "index", description: "Index name", required: true }];
 IndexExport.examples = [

package/lib/commands/paas/login.js

@@ -20,10 +20,6 @@ class PaasLogin extends PaasKommand_1.PaasKommand {
             await this.authenticateNPM(username, password);
             return;
         }
-        if (this.flags.only_npm) {
-            await this.authenticateNPM(username, password);
-            return;
-        }
         await this.initPaasClient({ username, password });
         const apiKey = await this.paas.auth.createApiKey("Kourou PaaS API Key");
         this.createProjectCredentials(apiKey);
@@ -53,15 +49,19 @@ class PaasLogin extends PaasKommand_1.PaasKommand {
                 password,
             }),
         };
-        const response = await (0, node_fetch_1.default)(`https://packages.paas.kuzzle.io/-/user/org.couchdb.user:${username}`, options);
-        const { token } = await response.json();
+        const response = await (0, node_fetch_1.default)(`https://${this.packagesHost}/-/user/org.couchdb.user:${username}`, options);
+        const json = await response.json();
+        if (response.status !== 201) {
+            throw new Error(json.error);
+        }
+        const { token } = json;
         (0, child_process_1.spawnSync)("npm", [
             "config",
             "set",
             "@kuzzleio:registry",
-            "https://packages.paas.kuzzle.io",
+            `https://${this.packagesHost}`,
         ], { stdio: "inherit" });
-        (0, child_process_1.spawnSync)("npm", ["set", "//packages.paas.kuzzle.io/:_authToken", token], {
+        (0, child_process_1.spawnSync)("npm", ["set", `//${this.packagesHost}/:_authToken`, token], {
             stdio: "inherit",
         });
     }

package/lib/commands/paas/logs.js

@@ -15,7 +15,15 @@ class PaasLogs extends PaasKommand_1.PaasKommand {
         /**
          * Allowed colors for pod names.
          */
-        this.allColors = [chalk_1.default.red, chalk_1.default.green, chalk_1.default.yellow, chalk_1.default.blue, chalk_1.default.magenta, chalk_1.default.cyan, chalk_1.default.gray];
+        this.allColors = [
+            chalk_1.default.red,
+            chalk_1.default.green,
+            chalk_1.default.yellow,
+            chalk_1.default.blue,
+            chalk_1.default.magenta,
+            chalk_1.default.cyan,
+            chalk_1.default.gray,
+        ];
         /**
          * Available colors for pod names.
          */
@@ -34,8 +42,12 @@ class PaasLogs extends PaasKommand_1.PaasKommand {
         this.logInfo(`Logged as "${user._id}" for project "${this.flags.project || this.getProject()}"`);
         const separator = "\t";
         // Parse the time arguments
-        const since = this.flags.since ? chrono.parseDate(this.flags.since).toISOString() : undefined;
-        const until = this.flags.until ? chrono.parseDate(this.flags.until).toISOString() : undefined;
+        const since = this.flags.since
+            ? chrono.parseDate(this.flags.since).toISOString()
+            : undefined;
+        const until = this.flags.until
+            ? chrono.parseDate(this.flags.until).toISOString()
+            : undefined;
         // Perform the streamed request
         const incomingMessage = await this.paas.queryHttpStream({
             controller: "application",
@@ -59,18 +71,24 @@ class PaasLogs extends PaasKommand_1.PaasKommand {
             // Display the response
             for (var lineStream_1 = (0, tslib_1.__asyncValues)(lineStream), lineStream_1_1; lineStream_1_1 = await lineStream_1.next(), !lineStream_1_1.done;) {
                 const line = lineStream_1_1.value;
-                // Parse the data
-                const data = JSON.parse(line);
-                // Exclude logs that are empty or that are not from a pod
-                if (!data.content || !data.podName) {
-                    continue;
+                try {
+                    const data = JSON.parse(line);
+                    // Exclude logs that are empty or that are not from a pod
+                    if (!data.content || !data.podName) {
+                        continue;
+                    }
+                    // Get the pod color
+                    const podColor = this.getPodColor(data.podName);
+                    // Display the log
+                    const timestamp = this.flags.timestamp
+                        ? `[${new Date(data.timeStamp).toLocaleString()}] `
+                        : "";
+                    const name = podColor(`${data.podName}${separator}`);
+                    this.log(`${timestamp}${name}| ${data.content}`);
+                }
+                catch (error) {
+                    this.logKo(`Error while parsing log: ${error} (Received: "${line}")`);
                 }
-                // Get the pod color
-                const podColor = this.getPodColor(data.podName);
-                // Display the log
-                const timestamp = this.flags.timestamp ? `[${new Date(data.timeStamp).toLocaleString()}] ` : "";
-                const name = podColor(`${data.podName}${separator}`);
-                this.log(`${timestamp}${name}| ${data.content}`);
             }
         }
         catch (e_1_1) { e_1 = { error: e_1_1 }; }
@@ -101,13 +119,15 @@ class PaasLogs extends PaasKommand_1.PaasKommand {
     }
     getNumberOfSpaces(names, currentName) {
         const end = 10;
-        let max = { name: '', length: 0 };
+        let max = { name: "", length: 0 };
         for (const name of names) {
             if (max.length < name.length) {
                 max = { name: name, length: name.length };
             }
         }
-        return currentName === max.name ? end : end + (max.length - currentName.length);
+        return currentName === max.name
+            ? end
+            : end + (max.length - currentName.length);
     }
     /**
      * Returns the color to use for a pod name.

package/lib/support/PaasKommand.d.ts

@@ -11,6 +11,7 @@ export declare class PaasKommand extends Kommand {
     protected host: string;
     protected port: number;
     protected ssl: any;
+    protected packagesHost: string;
     protected paas: KuzzleSDK;
     initPaasClient(credentials: PaaSClientCredentials): Promise<void>;
     fileProjectCredentials(project: string): string;

package/lib/support/PaasKommand.js

@@ -18,6 +18,9 @@ class PaasKommand extends common_1.Kommand {
         this.ssl = process.env.KUZZLE_PAAS_SSL
             ? JSON.parse(process.env.KUZZLE_PAAS_SSL)
             : true;
+        this.packagesHost = process.env.KUZZLE_PAAS_PACKAGES_HOST
+            ? process.env.KUZZLE_PAAS_PACKAGES_HOST
+            : "packages.paas.kuzzle.io";
         // Instantiate a dummy SDK to avoid the this.paas? notation everywhere -_-
         this.paas = new kuzzle_1.KuzzleSDK({});
     }

package/lib/support/dump-collection.js

@@ -81,6 +81,7 @@ class AbstractDumper {
      * @returns a promise resolving when the dump is finished.
      */
     async dump() {
+        fs_1.default.mkdirSync(this.collectionDir, { recursive: true });
         this.filename = path_1.default.join(this.collectionDir, `documents.${this.fileExtension}`);
         this.writeStream = fs_1.default.createWriteStream(this.filename);
         const waitWrite = new Promise((resolve, reject) => this.writeStream ? this.writeStream.on("finish", resolve) : reject());

package/lib/support/kuzzle.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.KuzzleSDK = exports.kuzzleFlags = void 0;
 const tslib_1 = require("tslib");
 const http_1 = (0, tslib_1.__importDefault)(require("http"));
+const https_1 = (0, tslib_1.__importDefault)(require("https"));
 const command_1 = require("@oclif/command");
 const kuzzle_sdk_1 = require("kuzzle-sdk");
 const SECOND = 1000;
@@ -163,14 +164,15 @@ class KuzzleSDK {
         const options = {
             method: "POST",
             headers: {
-                "Authorization": `Bearer ${this.sdk.jwt}`,
+                Authorization: `Bearer ${this.sdk.jwt}`,
                 "Content-Length": Buffer.byteLength(body),
                 "Content-Type": "application/json",
             },
         };
         // Send the request
         return new Promise((resolve, reject) => {
-            const req = http_1.default.request(url, options, (res) => {
+            const httpModule = this.ssl ? https_1.default : http_1.default;
+            const req = httpModule.request(url, options, (res) => {
                 resolve(res);
             });
             req.on("error", reject);

package/lib/support/restore-collection.js

@@ -6,18 +6,16 @@ const fs_1 = (0, tslib_1.__importDefault)(require("fs"));
 const chalk_1 = (0, tslib_1.__importDefault)(require("chalk"));
 const ndjson_1 = (0, tslib_1.__importDefault)(require("ndjson"));
 function handleError(log, dumpFile, error) {
-    if (error.status === 206) {
+    if (error.errors) {
         const errorFile = `${dumpFile
             .split(".")
             .slice(0, -1)
             .join(".")}-errors.jsonl`;
         const writeStream = fs_1.default.createWriteStream(errorFile, { flags: "a" });
-        const serialize = ndjson_1.default.stringify().pipe(writeStream);
-        serialize.on("data", (line) => writeStream.write(line));
         for (const partialError of error.errors) {
-            serialize.write(partialError);
+            writeStream.write(JSON.stringify(partialError) + '\n');
         }
-        serialize.end();
+        writeStream.end();
         log(chalk_1.default.red(`[X] Error importing ${dumpFile}. See errors in ${errorFile}`));
     }
     else {
@@ -31,6 +29,7 @@ async function restoreCollectionData(sdk, log, batchSize, dumpFile, index, colle
         action: "mWrite",
         index: "",
         collection: "",
+        strict: true,
         body: {
             documents: [{}],
         },
@@ -90,8 +89,8 @@ async function restoreCollectionData(sdk, log, batchSize, dumpFile, index, colle
                 mWriteRequest.body.documents = documents;
                 sdk
                     .query(mWriteRequest)
-                    .then(() => {
-                    total += mWriteRequest.body.documents.length;
+                    .then((response) => {
+                    total += response.result.successes.length;
                     process.stdout.write(`  ${total} documents imported`);
                     process.stdout.write("\r");
                     resolve(undefined);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "kourou",
   "description": "The CLI that helps you manage your Kuzzle instances",
-  "version": "0.25.0",
+  "version": "0.25.2",
   "author": "The Kuzzle Team <support@kuzzle.io>",
   "bin": {
     "kourou": "./bin/run"