kourou 0.27.1 → 0.28.1

package/README.md

@@ -27,7 +27,7 @@ $ npm install -g kourou
 $ kourou COMMAND
 running command...
 $ kourou (-v|--version|version)
-kourou/0.27.1 darwin-arm64 node-v18.17.1
+kourou/0.28.1 darwin-arm64 node-v20.10.0
 $ kourou --help [COMMAND]
 USAGE
   $ kourou COMMAND
@@ -149,6 +149,7 @@ All other arguments and options will be passed as-is to the `sdk:query` method.
 * [`kourou es:snapshot:create REPOSITORY NAME`](#kourou-essnapshotcreate-repository-name)
 * [`kourou es:snapshot:create-repository REPOSITORY LOCATION`](#kourou-essnapshotcreate-repository-repository-location)
 * [`kourou es:snapshot:list REPOSITORY`](#kourou-essnapshotlist-repository)
+* [`kourou es:snapshot:restore REPOSITORY NAME`](#kourou-essnapshotrestore-repository-name)
 * [`kourou file:decrypt FILE`](#kourou-filedecrypt-file)
 * [`kourou file:encrypt FILE`](#kourou-fileencrypt-file)
 * [`kourou file:test FILE`](#kourou-filetest-file)
@@ -161,6 +162,7 @@ All other arguments and options will be passed as-is to the `sdk:query` method.
 * [`kourou instance:logs`](#kourou-instancelogs)
 * [`kourou instance:spawn`](#kourou-instancespawn)
 * [`kourou paas:deploy ENVIRONMENT APPLICATIONID IMAGE`](#kourou-paasdeploy-environment-applicationid-image)
+* [`kourou paas:elasticsearch:dump ENVIRONMENT APPLICATIONID DUMPDIRECTORY`](#kourou-paaselasticsearchdump-environment-applicationid-dumpdirectory)
 * [`kourou paas:init PROJECT`](#kourou-paasinit-project)
 * [`kourou paas:login`](#kourou-paaslogin)
 * [`kourou paas:logs ENVIRONMENT APPLICATION`](#kourou-paaslogs-environment-application)
@@ -794,6 +796,23 @@ OPTIONS
 
 _See code: [lib/commands/es/snapshot/list.js](lib/commands/es/snapshot/list.js)_
 
+## `kourou es:snapshot:restore REPOSITORY NAME`
+
+Restore a snapshot repository inside an ES instance
+
+```
+USAGE
+  $ kourou es:snapshot:restore REPOSITORY NAME
+
+ARGUMENTS
+  REPOSITORY  ES repository name
+  NAME        ES snapshot name
+
+OPTIONS
+  -n, --node=node  [default: http://localhost:9200] Elasticsearch server URL
+  --help           show CLI help
+```
+
 ## `kourou file:decrypt FILE`
 
 Decrypts an encrypted file.
@@ -1064,6 +1083,25 @@ OPTIONS
 
 _See code: [lib/commands/paas/deploy.js](lib/commands/paas/deploy.js)_
 
+## `kourou paas:elasticsearch:dump ENVIRONMENT APPLICATIONID DUMPDIRECTORY`
+
+Dump data from the Elasticsearch of a PaaS application
+
+```
+USAGE
+  $ kourou paas:elasticsearch:dump ENVIRONMENT APPLICATIONID DUMPDIRECTORY
+
+ARGUMENTS
+  ENVIRONMENT    Project environment name
+  APPLICATIONID  Application Identifier
+  DUMPDIRECTORY  Directory where to store dump files
+
+OPTIONS
+  --batch-size=batch-size  [default: 2000] Maximum batch size
+  --help                   show CLI help
+  --project=project        Current PaaS project
+```
+
 ## `kourou paas:init PROJECT`
 
 Initialize a PaaS project in current directory

package/lib/commands/es/snapshot/create.js

@@ -9,7 +9,11 @@ class EsSnapshotsCreate extends common_1.Kommand {
         const esRequest = {
             repository: this.args.repository,
             snapshot: this.args.name,
-            body: {},
+            body: {
+                indices: "*",
+                include_global_state: false,
+                partial: false,
+            },
         };
         const response = await esClient.snapshot.create(esRequest);
         this.logOk(`Success ${JSON.stringify(response.body)}`);

package/lib/commands/es/snapshot/restore.d.ts

@@ -0,0 +1,16 @@
+import { flags } from "@oclif/command";
+import { Kommand } from "../../../common";
+export default class EsSnapshotsRestore extends Kommand {
+    static initSdk: boolean;
+    static description: string;
+    static flags: {
+        node: flags.IOptionFlag<string>;
+        help: import("@oclif/parser/lib/flags").IBooleanFlag<void>;
+    };
+    static args: {
+        name: string;
+        description: string;
+        required: boolean;
+    }[];
+    runSafe(): Promise<void>;
+}

package/lib/commands/es/snapshot/restore.js

@@ -0,0 +1,40 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const command_1 = require("@oclif/command");
+const elasticsearch_1 = require("@elastic/elasticsearch");
+const common_1 = require("../../../common");
+class EsSnapshotsRestore extends common_1.Kommand {
+    async runSafe() {
+        const esClient = new elasticsearch_1.Client({ node: this.flags.node });
+        await esClient.indices.close({
+            index: "*",
+            expand_wildcards: "all",
+        });
+        const esRequest = {
+            repository: this.args.repository,
+            snapshot: this.args.name,
+            body: {
+                feature_states: ["none"],
+                include_global_state: false,
+                indices: "*",
+            },
+        };
+        const response = await esClient.snapshot.restore(esRequest);
+        this.logOk(`Success ${JSON.stringify(response.body)}`);
+    }
+}
+exports.default = EsSnapshotsRestore;
+EsSnapshotsRestore.initSdk = false;
+EsSnapshotsRestore.description = "Restore a snapshot repository inside an ES instance";
+EsSnapshotsRestore.flags = {
+    node: command_1.flags.string({
+        char: "n",
+        description: "Elasticsearch server URL",
+        default: "http://localhost:9200",
+    }),
+    help: command_1.flags.help(),
+};
+EsSnapshotsRestore.args = [
+    { name: "repository", description: "ES repository name", required: true },
+    { name: "name", description: "ES snapshot name", required: true },
+];

package/lib/commands/paas/elasticsearch/dump.d.ts

@@ -0,0 +1,35 @@
+import { flags } from "@oclif/command";
+import { PaasKommand } from "../../../support/PaasKommand";
+declare class PaasEsDump extends PaasKommand {
+    static description: string;
+    static flags: {
+        help: import("@oclif/parser/lib/flags").IBooleanFlag<void>;
+        project: flags.IOptionFlag<string | undefined>;
+        "batch-size": import("@oclif/parser/lib/flags").IOptionFlag<number>;
+    };
+    static args: {
+        name: string;
+        description: string;
+        required: boolean;
+    }[];
+    runSafe(): Promise<void>;
+    /**
+      * @description Get all indexes from the Elasticsearch of the PaaS application.
+      * @returns The indexes.
+      */
+    private getAllIndexes;
+    /**
+      * @description Dump documents from the Elasticsearch of the PaaS application.
+      * @param pitId ID of the PIT opened on Elasticsearch.
+      * @param searchAfter Cursor for dumping documents after a certain one.
+      * @returns The dumped documents.
+      */
+    private dumpDocuments;
+    private dumpAllDocuments;
+    /**
+      * @description Finish the document dumping session.
+      * @param pitId ID of the PIT opened on Elasticsearch.
+      */
+    private finishDump;
+}
+export default PaasEsDump;

package/lib/commands/paas/elasticsearch/dump.js

@@ -0,0 +1,172 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const tslib_1 = require("tslib");
+const path_1 = (0, tslib_1.__importDefault)(require("path"));
+const promises_1 = (0, tslib_1.__importDefault)(require("node:fs/promises"));
+const ndjson_1 = (0, tslib_1.__importDefault)(require("ndjson"));
+const command_1 = require("@oclif/command");
+const PaasKommand_1 = require("../../../support/PaasKommand");
+class PaasEsDump extends PaasKommand_1.PaasKommand {
+    async runSafe() {
+        // Check that the batch size is positive
+        if (this.flags["batch-size"] <= 0) {
+            this.logKo(`The batch size must be greater than zero. (Specified batch size: ${this.flags["batch-size"]})`);
+            process.exit(1);
+        }
+        // Log in to the PaaS
+        const apiKey = await this.getCredentials();
+        await this.initPaasClient({ apiKey });
+        const user = await this.paas.auth.getCurrentUser();
+        this.logInfo(`Logged as "${user._id}" for project "${this.flags.project || this.getProject()}"`);
+        // Create the dump directory
+        await promises_1.default.mkdir(this.args.dumpDirectory, { recursive: true });
+        // Dump the indexes
+        this.logInfo("Dumping Elasticsearch indexes...");
+        const indexesResult = await this.getAllIndexes();
+        await promises_1.default.writeFile(path_1.default.join(this.args.dumpDirectory, "indexes.json"), JSON.stringify(indexesResult));
+        this.logOk("Elasticsearch indexes dumped!");
+        // Dump all the documents
+        this.logInfo("Dumping Elasticsearch documents...");
+        await this.dumpAllDocuments();
+        this.logOk("Elasticsearch documents dumped!");
+        this.logOk(`The dumped files are available under "${path_1.default.resolve(this.args.dumpDirectory)}"`);
+    }
+    /**
+      * @description Get all indexes from the Elasticsearch of the PaaS application.
+      * @returns The indexes.
+      */
+    async getAllIndexes() {
+        const { result } = await this.paas.query({
+            controller: "application/storage",
+            action: "getIndexes",
+            environmentId: this.args.environment,
+            projectId: this.flags.project || this.getProject(),
+            applicationId: this.args.applicationId,
+            body: {},
+        });
+        return result;
+    }
+    /**
+      * @description Dump documents from the Elasticsearch of the PaaS application.
+      * @param pitId ID of the PIT opened on Elasticsearch.
+      * @param searchAfter Cursor for dumping documents after a certain one.
+      * @returns The dumped documents.
+      */
+    async dumpDocuments(pitId, searchAfter) {
+        const { result } = await this.paas.query({
+            controller: "application/storage",
+            action: "dumpDocuments",
+            environmentId: this.args.environment,
+            projectId: this.flags.project || this.getProject(),
+            applicationId: this.args.applicationId,
+            body: {
+                pitId,
+                searchAfter: JSON.stringify(searchAfter),
+                size: this.flags["batch-size"],
+            },
+        });
+        return result;
+    }
+    async dumpAllDocuments() {
+        // Prepare dumping all documents
+        let pitId = "";
+        let searchAfter = [];
+        let dumpedDocuments = 0;
+        let totalDocuments = 0;
+        const fd = await promises_1.default.open(path_1.default.join(this.args.dumpDirectory, "documents.jsonl"), "w");
+        const writeStream = fd.createWriteStream();
+        const ndjsonStream = ndjson_1.default.stringify();
+        writeStream.on("error", (error) => {
+            throw error;
+        });
+        ndjsonStream.on("data", (line) => {
+            writeStream.write(line);
+        });
+        const teardown = async () => {
+            // Finish the dump session if a PIT ID is set
+            if (pitId.length > 0) {
+                await this.finishDump(pitId);
+            }
+            // Close the open streams/file
+            writeStream.close();
+            await fd.close();
+        };
+        try {
+            // Dump the first batch
+            let result = await this.dumpDocuments(pitId, searchAfter);
+            let hits = result.hits.hits;
+            while (hits.length > 0) {
+                // Update the PIT ID and the cursor for the next dump
+                pitId = result.pit_id;
+                searchAfter = hits[hits.length - 1].sort;
+                // Save the documents
+                for (let i = 0; i < hits.length; ++i) {
+                    ndjsonStream.write(hits[i]);
+                }
+                dumpedDocuments += hits.length;
+                totalDocuments = result.hits.total.value;
+                this.logInfo(`Dumping Elasticsearch documents: ${Math.floor(dumpedDocuments / totalDocuments * 100)}% (${dumpedDocuments}/${totalDocuments})`);
+                // Dump the next batch
+                result = await this.dumpDocuments(pitId, searchAfter);
+                hits = result.hits.hits;
+            }
+        }
+        catch (error) {
+            teardown();
+            this.logKo(`Error while dumping the documents: ${error}`);
+            process.exit(1);
+        }
+        // Finish the dump
+        teardown();
+    }
+    /**
+      * @description Finish the document dumping session.
+      * @param pitId ID of the PIT opened on Elasticsearch.
+      */
+    async finishDump(pitId) {
+        try {
+            await this.paas.query({
+                controller: "application/storage",
+                action: "finishDumpDocuments",
+                environmentId: this.args.environment,
+                projectId: this.flags.project || this.getProject(),
+                applicationId: this.args.applicationId,
+                body: {
+                    pitId,
+                },
+            });
+        }
+        catch (error) {
+            this.logInfo(`Unable to cleanly finish the dump session: ${error}`);
+        }
+    }
+}
+PaasEsDump.description = "Dump data from the Elasticsearch of a PaaS application";
+PaasEsDump.flags = {
+    help: command_1.flags.help(),
+    project: command_1.flags.string({
+        description: "Current PaaS project",
+    }),
+    "batch-size": command_1.flags.integer({
+        description: "Maximum batch size",
+        default: 2000,
+    }),
+};
+PaasEsDump.args = [
+    {
+        name: "environment",
+        description: "Project environment name",
+        required: true,
+    },
+    {
+        name: "applicationId",
+        description: "Application Identifier",
+        required: true,
+    },
+    {
+        name: "dumpDirectory",
+        description: "Directory where to store dump files",
+        required: true,
+    }
+];
+exports.default = PaasEsDump;

package/lib/commands/paas/snapshots/restore.js

@@ -14,10 +14,7 @@ class PaasSnapshotsRestore extends PaasKommand_1.PaasKommand {
             environmentId: this.args.environment,
             projectId: this.flags.project || this.getProject(),
             applicationId: this.args.applicationId,
-            body: {
-                repository: "automated",
-                snapshot: this.args.snapshotId,
-            },
+            snapshotId: this.args.snapshotId
         });
         this.logInfo("Ok");
     }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "kourou",
   "description": "The CLI that helps you manage your Kuzzle instances",
-  "version": "0.27.1",
+  "version": "0.28.1",
   "author": "The Kuzzle Team <support@kuzzle.io>",
   "bin": {
     "kourou": "./bin/run"
@@ -56,7 +56,7 @@
     "@types/listr": "^0.14.2",
     "@types/mocha": "^8.2.2",
     "@types/ndjson": "^2.0.0",
-    "@types/node": "^14.14.41",
+    "@types/node": "^18.19.0",
     "@types/node-emoji": "^1.8.1",
     "@types/node-fetch": "^2.6.1",
     "@types/tar": "^6.1.3",