kourou 1.2.0 → 1.4.0-dev.1

package/README.md

@@ -27,7 +27,7 @@ $ npm install -g kourou
 $ kourou COMMAND
 running command...
 $ kourou (-v|--version|version)
-kourou/1.2.0 linux-x64 node-v22.16.0
+kourou/1.4.0-dev.1 linux-x64 node-v24.18.0
 $ kourou --help [COMMAND]
 USAGE
   $ kourou COMMAND
@@ -364,8 +364,11 @@ ARGUMENTS
   DESTINATION  Directory to scaffold the app
 
 OPTIONS
-  --flavor=flavor  [default: generic] Template flavor ("generic", "iot").
+  --flavor=flavor  [default: generic] Template flavor ("generic", "iot", "hypervision").
   --help           show CLI help
+
+  --token=token    GitHub token used to clone private template repositories.
+                   Defaults to the GITHUB_TOKEN environment variable, then to the GitHub CLI credentials.
 ```
 
 _See code: [lib/commands/app/scaffold.js](lib/commands/app/scaffold.js)_

package/lib/commands/app/scaffold.d.ts

@@ -7,6 +7,7 @@ export default class AppScaffold extends Kommand {
     static flags: {
         help: import("@oclif/parser/lib/flags").IBooleanFlag<void>;
         flavor: flags.IOptionFlag<string>;
+        token: flags.IOptionFlag<string | undefined>;
     };
     static args: {
         name: string;
@@ -14,9 +15,28 @@ export default class AppScaffold extends Kommand {
         required: boolean;
     }[];
     runSafe(): Promise<void>;
-    getRepo(flavor: string): "template-kuzzle-project" | "template-kiotp-project";
+    getRepo(flavor: string): "template-kuzzle-project" | "template-kiotp-project" | "template-hypervision-project";
     checkDestination(destination: string): Promise<void>;
     prepareTemplate(): Promise<void>;
+    /**
+     * Returns a GitHub token allowing to clone private template repositories,
+     * or undefined if none can be found.
+     *
+     * Looked up in the --token flag, then in the usual environment variables,
+     * then in the GitHub CLI credentials.
+     */
+    getToken(): Promise<string | undefined>;
+    /**
+     * Returns the URLs to try to clone the template repository from, in order.
+     *
+     * Some templates are hosted in private repositories, so we need credentials:
+     * either a token, or the SSH key of the user.
+     */
+    getCloneUrls(repo: string): Promise<string[]>;
+    /**
+     * Removes any token from a string before displaying it.
+     */
+    hideToken(message: string): string;
     cloneTemplate(flavor: string): Promise<void>;
     copyTemplate(destination: string): Promise<void>;
     cleanup(destination: string): Promise<void>;

package/lib/commands/app/scaffold.js

@@ -47,6 +47,8 @@ class AppScaffold extends common_1.Kommand {
                 return "template-kuzzle-project";
             case "iot":
                 return "template-kiotp-project";
+            case "hypervision":
+                return "template-hypervision-project";
             default:
                 return "template-kuzzle-project";
         }
@@ -69,12 +71,81 @@ class AppScaffold extends common_1.Kommand {
     async prepareTemplate() {
         await (0, execute_1.execute)("rm", "-rf", this.templatesDir);
     }
+    /**
+     * Returns a GitHub token allowing to clone private template repositories,
+     * or undefined if none can be found.
+     *
+     * Looked up in the --token flag, then in the usual environment variables,
+     * then in the GitHub CLI credentials.
+     */
+    async getToken() {
+        const token = this.flags.token || process.env.GITHUB_TOKEN || process.env.GH_TOKEN;
+        if (token) {
+            return token;
+        }
+        try {
+            const { stdout } = await (0, execute_1.execute)("gh", "auth", "token");
+            return stdout.trim() || undefined;
+        }
+        catch (error) {
+            // GitHub CLI is not installed or not authenticated
+            return undefined;
+        }
+    }
+    /**
+     * Returns the URLs to try to clone the template repository from, in order.
+     *
+     * Some templates are hosted in private repositories, so we need credentials:
+     * either a token, or the SSH key of the user.
+     */
+    async getCloneUrls(repo) {
+        const urls = [];
+        const token = await this.getToken();
+        if (token) {
+            urls.push(`https://x-access-token:${token}@github.com/kuzzleio/${repo}`);
+        }
+        urls.push(`git@github.com:kuzzleio/${repo}`);
+        urls.push(`https://github.com/kuzzleio/${repo}`);
+        return urls;
+    }
+    /**
+     * Removes any token from a string before displaying it.
+     */
+    hideToken(message) {
+        return message.replace(/x-access-token:[^@]*@/g, "x-access-token:***@");
+    }
     async cloneTemplate(flavor) {
         const repo = this.getRepo(flavor);
-        await (0, execute_1.execute)("git", "clone", "--depth=1", `https://github.com/kuzzleio/${repo}`, "--branch", "main", "--single-branch", this.templatesDir);
+        const urls = await this.getCloneUrls(repo);
+        let lastError;
+        for (const url of urls) {
+            try {
+                await (0, execute_1.execute)("git", "clone", "--depth=1", url, "--branch", "stable", "--single-branch", this.templatesDir, {
+                    env: Object.assign(Object.assign({}, process.env), { 
+                        // Never prompt for credentials, we want to fail fast and try
+                        // the next URL instead of hanging on a password prompt
+                        GIT_TERMINAL_PROMPT: "0", GIT_SSH_COMMAND: process.env.GIT_SSH_COMMAND ||
+                            "ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new" }),
+                });
+                return;
+            }
+            catch (error) {
+                // Clone failed (e.g. the repository is private and those credentials
+                // are not allowed to read it), let's try the next URL
+                lastError = error;
+                await (0, execute_1.execute)("rm", "-rf", this.templatesDir);
+            }
+        }
+        throw new Error(`Unable to clone the template repository "kuzzleio/${repo}". ` +
+            "If this repository is private, provide credentials with the --token flag, " +
+            "the GITHUB_TOKEN environment variable, the GitHub CLI (gh auth login) " +
+            `or a SSH key allowed to read it.\n${this.hideToken((lastError === null || lastError === void 0 ? void 0 : lastError.message) || "")}`);
     }
     async copyTemplate(destination) {
-        await (0, execute_1.execute)("cp", "-r", `${this.templatesDir}/`, `${destination}/`);
+        // -R (and not -r) so symlinks contained in the template are copied as
+        // symlinks instead of being dereferenced (BSD cp -r follows them and fails
+        // on dangling symlinks)
+        await (0, execute_1.execute)("cp", "-R", `${this.templatesDir}/`, `${destination}/`);
     }
     async cleanup(destination) {
         await (0, execute_1.execute)("rm", "-rf", this.templatesDir);
@@ -88,7 +159,11 @@ AppScaffold.flags = {
     help: command_1.flags.help(),
     flavor: command_1.flags.string({
         default: "generic",
-        description: `Template flavor ("generic", "iot").`,
+        description: `Template flavor ("generic", "iot", "hypervision").`,
+    }),
+    token: command_1.flags.string({
+        description: `GitHub token used to clone private template repositories.
+    Defaults to the GITHUB_TOKEN environment variable, then to the GitHub CLI credentials.`,
     }),
 };
 AppScaffold.args = [

package/lib/common.js

@@ -70,6 +70,7 @@ class Kommand extends command_1.Command {
         this.log(chalk_1.default.red(`[X] ${message}`));
     }
     async run() {
+        var _a, _b, _c, _d, _e;
         const kommand = this.constructor;
         const result = this.parse(kommand);
         this.args = result.args;
@@ -102,7 +103,13 @@ class Kommand extends command_1.Command {
             const errorLink = typeof error.id === "string" && error.id.split(".").length === 3
                 ? ` (https://docs.kuzzle.io/core/2/api/errors/error-codes/${error.id.split(".")[0]})`
                 : "";
-            this.logKo(`Error stack: \n${stack || error.message}\n\nError status: ${error.status}\n\nError id: ${error.id}${errorLink}`);
+            // Elasticsearch client errors keep their details in "meta", they would
+            // be lost otherwise
+            const esBody = (_b = (_a = error.meta) === null || _a === void 0 ? void 0 : _a.body) !== null && _b !== void 0 ? _b : error.body;
+            const esDetails = esBody
+                ? `\n\nElasticsearch response: ${JSON.stringify(esBody)}`
+                : "";
+            this.logKo(`Error stack: \n${stack || error.message}${esDetails}\n\nError status: ${(_d = (_c = error.status) !== null && _c !== void 0 ? _c : error.statusCode) !== null && _d !== void 0 ? _d : (_e = error.meta) === null || _e === void 0 ? void 0 : _e.statusCode}\n\nError id: ${error.id}${errorLink}`);
             if (Array.isArray(error.errors)) {
                 for (const e of error.errors) {
                     this.logKo(`${e.document._id} : ${e.reason}`);

package/lib/support/execute.js

@@ -33,6 +33,9 @@ function execute(...args) {
     process.stdout.on("data", (data) => (stdout += data.toString()));
     process.stderr.on("data", (data) => (stderr += data.toString()));
     const executor = new Promise((resolve, reject) => {
+        // Command cannot be spawned (e.g. binary not installed).
+        // Without this listener, Node throws the error instead of rejecting.
+        process.on("error", (error) => reject(error));
         process.on("close", (code) => {
             if (code === 0) {
                 resolve({ stdout, stderr, exitCode: code });

package/lib/support/migrate/providers/bulk.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Splits documents into chunks so a bulk request never exceeds the
+ * Elasticsearch "http.max_content_length" limit (100mb by default).
+ */
+export declare function chunkDocuments(docs: any[], chunkSize?: number): Generator<any[]>;
+/**
+ * A bulk request answers with a 200 even when some documents were rejected,
+ * so the per-item errors have to be checked explicitly.
+ */
+export declare function throwOnBulkErrors(response: any): void;

package/lib/support/migrate/providers/bulk.js

@@ -0,0 +1,38 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.throwOnBulkErrors = exports.chunkDocuments = void 0;
+const DEFAULT_CHUNK_SIZE = 1000;
+/**
+ * Splits documents into chunks so a bulk request never exceeds the
+ * Elasticsearch "http.max_content_length" limit (100mb by default).
+ */
+function* chunkDocuments(docs, chunkSize = DEFAULT_CHUNK_SIZE) {
+    const size = chunkSize > 0 ? chunkSize : DEFAULT_CHUNK_SIZE;
+    for (let i = 0; i < docs.length; i += size) {
+        yield docs.slice(i, i + size);
+    }
+}
+exports.chunkDocuments = chunkDocuments;
+/**
+ * A bulk request answers with a 200 even when some documents were rejected,
+ * so the per-item errors have to be checked explicitly.
+ */
+function throwOnBulkErrors(response) {
+    if (!response || !response.errors) {
+        return;
+    }
+    const failures = (response.items || [])
+        .map((item) => item.index || item.create || item.update || item.delete)
+        .filter((item) => item && item.error);
+    if (failures.length === 0) {
+        return;
+    }
+    const [first] = failures;
+    const error = new Error(`${failures.length} document(s) rejected by Elasticsearch. First failure on "${first._id}": ${first.error.type} - ${first.error.reason}`);
+    error.errors = failures.map((item) => ({
+        document: { _id: item._id },
+        reason: `${item.error.type} - ${item.error.reason}`,
+    }));
+    throw error;
+}
+exports.throwOnBulkErrors = throwOnBulkErrors;

package/lib/support/migrate/providers/elasticsearch7.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Elasticsearch7 = void 0;
 const sdk_es7_1 = require("sdk-es7");
+const bulk_1 = require("./bulk");
 class Elasticsearch7 {
     constructor(url, options) {
         const { auth, url: cleanedUrl } = this.extractCredentials(url);
@@ -93,13 +94,20 @@ class Elasticsearch7 {
         }
     }
     async writeData(index, docs) {
-        const bulk = [];
-        for (const doc of docs) {
-            bulk.push({ index: { _index: index, _id: doc._id } });
-            bulk.push(doc._source);
+        let count = 0;
+        // Documents are sent by chunks, otherwise the bulk request may exceed
+        // the Elasticsearch "http.max_content_length" limit (100mb by default)
+        for (const chunk of (0, bulk_1.chunkDocuments)(docs, this.options.batchSize)) {
+            const bulk = [];
+            for (const doc of chunk) {
+                bulk.push({ index: { _index: index, _id: doc._id } });
+                bulk.push(doc._source);
+            }
+            const { body } = await this.client.bulk({ body: bulk });
+            (0, bulk_1.throwOnBulkErrors)(body);
+            count += bulk.length / 2;
         }
-        await this.client.bulk({ body: bulk });
-        return bulk.length / 2;
+        return count;
     }
     async clear() {
         await this.client.indices.delete({ index: "_all" });

package/lib/support/migrate/providers/elasticsearch8.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Elasticsearch8 = void 0;
 const sdk_es8_1 = require("sdk-es8");
+const bulk_1 = require("./bulk");
 class Elasticsearch8 {
     constructor(url, options) {
         const { auth, url: cleanedUrl } = this.extractCredentials(url);
@@ -95,13 +96,20 @@ class Elasticsearch8 {
         }
     }
     async writeData(index, docs) {
-        const bulk = [];
-        for (const doc of docs) {
-            bulk.push({ index: { _index: index, _id: doc._id } });
-            bulk.push(doc._source);
+        let count = 0;
+        // Documents are sent by chunks, otherwise the bulk request may exceed
+        // the Elasticsearch "http.max_content_length" limit (100mb by default)
+        for (const chunk of (0, bulk_1.chunkDocuments)(docs, this.options.batchSize)) {
+            const bulk = [];
+            for (const doc of chunk) {
+                bulk.push({ index: { _index: index, _id: doc._id } });
+                bulk.push(doc._source);
+            }
+            const response = await this.client.bulk({ body: bulk });
+            (0, bulk_1.throwOnBulkErrors)(response);
+            count += bulk.length / 2;
         }
-        await this.client.bulk({ body: bulk });
-        return bulk.length / 2;
+        return count;
     }
     async clear() {
         const indices = await this.listIndices();

package/lib/support/migrate/providers/index.d.ts

@@ -1,4 +1,5 @@
 import { IndexSpecification } from "../types";
+export * from "./bulk";
 export * from "./elasticsearch7";
 export * from "./elasticsearch8";
 export * from "./file";

package/lib/support/migrate/providers/index.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.isURL = void 0;
 const tslib_1 = require("tslib");
+(0, tslib_1.__exportStar)(require("./bulk"), exports);
 (0, tslib_1.__exportStar)(require("./elasticsearch7"), exports);
 (0, tslib_1.__exportStar)(require("./elasticsearch8"), exports);
 (0, tslib_1.__exportStar)(require("./file"), exports);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "kourou",
   "description": "The CLI that helps you manage your Kuzzle instances",
-  "version": "1.2.0",
+  "version": "1.4.0-dev.1",
   "author": "The Kuzzle Team <support@kuzzle.io>",
   "bin": {
     "kourou": "./bin/run"
@@ -57,7 +57,7 @@
     "@types/listr": "0.14.2",
     "@types/mocha": "8.2.2",
     "@types/ndjson": "2.0.0",
-    "@types/node": "18.19.0",
+    "@types/node": "20.14.15",
     "@types/node-emoji": "1.8.1",
     "@types/node-fetch": "2.6.1",
     "@types/tar": "6.1.3",
@@ -75,15 +75,11 @@
     "should": "13.2.3",
     "source-map-support": "0.5.19",
     "ts-node": "10.9.*",
-    "semantic-release-config-kuzzle": "1.0.0",
-    "semantic-release-slack-bot": "4.0.2",
-    "@semantic-release/changelog": "6.0.3",
-    "@semantic-release/commit-analyzer": "13.0.0",
-    "@semantic-release/git": "10.0.1",
-    "@semantic-release/release-notes-generator": "14.0.1"
+    "semantic-release": "25.0.8",
+    "semantic-release-config-kuzzle": "1.7.0"
   },
   "engines": {
-    "node": ">=18.0.0"
+    "node": ">=20.0.0"
   },
   "files": [
     "/bin",