@magda/scripts 4.2.2-alpha.0 → 4.2.3

package/create-secrets/index.js

@@ -1,111 +0,0 @@
-#!/usr/bin/env node
-import { askQuestions, getEnvVarInfo } from "./askQuestions.js";
-import k8sExecution from "./k8sExecution.js";
-import preloadConfig from "./preloadConfig.js";
-import clear from "clear";
-import chalk from "chalk";
-import Configstore from "configstore";
-import { require } from "@magda/esm-utils";
-
-const appName = "magda-create-secrets";
-const pkg = require("../package.json");
-const config = new Configstore("magda-create-secrets", {});
-
-const program = require("commander");
-
-program
-    .version(pkg.version)
-    .usage("[options]")
-    .description(`A tool for magda k8s secrets setup. Version: ${pkg.version}`)
-    .option(
-        "-E, --execute [configFilePath]",
-        "Create k8s secrets in cluster using `${appName}` config file/data without asking any user input. \n" +
-            "   If you want to supply config data via STDIN, you can set `configFilePath` parameter to `-`. \n" +
-            `   e.g. \`echo $CONFIG_CONTENT | ${appName} -E -\` or \`cat config.json | ${appName} --execute=-\`\n` +
-            "   If configFilePath is not specify, program will attempt to load config file from: \n" +
-            `   either \`$XDG_CONFIG_HOME/configstore/${appName}.json\` \n   or \`~/.config/configstore/${appName}.json\``
-    )
-    .option("-P, --print", "Print previously saved local config data to stdout")
-    .option("-D, --delete", "Delete previously saved local config data");
-
-program.on("--help", function () {
-    const envInfo = getEnvVarInfo();
-    console.log("  Available Setting ENV Variables:");
-    console.log("");
-    envInfo.forEach((item) => {
-        console.log(`    ${item.name} : ${item.description}`);
-    });
-    console.log("");
-});
-
-program.parse(process.argv);
-
-const programOptions = program.opts();
-
-if (programOptions.print) {
-    process.stdout.write(JSON.stringify(config.all), "utf-8", function () {
-        process.exit();
-    });
-} else if (programOptions.delete) {
-    config.clear();
-    console.log(chalk.green("All local saved config data has been removed!"));
-    process.exit();
-} else if (programOptions.execute) {
-    console.log(
-        chalk.green(`${appName} tool version: ${pkg.version} Execute Mode`)
-    );
-    let configDataBak = {};
-    let hasError = false;
-    preloadConfig(config, programOptions.execute)
-        .then(function (data) {
-            if (programOptions.execute !== true) {
-                //--- we only need to receovey user's local config
-                //--- when the config is fed by STDIN or external file
-                configDataBak = config.all;
-            }
-            config.all = data;
-            return k8sExecution(config, true);
-        })
-        .catch(function (error) {
-            hasError = true;
-            console.log(chalk.red(`Failed to create secrets: ${error}`));
-            console.error(error);
-        })
-        .then(function () {
-            //--- can't use finally
-            if (programOptions.execute !== true) {
-                //--- recover origin config data
-                config.all = configDataBak;
-            }
-            if (hasError) {
-                process.exit(1);
-            }
-        });
-} else {
-    clear();
-    console.log("\n");
-    console.log(chalk.green(`${appName} tool version: ${pkg.version}`));
-    askQuestions(config).then(function (shouldCreateSecrets) {
-        if (shouldCreateSecrets) {
-            k8sExecution(config).then(
-                function () {
-                    console.log(
-                        chalk.green(
-                            "All required secrets have been successfully created!"
-                        )
-                    );
-                    process.exit();
-                },
-                function (error) {
-                    console.log(
-                        chalk.red(`Failed to create required secrets: ${error}`)
-                    );
-                    console.error(error);
-                    process.exit(1);
-                }
-            );
-        } else {
-            process.exit();
-        }
-    });
-}

package/create-secrets/k8sExecution.js

@@ -1,475 +0,0 @@
-import childProcess from "child_process";
-import process from "process";
-import chalk from "chalk";
-import trim from "lodash/trim.js";
-import {
-    getEnvVarInfo,
-    askIfCreateNamespace,
-    settingNameToEnvVarName
-} from "./askQuestions.js";
-import { Base64 } from "js-base64";
-import pwgen from "./pwgen.js";
-
-const dbPasswordNames = [
-    "authorization-db",
-    "authorization-db-client",
-    "combined-db",
-    "combined-db-client",
-    "content-db",
-    "content-db-client",
-    "registry-db",
-    "registry-db-client",
-    "session-db",
-    "session-db-client",
-    "tenant-db",
-    "tenant-db-client"
-];
-
-function k8sExecution(config, shouldNotAsk = false) {
-    try {
-        return doK8sExecution(config, shouldNotAsk);
-    } catch (e) {
-        return Promise.reject(e);
-    }
-}
-
-function doK8sExecution(config, shouldNotAsk = false) {
-    const env = getEnvByClusterType(config);
-    let configData = Object.assign({}, config.all);
-    const allowEnvVarOverride = configData["allow-env-override-settings"];
-
-    configData = overrideSettingWithEnvVarsBasedOnQuestionAnswers(
-        env,
-        configData
-    );
-
-    if (allowEnvVarOverride) {
-        configData = overrideSettingWithEnvVars(env, configData);
-    }
-
-    let promise = Promise.resolve().then(function () {
-        /**
-         * All errors / exceptions should be process through promise chain rather than stop program here.
-         * There are different logic outside doK8sExecution requires some clean-up job to be done before exit program.
-         */
-        checkIfKubectlValid(env, configData);
-        configData["cluster-namespace"] = trim(configData["cluster-namespace"]);
-        if (!configData["cluster-namespace"]) {
-            throw new Error(
-                "Cluster namespace cannot be empty! \n " +
-                    "If you've set cluster namespace, make sure it's not overrided by env variable."
-            );
-        }
-    });
-
-    if (!checkNamespace(env, configData["cluster-namespace"], config)) {
-        if (shouldNotAsk) {
-            promise = promise.then(function () {
-                // --- leave error to be handled at end of then chain. see above
-                throw new Error(
-                    `Namespace ${configData["cluster-namespace"]} doesn't exist. Please create and try again.`
-                );
-            });
-        }
-        promise = promise
-            .then(
-                askIfCreateNamespace.bind(null, configData["cluster-namespace"])
-            )
-            .then(function (shouldCreateNamespace) {
-                if (!shouldCreateNamespace) {
-                    throw new Error(
-                        `You need to create namespace \`${configData["cluster-namespace"]}\` before try again.`
-                    );
-                } else {
-                    createNamespace(
-                        env,
-                        configData["cluster-namespace"],
-                        configData
-                    );
-                }
-            });
-    }
-    return promise.then(function () {
-        const namespace = configData["cluster-namespace"];
-
-        if (configData["use-cloudsql-instance-credentials"] === true) {
-            createFileContentSecret(
-                env,
-                namespace,
-                configData,
-                "cloudsql-instance-credentials",
-                "credentials.json",
-                configData["cloudsql-instance-credentials"]["data"]
-            );
-            createSecret(
-                env,
-                namespace,
-                configData,
-                "cloudsql-db-credentials",
-                {
-                    password: configData["cloudsql-db-credentials"]
-                }
-            );
-        }
-
-        if (configData["use-storage-account-credentials"] === true) {
-            createFileContentSecret(
-                env,
-                namespace,
-                configData,
-                "storage-account-credentials",
-                "db-service-account-private-key.json",
-                configData["storage-account-credentials"]["data"]
-            );
-        }
-
-        if (configData["use-smtp-secret"] === true) {
-            createSecret(env, namespace, configData, "smtp-secret", {
-                username: configData["smtp-secret-username"],
-                password: configData["smtp-secret-password"]
-            });
-        }
-
-        createDbPasswords(env, namespace, configData);
-        createMinioCredentials(env, namespace, configData);
-
-        createWebAccessPassword(env, namespace, configData);
-
-        if (configData["use-regcred"] === true) {
-            /**
-             * always use `regcred-password`
-             * `use-regcred-password-from-env` has been taken care seperately
-             */
-            createDockerRegistrySecret(
-                env,
-                namespace,
-                configData,
-                "regcred",
-                configData["regcred-password"],
-                "registry.gitlab.com",
-                "gitlab-ci-token",
-                configData["regcred-email"]
-            );
-        }
-
-        if (
-            configData["use-oauth-secrets-google"] === true ||
-            configData["use-oauth-secrets-facebook"] === true ||
-            configData["use-oauth-secrets-arcgis"] === true ||
-            configData["use-oauth-secrets-aaf"] === true
-        ) {
-            const data = {};
-
-            if (configData["use-oauth-secrets-google"]) {
-                data["google-client-secret"] =
-                    configData["oauth-secrets-google"];
-            }
-
-            if (configData["use-oauth-secrets-facebook"]) {
-                data["facebook-client-secret"] =
-                    configData["oauth-secrets-facebook"];
-            }
-
-            if (configData["use-oauth-secrets-arcgis"]) {
-                data["arcgis-client-secret"] =
-                    configData["oauth-secrets-arcgis"];
-            }
-
-            if (configData["use-oauth-secrets-aaf"]) {
-                data["aaf-client-secret"] = configData["oauth-secrets-aaf"];
-            }
-
-            createSecret(env, namespace, configData, "oauth-secrets", data);
-        }
-
-        createSecret(env, namespace, configData, "auth-secrets", {
-            "jwt-secret": pwgen(64),
-            "session-secret": pwgen()
-        });
-    });
-}
-
-function getEnvByClusterType(config) {
-    const localClusterType = config.get("local-cluster-type");
-
-    if (
-        typeof localClusterType === "undefined" ||
-        localClusterType !== "minikube"
-    ) {
-        return Object.assign({}, process.env);
-    }
-
-    const dockerEnvProcess = childProcess.execSync(
-        "minikube docker-env --shell bash",
-        { encoding: "utf8" }
-    );
-    const dockerEnv = dockerEnvProcess
-        .split("\n")
-        .filter((line) => line.indexOf("export ") === 0)
-        .reduce(function (env, line) {
-            const match = /^export (\w+)="(.*)"$/.exec(line);
-            if (match) {
-                env[match[1]] = match[2];
-            }
-            return env;
-        }, {});
-
-    const env = Object.assign({}, process.env, dockerEnv);
-    return env;
-}
-
-/**
- * the difference between this function and `overrideSettingWithEnvVars` is:
- * `overrideSettingWithEnvVars` allows users to override any questions answers
- * and it will only be run when the answer to question
- * `Do you want to allow environment variables (see --help for full list) to override current settings at runtime?`
- * is `YES`.
- * This function will always be run so if user said YES to a specific question
- * (e.g. Do you want namespace to be overiden), that particular question answer will be overriden.
- */
-function overrideSettingWithEnvVarsBasedOnQuestionAnswers(env, configData) {
-    if (
-        configData["get-namespace-from-env"] === true &&
-        env[settingNameToEnvVarName("cluster-namespace")]
-    ) {
-        configData["cluster-namespace"] =
-            env[settingNameToEnvVarName("cluster-namespace")];
-    }
-
-    if (
-        configData["use-regcred-password-from-env"] === true &&
-        env["CI_JOB_TOKEN"]
-    ) {
-        configData["regcred-password"] = env["CI_JOB_TOKEN"];
-    }
-
-    if (
-        typeof configData["manual-db-passwords"] === "object" &&
-        configData["manual-db-passwords"]["answer"] === false &&
-        configData["manual-db-passwords"]["password"]
-    ) {
-        configData["db-passwords"] =
-            configData["manual-db-passwords"]["password"];
-    }
-
-    if (
-        typeof configData["manual-web-access-password"] === "object" &&
-        configData["manual-web-access-password"]["answer"] === false &&
-        configData["manual-web-access-password"]["password"]
-    ) {
-        configData["web-access-password"] =
-            configData["manual-web-access-password"]["password"];
-    }
-
-    return configData;
-}
-
-function overrideSettingWithEnvVars(env, configData) {
-    getEnvVarInfo().forEach((item) => {
-        const envVal = env[item.name];
-        if (typeof envVal === "undefined") return;
-        if (item.dataType === "boolean") {
-            const value = envVal.toLowerCase().trim();
-            if (value === "false" || value === "0") {
-                configData[item.settingName] = false;
-            } else {
-                configData[item.settingName] = true;
-            }
-        } else if (item.dataType === "jsonfile") {
-            configData[item.settingName] = {
-                data: JSON.parse(envVal)
-            };
-        } else {
-            configData[item.settingName] = envVal;
-        }
-    });
-
-    return configData;
-}
-
-function checkIfKubectlValid(env, configData) {
-    try {
-        childProcess.execSync(getKubectlCommand(configData), {
-            stdio: "ignore",
-            env: env
-        });
-    } catch (e) {
-        throw new Error(
-            `Failed to execute \`kubectl\` utility: ${e}\n` +
-                "Make sure you have install & config `kubectl` properly before try again."
-        );
-    }
-}
-
-function checkNamespace(env, namespace, configData) {
-    try {
-        childProcess.execSync(
-            `${getKubectlCommand(configData)} get namespace ${namespace}`,
-            {
-                stdio: "ignore",
-                env: env
-            }
-        );
-        return true;
-    } catch (e) {
-        console.log(
-            chalk.red(
-                `Failed to get k8s namespace ${namespace} or namespace has not been created yet: ${e}`
-            )
-        );
-        return false;
-    }
-}
-
-function createNamespace(env, namespace, configData) {
-    childProcess.execSync(
-        `${getKubectlCommand(configData)} create namespace ${namespace}`,
-        {
-            stdio: "inherit",
-            env: env
-        }
-    );
-}
-
-function buildTemplateObject(name, namespace) {
-    return {
-        apiVersion: "v1",
-        kind: "Secret",
-        type: "Opaque",
-        metadata: {
-            name,
-            namespace,
-            annotations: {
-                "replicator.v1.mittwald.de/replication-allowed": "true",
-                "replicator.v1.mittwald.de/replication-allowed-namespaces": `${namespace}-openfaas-fn`
-            },
-            creationTimestamp: null
-        }
-    };
-}
-
-function createDbPasswords(env, namespace, configData) {
-    /**
-     * dbPasswordNames is defined as const at top of the file
-     */
-    const data = {};
-    dbPasswordNames.forEach((key) => {
-        data[key] = configData["db-passwords"];
-    });
-    createSecret(env, namespace, configData, "db-passwords", data);
-}
-
-function createMinioCredentials(env, namespace, configData) {
-    const data = {
-        accesskey: configData["accesskey"],
-        secretkey: configData["secretkey"]
-    };
-    createSecret(env, namespace, configData, "storage-secrets", data);
-}
-
-function createWebAccessPassword(env, namespace, configData) {
-    if (configData["use-web-access-secret"] === false) {
-        return;
-    }
-    const data = {
-        username: configData["web-access-username"],
-        password: configData["web-access-password"]
-    };
-    createSecret(env, namespace, configData, "web-access-secret", data);
-}
-
-function createFileContentSecret(
-    env,
-    namespace,
-    configData,
-    secretName,
-    fileName,
-    content
-) {
-    if (typeof content !== "string") {
-        content = JSON.stringify(content);
-    }
-
-    createSecret(env, namespace, configData, secretName, {
-        [fileName]: content
-    });
-}
-
-function createDockerRegistrySecret(
-    env,
-    namespace,
-    configData,
-    secretName,
-    password,
-    dockerServer,
-    username,
-    email
-) {
-    const dockerConfig = {
-        auths: {
-            [dockerServer]: {
-                username: username,
-                password: password,
-                email: email,
-                auth: Base64.encode(`${username}:${password}`)
-            }
-        }
-    };
-
-    const data = {};
-    data[".dockerconfigjson"] = JSON.stringify(dockerConfig);
-    createSecret(
-        env,
-        namespace,
-        configData,
-        secretName,
-        data,
-        true,
-        "kubernetes.io/dockerconfigjson"
-    );
-}
-
-function createSecret(
-    env,
-    namespace,
-    configData,
-    secretName,
-    data,
-    encodeAllDataFields,
-    type
-) {
-    const configObj = buildTemplateObject(secretName, namespace);
-    configObj.data = data;
-
-    if (type) configObj.type = type;
-
-    if (encodeAllDataFields !== false) {
-        Object.keys(configObj.data).forEach((key) => {
-            configObj.data[key] = Base64.encode(configObj.data[key]);
-        });
-    }
-
-    const configContent = JSON.stringify(configObj);
-
-    childProcess.execSync(
-        `${getKubectlCommand(configData)} apply --namespace ${namespace} -f -`,
-        {
-            input: configContent,
-            env: env
-        }
-    );
-
-    console.log(
-        chalk.green(
-            `Successfully created secret \`${secretName}\` in namespace \`${namespace}\`.`
-        )
-    );
-}
-
-function getKubectlCommand(configData) {
-    const clusterType = configData["local-cluster-type"];
-
-    return clusterType === "microk8s" ? "microk8s kubectl" : "kubectl";
-}
-
-export default k8sExecution;

package/create-secrets/preloadConfig.js

@@ -1,103 +0,0 @@
-import fs from "fs";
-import chalk from "chalk";
-import trim from "lodash/trim.js";
-
-function preloadConfig(configStore, executeOption) {
-    return Promise.resolve().then(function () {
-        if (executeOption === true) {
-            console.log(
-                chalk.yellow(
-                    `Loading config data from \`${configStore.path}\`...`
-                )
-            );
-            // --- trigger config file read
-            const configData = configStore.all;
-            if (!configStore.size) {
-                throw new Error(`Error: loaded config object is empty!`);
-            }
-            console.log(
-                chalk.green(
-                    `Successfully loaded config data from \`${configStore.path}\`.`
-                )
-            );
-            return configData;
-        } else if (executeOption === "-") {
-            return readConfigFromStdin(configStore);
-        } else {
-            return readConfigFromFile(configStore, executeOption);
-        }
-    });
-}
-
-function readConfigFromStdin(configStore) {
-    console.log(chalk.yellow(`Loading config data from STDIN ...`));
-    let configContent = "";
-    return new Promise(function (resolve, reject) {
-        process.stdin.setEncoding("utf8");
-
-        process.stdin.on("readable", () => {
-            try {
-                const chunk = process.stdin.read();
-                if (chunk !== null) {
-                    configContent += chunk;
-                }
-            } catch (e) {
-                reject(e);
-            }
-        });
-
-        process.stdin.on("end", () => {
-            try {
-                const data = JSON.parse(configContent);
-                if (
-                    !data ||
-                    typeof data !== "object" ||
-                    !Object.keys(data).length
-                ) {
-                    throw new Error("Loaded config object is empty!");
-                }
-                console.log(
-                    chalk.green(`Successfully loaded config data from STDIN.`)
-                );
-                resolve(data);
-            } catch (e) {
-                reject(e);
-            }
-        });
-    });
-}
-
-function readConfigFromFile(configStore, executeOption) {
-    console.log(chalk.yellow(`Loading config data from ${executeOption}...`));
-    return new Promise(function (resolve, reject) {
-        try {
-            const filePath = trim(executeOption);
-            if (!fs.existsSync(filePath)) {
-                throw new Error(
-                    "The config file path specified does not exist or cannot read."
-                );
-            }
-            const content = fs.readFileSync(filePath, {
-                encoding: "utf-8"
-            });
-            const data = JSON.parse(content);
-            if (
-                !data ||
-                typeof data !== "object" ||
-                !Object.keys(data).length
-            ) {
-                throw new Error("Loaded config object is empty!");
-            }
-            console.log(
-                chalk.green(
-                    `Successfully loaded config data from \`${filePath}\`.`
-                )
-            );
-            resolve(data);
-        } catch (e) {
-            reject(e);
-        }
-    });
-}
-
-export default preloadConfig;