@catladder/cli 0.0.0-test4 → 0.0.0-updatemongo6-52a2af91

package/src/apps/{shell → cli}/commands/mongodb/projectMongoDestroyMember.ts

@@ -3,9 +3,9 @@ import Vorpal from "vorpal";
 import { logError } from "../../../../utils/log";
 import {
   getProjectNamespace,
-  getProjectPvcs
+  getProjectPvcs,
 } from "../../../../utils/projects";
-import { envAutocompletion } from "../project/utils/autocompletions";
+import { envAndComponents } from "../project/utils/autocompletions";
 import ensureCluster from "../project/utils/ensureCluster";
 import { getMongoDbPodsWithReplInfo } from "./utils";
 
@@ -31,18 +31,18 @@ const removeFinalizerAndDelete = async (
 ) => {
   return Promise.all([
     removeFinalizer(namespace, type, name),
-    await deleteResource(namespace, type, name)
+    await deleteResource(namespace, type, name),
   ]);
 };
-export default (vorpal: Vorpal) =>
+export default async (vorpal: Vorpal) =>
   vorpal
     .command(
-      "project-mongo-destroy-member <env>",
+      "project-mongo-destroy-member <envComponent>",
       "DESTROY a member of a replicaset in order to reinitialize it"
     )
-    .autocomplete(envAutocompletion)
-    .action(async function({ env }) {
-      await ensureCluster.call(this);
+    .autocomplete(await envAndComponents())
+    .action(async function ({ envComponent }) {
+      await ensureCluster.call(this, envComponent);
       this.log(
         "this command tries to delete a (secondary) member of the replicaset, it's persistent volume claim (pvc) and the volume"
       );
@@ -67,18 +67,18 @@ export default (vorpal: Vorpal) =>
         default: true,
         message: "DO YOU UNDERSTAND?",
         name: "understood",
-        type: "confirm"
+        type: "confirm",
       });
 
       if (!understood) {
         throw new Error("abort");
       }
 
-      const namespace = await getProjectNamespace(env);
-      const mongodbPods = await getMongoDbPodsWithReplInfo(env);
+      const namespace = await getProjectNamespace(envComponent);
+      const mongodbPods = await getMongoDbPodsWithReplInfo(envComponent);
 
-      const pvcs = await getProjectPvcs(env);
-      const secondaries = mongodbPods.filter(pod => !pod.isMaster);
+      const pvcs = await getProjectPvcs(envComponent);
+      const secondaries = mongodbPods.filter((pod) => !pod.isMaster);
 
       if (secondaries.length === 0) {
         logError(this, "sorry, no pods found");
@@ -89,16 +89,16 @@ export default (vorpal: Vorpal) =>
         await this.prompt({
           type: "list",
           name: "podName",
-          choices: secondaries.map(p => ({
+          choices: secondaries.map((p) => ({
             name: `[ secondary ] ${p.podName}`,
-            value: p.podName
+            value: p.podName,
           })),
-          message: "Which pod? 🤔"
+          message: "Which pod? 🤔",
         })
       ).podName;
 
       const thePvc = pvcs.find(
-        pvc => pvc.metadata.name === `datadir-${podName}`
+        (pvc) => pvc.metadata.name === `datadir-${podName}`
       );
       if (!thePvc) {
         logError(this, `sorry, no pvc found for ${podName}`);
@@ -109,7 +109,7 @@ export default (vorpal: Vorpal) =>
         message:
           "THIS WILL DESTROY THE POD, ITS VOLUME AND ALL ITS DATA 🙀🙀🙀!!! continue? 🤔",
         name: "shouldContinue",
-        type: "confirm"
+        type: "confirm",
       });
 
       if (!shouldContinue) {

package/src/apps/{shell → cli}/commands/mongodb/projectMongoGetShell.ts

@@ -1,24 +1,26 @@
 import Vorpal from "vorpal";
 import { logError } from "../../../../utils/log";
 import { getProjectNamespace } from "../../../../utils/projects";
-import { envAutocompletion } from "../project/utils/autocompletions";
+import { envAndComponents } from "../project/utils/autocompletions";
 import ensureCluster from "../project/utils/ensureCluster";
 import {
   getMongodbShell,
-  getProjectMongodbAllPodsSortedWithLabel
+  getProjectMongodbAllPodsSortedWithLabel,
 } from "./utils";
 
-export default (vorpal: Vorpal) =>
+export default async (vorpal: Vorpal) =>
   vorpal
     .command(
-      "project-mongo-get-shell <env>",
+      "project-mongo-get-shell <envComponent>",
       "get a shell to a mongodb in the environment"
     )
-    .autocomplete(envAutocompletion)
-    .action(async function({ env }) {
-      await ensureCluster.call(this);
-      const namespace = await getProjectNamespace(env);
-      const podNames = await getProjectMongodbAllPodsSortedWithLabel(env);
+    .autocomplete(await envAndComponents())
+    .action(async function ({ envComponent }) {
+      await ensureCluster.call(this, envComponent);
+      const namespace = await getProjectNamespace(envComponent);
+      const podNames = await getProjectMongodbAllPodsSortedWithLabel(
+        envComponent
+      );
       if (podNames.length === 0) {
         logError(this, "sorry, no pods found");
         return;
@@ -32,7 +34,7 @@ export default (vorpal: Vorpal) =>
             type: "list",
             name: "podName",
             choices: podNames,
-            message: "Which pod? 🤔"
+            message: "Which pod? 🤔",
           })
         ).podName;
       }

package/src/apps/cli/commands/mongodb/projectMongoPortForward.ts

@@ -0,0 +1,62 @@
+import Vorpal from "vorpal";
+import { getEnvVars, parseChoice } from "../../../../config/getProjectConfig";
+import { logError } from "../../../../utils/log";
+import { startPortForward } from "../../../../utils/portForward";
+import { getProjectNamespace } from "../../../../utils/projects";
+import { envAndComponents } from "../project/utils/autocompletions";
+import ensureCluster from "../project/utils/ensureCluster";
+import { getProjectMongodbAllPodsSortedWithLabel } from "./utils";
+import clipboard from "clipboardy";
+
+export default async (vorpal: Vorpal) =>
+  vorpal
+    .command(
+      "project-mongo-port-forward <envComponent>",
+      "port foward to a mongodb"
+    )
+    .autocomplete(await envAndComponents())
+    .action(async function ({ envComponent }) {
+      await ensureCluster.call(this, envComponent);
+      const namespace = await getProjectNamespace(envComponent);
+      const podNames = await getProjectMongodbAllPodsSortedWithLabel(
+        envComponent
+      );
+      if (podNames.length === 0) {
+        logError(this, "sorry, no pods found");
+        return;
+      }
+      let podName;
+      if (podNames.length === 1) {
+        podName = podNames[0].value;
+      } else {
+        podName = (
+          await this.prompt({
+            type: "list",
+            name: "podName",
+            choices: podNames,
+            message: "Which pod? 🤔",
+          })
+        ).podName;
+      }
+
+      const { localPort } = await this.prompt({
+        type: "number",
+        name: "localPort",
+        default: "30000",
+        message: "Local port: ",
+      });
+      const { env, componentName } = parseChoice(envComponent);
+      const envVars = await getEnvVars(this, env, componentName);
+      const MONGODB_ROOT_PASSWORD = envVars?.MONGODB_ROOT_PASSWORD;
+      const connectionUrl = `mongodb://root:${MONGODB_ROOT_PASSWORD}@localhost:${localPort}`;
+      clipboard.writeSync(connectionUrl);
+      this.log("");
+      this.log("username: root");
+      this.log(`password: ${MONGODB_ROOT_PASSWORD}`);
+      this.log(`connection string: ${connectionUrl}`);
+      this.log("");
+      this.log("👆 connection string has been copied to your clipboard!");
+      this.log("");
+
+      return startPortForward(podName, localPort, 27017, namespace);
+    });

package/src/apps/{shell → cli}/commands/mongodb/utils/index.ts

@@ -1,15 +1,19 @@
 import { exec, spawn } from "child-process-promise";
-import { Env } from "../../../../../types/types";
 import {
   getProjectNamespace,
   getProjectPodNames,
 } from "../../../../../utils/projects";
 
 const filterMongoDbs = (podNames: string[]) =>
-  podNames.filter((name) => name.includes("mongodb-replicaset"));
+  podNames.filter(
+    (name) =>
+      name.includes("mongodb") &&
+      !name.includes("mongodb-backup") &&
+      !name.includes("arbiter")
+  );
 
-export const getProjectMongodbAllPods = async (env: Env) =>
-  filterMongoDbs(await getProjectPodNames(env));
+export const getProjectMongodbAllPods = async (envComponent: string) =>
+  filterMongoDbs(await getProjectPodNames(envComponent));
 
 export const getMongodbShell = async (namespace: string, podName: string) => {
   const command = `kubectl exec -it ${podName} --namespace ${namespace} mongo`;
@@ -44,23 +48,28 @@ export const executeMongodbCommand = async (
 };
 
 export const podIsMaster = async (namespace: string, podName: string) => {
-  const result = await executeMongodbCommand(
-    namespace,
-    podName,
-    "db.isMaster()"
-  );
+  try {
+    const result = await executeMongodbCommand(
+      namespace,
+      podName,
+      "db.isMaster()"
+    );
 
-  return result.ismaster;
+    return result.ismaster;
+  } catch (e) {
+    // maybe shutting down ?
+    return null;
+  }
 };
 
 const spaces = (n: number) => " ".repeat(n);
 
-export const getMongoDbPodsWithReplInfo = async (env: Env) => {
-  const namespace = await getProjectNamespace(env);
+export const getMongoDbPodsWithReplInfo = async (envComponent: string) => {
+  const namespace = await getProjectNamespace(envComponent);
   return (
     await Promise.all(
       (
-        await getProjectMongodbAllPods(env)
+        await getProjectMongodbAllPods(envComponent)
       ).map(async (podName) => ({
         podName,
         componentName: podName.replace(/-mongodb-replicaset-[0-9]+/, ""),
@@ -70,8 +79,10 @@ export const getMongoDbPodsWithReplInfo = async (env: Env) => {
   ).sort((podA, podB) => (podA.isMaster ? (podB.isMaster ? 0 : -1) : 1));
 };
 
-export const getProjectMongodbAllPodsSortedWithLabel = async (env: Env) => {
-  const pods = await getMongoDbPodsWithReplInfo(env);
+export const getProjectMongodbAllPodsSortedWithLabel = async (
+  envComponent: string
+) => {
+  const pods = await getMongoDbPodsWithReplInfo(envComponent);
   const maxComponentNameLength = Math.max(
     ...pods.map((c) => c.componentName.length)
   );

package/src/apps/cli/commands/project/commandCloudSqlProxy.ts

@@ -0,0 +1,88 @@
+import { isOfDeployType } from "@catladder/pipeline";
+import { spawn } from "child-process-promise";
+import { writeFile } from "fs-extra";
+import { withFile } from "tmp-promise";
+import Vorpal from "vorpal";
+import {
+  getEnvVars,
+  getGitlabVar,
+  getPipelineContextByChoice,
+  getProjectConfig,
+  parseChoice,
+} from "../../../../config/getProjectConfig";
+import { envAndComponents } from "./utils/autocompletions";
+
+export default async (vorpal: Vorpal) =>
+  vorpal
+    .command("project-cloud-sql-proxy <envComponent>", "proxy to cloud sql db")
+    .autocomplete(await envAndComponents())
+    .action(async function ({ envComponent }) {
+      const { env, componentName } = parseChoice(envComponent);
+
+      const config = await getProjectConfig();
+      // skynet-164509:europe-west6:pvl-cyclomania-review=tcp:5432
+
+      const { localPort } = await this.prompt({
+        type: "number",
+        name: "localPort",
+        default: "54320",
+        message: "Local port: ",
+      });
+
+      const envVars = await getEnvVars(this, env, componentName);
+      const POSTGRESQL_PASSWORD = envVars?.POSTGRESQL_PASSWORD;
+
+      const context = await getPipelineContextByChoice(env, componentName);
+      if (!isOfDeployType(context.componentConfig.deploy, "kubernetes")) {
+        throw new Error("currently only supported for kubernetes deployment");
+      }
+      this.log("");
+      this.log(`postgres-PW: ${POSTGRESQL_PASSWORD}`);
+      this.log("");
+      this.log(
+        `POSTGRESQL_URL=postgresql://postgres:${POSTGRESQL_PASSWORD}@localhost:${localPort}/${context.environment.envVars.KUBE_APP_NAME}?schema=public`
+      );
+      this.log("");
+
+      const values = context.componentConfig.deploy.values;
+
+      const projectId =
+        values?.cloudsql?.projectId ||
+        context.componentConfig.deploy.cluster?.projectId;
+
+      const defaultInstanceId = `${config.customerName}-${config.appName}-${env}`;
+      const instanceId = values?.cloudsql?.instanceId || defaultInstanceId;
+
+      const defaultRegion = "europe-west6"; // currently hardcoded
+      const region = values?.cloudsql?.region || defaultRegion;
+
+      const instanceName = `${projectId}:${region}:${instanceId}=tcp:${localPort}`;
+
+      const cloudsqlProxyCredentials = await getGitlabVar(
+        this,
+        env,
+        componentName,
+        "cloudsqlProxyCredentials"
+      );
+
+      if (!cloudsqlProxyCredentials) {
+        // we store cloudsqlProxyCredentials on gitlab, but its currently get pushed via bitwarden due to legacy reasons
+        // this will be fixed with when https://git.panter.ch/catladder/catladder/-/merge_requests/32/ is merged
+        this.log(
+          "cloudsqlProxyCredentials env var missing in gitlab. Please contact gilde-ci-cd about that."
+        );
+        throw new Error("cloudsqlProxyCredentials missing in secrets");
+      }
+      await withFile(async ({ path: tmpFilePath }) => {
+        await writeFile(tmpFilePath, cloudsqlProxyCredentials);
+
+        await spawn(
+          "cloud_sql_proxy",
+          ["-instances", instanceName, "-credential_file", tmpFilePath],
+          {
+            stdio: "inherit",
+            shell: true,
+          }
+        );
+      });
+    });

package/src/apps/cli/commands/project/commandConfigSecrets.ts

@@ -0,0 +1,200 @@
+/* eslint-disable no-constant-condition */
+import { getSecretVarName } from "@catladder/pipeline";
+import { stripIndents } from "common-tags";
+import { difference } from "lodash";
+import Vorpal, { CommandInstance } from "vorpal";
+import {
+  getAllComponentsWithAllEnvsHierarchical,
+  getEnvironment,
+  getEnvVars,
+  getProjectComponents,
+  parseChoice,
+} from "../../../../config/getProjectConfig";
+import { editAsFile } from "../../../../utils/editAsFile";
+import { upsertAllVariables } from "../../../../utils/gitlab";
+import { delay } from "../../../../utils/promise";
+import { allEnvsAndAllComponents } from "./utils/autocompletions";
+
+type Vars = {
+  [env: string]: {
+    [componentName: string]: Record<string, string>;
+  };
+};
+/* for convenience, parse json objects. that makes it easier to edit secrets that are object */
+const resolveJson = (v: Vars) =>
+  Object.fromEntries(
+    Object.entries(v).map(([componentName, envs]) => {
+      return [
+        componentName,
+        Object.fromEntries(
+          Object.entries(envs).map(([env, secrets]) => [
+            env,
+            Object.fromEntries(
+              Object.entries(secrets).map(([key, value]) => {
+                try {
+                  return [key, JSON.parse(value)];
+                } catch (e) {
+                  return [key, value];
+                }
+              })
+            ),
+          ])
+        ),
+      ];
+    })
+  );
+
+const getEnvVarsToEdit = async (
+  instance: CommandInstance,
+  env: string,
+  componentName: string
+) => {
+  const { secretEnvVarKeys } = await getEnvironment(env, componentName);
+
+  const allEnvVars = await getEnvVars(instance, env, componentName);
+  return Object.fromEntries(
+    secretEnvVarKeys.map((key) => {
+      const value = allEnvVars[key];
+      // due to some quirky way to resolve these variables, unset variables have the $CL_ prefix, so we remove thouse here
+      const variableIsNotSet =
+        value === "$" + getSecretVarName(env, componentName, key);
+      return [key, variableIsNotSet ? "🚨 FILL ME" : value];
+    })
+  );
+};
+const doItFor = async (
+  instance: CommandInstance,
+  envAndComponents: {
+    [componentName: string]: string[];
+  }
+) => {
+  let valuesToEdit: Vars = Object.fromEntries(
+    await Promise.all(
+      Object.entries(envAndComponents).map(async ([componentName, envs]) => [
+        componentName,
+        Object.fromEntries(
+          await Promise.all(
+            envs.map(async (env) => [
+              env,
+              await getEnvVarsToEdit(instance, env, componentName),
+            ])
+          )
+        ),
+      ])
+    )
+  );
+  let hasErrors = true;
+  while (hasErrors) {
+    valuesToEdit = await editAsFile(
+      resolveJson(valuesToEdit),
+      stripIndents`
+        Please fill in all secrets for: 
+
+        ${Object.entries(envAndComponents)
+          .map(
+            ([componentName, envs]) => `- ${componentName}: ${envs.join(", ")}`
+          )
+          .join("\n")}
+
+        `
+    );
+    // check for errors
+    hasErrors = false;
+    for (const [componentName, envs] of Object.entries(envAndComponents)) {
+      for (const env of envs) {
+        const usedKeys = valuesToEdit[componentName][env]
+          ? Object.keys(valuesToEdit[componentName][env])
+          : [];
+        // check whether newValues have the exact number of keys
+        const { secretEnvVarKeys } = await getEnvironment(env, componentName);
+        const extranous = difference(usedKeys, secretEnvVarKeys);
+        const missing = difference(secretEnvVarKeys, usedKeys);
+
+        if (extranous.length > 0 || missing.length > 0) {
+          instance.log("");
+          instance.log(
+            `😿 Oh no! There is something wrong with "${componentName}"`
+          );
+          instance.log("");
+          if (extranous.length > 0) {
+            instance.log("these secrets are not declared in the config");
+            extranous.forEach((key) => instance.log(key));
+            instance.log("");
+          }
+          if (missing.length > 0) {
+            instance.log("these secrets have not been provided:");
+            missing.forEach((key) => instance.log(key));
+            instance.log("");
+          }
+
+          await delay(1000);
+          const { shouldContinue } = await instance.prompt({
+            default: true,
+            message: "Try again? 🤔",
+            name: "shouldContinue",
+            type: "confirm",
+          });
+
+          if (!shouldContinue) {
+            throw new Error("abort");
+          }
+          hasErrors = true;
+        }
+      }
+    }
+  }
+  instance.log("upserting all variables, please wait...");
+  instance.log("");
+  for (const [componentName, envs] of Object.entries(envAndComponents)) {
+    for (const env of envs) {
+      await upsertAllVariables(
+        instance,
+        valuesToEdit[componentName][env],
+        env,
+        componentName
+      );
+      instance.log("");
+      instance.log("✅ " + env + ":" + componentName);
+    }
+  }
+  instance.log("done! 😻");
+  instance.log("");
+};
+
+export const projectConfigSecrets = async (
+  vorpal: CommandInstance,
+  envComponent?: string
+) => {
+  if (!envComponent) {
+    const allEnvAndcomponents = await getAllComponentsWithAllEnvsHierarchical();
+    await doItFor(vorpal, allEnvAndcomponents);
+  } else {
+    const { env, componentName } = parseChoice(envComponent);
+
+    // componentName can be null. in this case, iterate over all  components
+    if (!componentName) {
+      const components = await getProjectComponents();
+      await doItFor(
+        vorpal,
+        Object.fromEntries(components.map((c) => [c, [env]]))
+      );
+    }
+    if (componentName) {
+      await doItFor(vorpal, {
+        [componentName]: [env],
+      });
+    }
+  }
+};
+
+export default async (vorpal: Vorpal) => {
+  vorpal
+    .command(
+      "project-config-secrets [envComponent]",
+      "setup/update secrets stored in pass"
+    )
+    .autocomplete(await allEnvsAndAllComponents())
+    .action(async function ({ envComponent }) {
+      return await projectConfigSecrets(this, envComponent);
+    });
+};

package/src/apps/{shell → cli}/commands/project/commandCopyDB.ts

@@ -1,27 +1,23 @@
 import { spawn } from "child-process-promise";
 import Vorpal from "vorpal";
 import {
-  GOOGLE_CLOUD_SQL_PASS_PATH,
-  GOOGLE_PROJECT,
-} from "../../../../config/constants";
+  getEnvVars,
+  getProjectConfig,
+  parseChoice,
+} from "../../../../config/getProjectConfig";
 import { readPass } from "../../../../utils/passwordstore";
-import {
-  getAllEnvVars,
-  getLocalProjectVariables,
-} from "../../../../utils/projects";
-import { envAutocompletion } from "./utils/autocompletions";
-import { promptForSubAppIfAny } from "./utils/monorepo";
+import { envAndComponents } from "./utils/autocompletions";
 
-export default (vorpal: Vorpal) =>
+export default async (vorpal: Vorpal) =>
   vorpal
-    .command("copy-db <env>", "replace local db with the one from an env")
-    .autocomplete(envAutocompletion)
-    .action(async function copyDB({ env }) {
-      const {
-        CUSTOMER_NAME,
-        APP_NAME,
-        COMPONENT_NAME = "web",
-      } = await getLocalProjectVariables();
+    .command(
+      "copy-db <envComponent>",
+      "replace local db with the one from an env"
+    )
+    .autocomplete(await envAndComponents())
+    .action(async function copyDB({ envComponent }) {
+      const { env, componentName } = parseChoice(envComponent);
+      const { customerName, appName } = await getProjectConfig();
 
       const { shouldContinue } = await this.prompt({
         type: "confirm",
@@ -34,15 +30,19 @@ export default (vorpal: Vorpal) =>
         return;
       }
 
-      const subapp = await promptForSubAppIfAny(this);
-
+      // TODO: reimpleent with new config
+      /*
+      throw new Error("needs reimplementation with new config");
       const GOOGLE_CLOUD_SQL_REGION = "europe-west6"; // currently hardcoded
-      const POSTGRESQL_PASSWORD = (await getAllEnvVars(env, subapp))
-        ?.POSTGRESQL_PASSWORD;
+      const { POSTGRESQL_PASSWORD } = await getEnvVars(
+        this,
+        env,
+        componentName
+      );
 
       const LOCAL_PORT = 54321;
 
-      const instanceName = `${GOOGLE_PROJECT}:${GOOGLE_CLOUD_SQL_REGION}:${CUSTOMER_NAME}-${APP_NAME}-${env}=tcp:${LOCAL_PORT}`;
+      const instanceName = `${"xxxxxx reimplement shoulld be project id"}:${GOOGLE_CLOUD_SQL_REGION}:${customerName}-${appName}-${env}=tcp:${LOCAL_PORT}`;
       const cloudsqlCredentials = await readPass(GOOGLE_CLOUD_SQL_PASS_PATH);
 
       const { POSTGRESQL_URL } = process.env;
@@ -72,9 +72,7 @@ export default (vorpal: Vorpal) =>
       dumptmp=$(mktemp /tmp/dump.XXXXXX)
 
       echo "Dumping file to $dumptmp"
-      pg_dump --dbname=postgres://postgres:${POSTGRESQL_PASSWORD}@localhost:${LOCAL_PORT}/${
-        subapp ?? COMPONENT_NAME
-      } --no-owner --no-privileges > $dumptmp
+      pg_dump --dbname=postgres://postgres:${POSTGRESQL_PASSWORD}@localhost:${LOCAL_PORT}/${componentName} --no-owner --no-privileges > $dumptmp
       psql -q -c "drop database ${localDBName}" 1> /dev/null
       psql -q -c "create database ${localDBName}" 1> /dev/null
       echo "Restoring dump..."
@@ -90,4 +88,5 @@ export default (vorpal: Vorpal) =>
       `;
 
       await spawn(copyDBScript, [], { shell: "bash", stdio: "inherit" });
+      */
     });